99 lines
3.1 KiB
Python
99 lines
3.1 KiB
Python
|
"""
|
|||
|
|
测试百度搜索爬虫
|
|||
|
|
"""
|
|||
|
|
|
|||
|
|
from loguru import logger
|
|||
|
|
from baidu_crawler import BaiduSearchCrawler
|
|||
|
|
from db_manager import QueryTaskManager
|
|||
|
|
from datetime import datetime
|
|||
|
|
import sys
|
|||
|
|
|
|||
|
|
logger.remove()
|
|||
|
|
logger.add(sys.stdout, format="<green>{time:HH:mm:ss}</green> | <level>{level: <8}</level> | <level>{message}</level>")
|
|||
|
|
|
|||
|
|
|
|||
|
|
def test_single_query():
|
|||
|
|
"""测试爬取单个查询词"""
|
|||
|
|
print("="*70)
|
|||
|
|
print(" 测试爬取单个查询词")
|
|||
|
|
print("="*70)
|
|||
|
|
|
|||
|
|
# headless=False可以看到浏览器
|
|||
|
|
# 会自动滚动直到无新内容
|
|||
|
|
crawler = BaiduSearchCrawler(headless=False)
|
|||
|
|
|
|||
|
|
# 测试查询，设置阈值50
|
|||
|
|
query_word = "糖尿病治疗"
|
|||
|
|
result = crawler.crawl_query(query_word, category="医疗", threshold_max=50)
|
|||
|
|
|
|||
|
|
print("\n爬取结果:")
|
|||
|
|
print(f" 查询词: {result['query_word']}")
|
|||
|
|
print(f" 是否成功: {result['success']}")
|
|||
|
|
print(f" 爬取数量: {result['crawled_count']}")
|
|||
|
|
print(f" 有效数量: {result['valid_count']}")
|
|||
|
|
print(f" 新增数量: {result['new_count']}")
|
|||
|
|
if result['error']:
|
|||
|
|
print(f" 错误信息: {result['error']}")
|
|||
|
|
|
|||
|
|
|
|||
|
|
def test_batch_crawl():
|
|||
|
|
"""测试批量爬取任务"""
|
|||
|
|
print("="*70)
|
|||
|
|
print(" 测试批量爬取任务")
|
|||
|
|
print("="*70)
|
|||
|
|
|
|||
|
|
# 先创建一些测试任务
|
|||
|
|
task_mgr = QueryTaskManager()
|
|||
|
|
task_date = datetime.now().strftime('%Y%m%d')
|
|||
|
|
|
|||
|
|
test_queries = [
|
|||
|
|
("高血压怎么治疗", "keyword", "医疗", 3, 30), # 阈值30
|
|||
|
|
("在线教育平台哪个好", "phrase", "教育", 5, 20), # 阈值20
|
|||
|
|
("免费法律咨询", "keyword", "法律", 4, 25), # 阈值25
|
|||
|
|
]
|
|||
|
|
|
|||
|
|
logger.info("创建测试任务...")
|
|||
|
|
for query, qtype, category, priority, threshold in test_queries:
|
|||
|
|
task_mgr.create_task(
|
|||
|
|
query_word=query,
|
|||
|
|
query_type=qtype,
|
|||
|
|
task_date=task_date,
|
|||
|
|
threshold_max=threshold, # 使用各自的阈值
|
|||
|
|
priority=priority,
|
|||
|
|
category=category,
|
|||
|
|
remark="测试任务"
|
|||
|
|
)
|
|||
|
|
|
|||
|
|
print()
|
|||
|
|
|
|||
|
|
# 执行批量爬取，会自动滚动直到达到阈值
|
|||
|
|
crawler = BaiduSearchCrawler(headless=False)
|
|||
|
|
stats = crawler.crawl_tasks(limit=3)
|
|||
|
|
|
|||
|
|
print("\n批量爬取统计:")
|
|||
|
|
print(f" 总任务数: {stats['total_tasks']}")
|
|||
|
|
print(f" 成功: {stats['success_count']}")
|
|||
|
|
print(f" 失败: {stats['failed_count']}")
|
|||
|
|
print(f" 总爬取: {stats['total_crawled']}")
|
|||
|
|
print(f" 新增保存: {stats['total_saved']}")
|
|||
|
|
|
|||
|
|
|
|||
|
|
if __name__ == "__main__":
|
|||
|
|
import argparse
|
|||
|
|
|
|||
|
|
parser = argparse.ArgumentParser(description='测试百度搜索爬虫')
|
|||
|
|
parser.add_argument('--mode', choices=['single', 'batch'], default='single',
|
|||
|
|
help='测试模式：single=单个查询, batch=批量任务')
|
|||
|
|
|
|||
|
|
args = parser.parse_args()
|
|||
|
|
|
|||
|
|
try:
|
|||
|
|
if args.mode == 'single':
|
|||
|
|
test_single_query()
|
|||
|
|
else:
|
|||
|
|
test_batch_crawl()
|
|||
|
|
except Exception as e:
|
|||
|
|
logger.error(f"测试失败: {str(e)}")
|
|||
|
|
import traceback
|
|||
|
|
traceback.print_exc()
|