yunqueai/ai_baijiahao
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity
Files
0d5bbb186473cf510ce36f6ef315f2f8c071edec
ai_baijiahao/app.py

1124 lines
41 KiB
Python
Raw Normal View History

Initial commit: 百家号文章采集系统
2025-12-19 22:48:58 +08:00
# -*- coding: utf-8 -*-
import json
import logging
import os
import re
from datetime import datetime, timedelta
from functools import wraps
import pandas as pd
import requests
from flask import Flask, render_template, request, jsonify, send_file, session, redirect, url_for
from flask_cors import CORS
from flask_socketio import SocketIO, emit
from task_queue import get_task_queue
from task_worker import start_task_worker
logging.basicConfig(level=logging.INFO)
logger = logging.getLogger(__name__)
app = Flask(__name__, static_folder='static', template_folder='templates')
app.secret_key = 'your-secret-key-change-this-in-production' # 请在生产环境中修改为随机密钥
CORS(app)
# 初始化 SocketIO
socketio = SocketIO(app, cors_allowed_origins="*", async_mode='threading')
# 简单的用户数据库（生产环境应使用数据库）
USERS = {
'admin': 'admin123', # 用户名: 密码
}
# 登录验证装饰器
def login_required(f):
@wraps(f)
def decorated_function(*args, **kwargs):
if 'username' not in session:
return jsonify({'success': False, 'message': '请先登录', 'need_login': True}), 401
return f(*args, **kwargs)
return decorated_function
class BaijiahaoScraper:
def __init__(self, uk, cookies=None, use_proxy=False, proxy_api_url=None):
self.uk = uk
self.api_url = 'https://mbd.baidu.com/webpage'
self.headers = {
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/142.0.0.0 Safari/537.36',
'Referer': 'https://baijiahao.baidu.com/',
'Accept': '*/*',
}
self.session = requests.Session()
self.session.keep_alive = False # 禁用长连接
if cookies:
self.session.cookies.update(cookies)
# 代理配置
self.use_proxy = use_proxy
self.proxy_api_url = proxy_api_url or 'http://api.tianqiip.com/getip?secret=lu29e593&num=1&type=txt&port=1&mr=1&sign=4b81a62eaed89ba802a8f34053e2c964'
self.current_proxy = None
def get_proxy(self):
"""从代理池获取一个代理IP"""
try:
print(f"正在从代理池获取IP: {self.proxy_api_url}")
response = requests.get(self.proxy_api_url, timeout=5) # 优化超时为5秒
content = response.content.decode("utf-8").strip()
print(f"代理API响应: {content}")
# 检查是否是JSON错误响应
if content.startswith('{'):
try:
import json
error_data = json.loads(content)
error_msg = error_data.get('msg', '未知错误')
print(f"❌ 代理IP池错误: {error_msg}")
raise Exception(f"代理IP池错误: {error_msg}")
except json.JSONDecodeError:
pass # 不是JSON，继续按IP处理
# 解析IP和端口
if ':' in content:
sj = content.strip().split(":", 1)
sj1 = sj[0] # IP
sj2 = sj[1] # 端口
print(f"IP: {sj1}, 端口: {sj2}")
# 构建代理配置
proxy_meta = f"http://{sj1}:{sj2}"
proxies = {
'http': proxy_meta,
'https': proxy_meta
}
self.current_proxy = proxies
print(f"代理配置成功: {proxies}")
return proxies
else:
print("代理IP格式错误")
raise Exception(f"代理IP格式错误: {content}")
except Exception as e:
print(f"获取代理IP失败: {e}")
raise
def make_request(self, url, **kwargs):
"""使用代理发起请求，失败时自动切换IP"""
max_retries = 3 # 最大重试次数
retry_count = 0
last_was_anti_crawl = False # 标记上次是否因反爬重试
while retry_count < max_retries:
if self.use_proxy:
# 如果没有当前代理或需要刷新，获取新代理
if not self.current_proxy or retry_count > 0:
print(f"{'立即切换' if retry_count > 0 else ''}获取代理IP（第{retry_count + 1}次）")
proxy = self.get_proxy()
if not proxy:
raise Exception("启用了代理但无法获取代理IP，拒绝使用本机IP")
# 必须有代理才请求
if not self.current_proxy:
raise Exception("启用了代理但当前无代理IP，拒绝使用本机IP")
kwargs['proxies'] = self.current_proxy
# 设置超时
if 'timeout' not in kwargs:
kwargs['timeout'] = 20
try:
import time
start = int(round(time.time() * 1000))
response = self.session.get(url, **kwargs)
cost_time = int(round(time.time() * 1000)) - start
print(f"请求耗时: {cost_time}ms")
# 检查是否被反爬
if self._check_anti_spider(response):
print("⚠️ 检测到反爬，立即切换IP（无需等待）")
self.current_proxy = None # 清空当前代理
retry_count += 1
last_was_anti_crawl = True
if retry_count < max_retries:
continue # 立即重试，不等待
else:
raise Exception("多次重试后仍被反爬拦截，请稍后再试")
return response
except requests.exceptions.ProxyError as e:
print(f"代理错误: {e}，立即切换IP（无需等待）")
self.current_proxy = None
retry_count += 1
last_was_anti_crawl = True
if retry_count < max_retries:
continue # 立即重试，不等待
else:
raise
except requests.exceptions.HTTPError as e:
# 检查是否是407错误（代理IP池限流）
if e.response and e.response.status_code == 407:
print(f"⚠️ 检测到407错误（代理IP池限流），等待3秒后重新获取IP...")
import time
time.sleep(3) # 等待3秒
if self.use_proxy:
print("重新获取代理IP...")
self.current_proxy = None
retry_count += 1
if retry_count < max_retries:
continue
raise
else:
print(f"请求失败: {e}")
raise
except Exception as e:
print(f"请求失败: {e}")
# 如果使用代理失败，尝试重新获取代理
if self.use_proxy:
print("立即切换代理（无需等待）")
self.current_proxy = None
retry_count += 1
if retry_count < max_retries:
continue # 立即重试，不等待
raise
raise Exception(f"请求失败，已重试{max_retries}次")
def _check_anti_spider(self, response):
"""检查响应是否被反爬拦截"""
# 检查状态码
if response.status_code in [403, 429, 503]:
return True
# 尝试解析JSON响应，检查is_need_foe字段
try:
data = response.json()
if data.get('data', {}).get('foe', {}).get('is_need_foe') == True:
print("检测到is_need_foe=True，需要切换IP")
return True
except:
pass # 如果不是JSON响应，继续检查文本内容
# 检查响应内容中的反爬特征
content = response.text.lower()
anti_spider_keywords = [
'验证码',
'captcha',
'请输入验证码',
'访问频繁',
'异常访问',
'请稍后再试',
'access denied',
'forbidden',
'安全验证',
'人机验证'
]
for keyword in anti_spider_keywords:
if keyword in content:
print(f"检测到反爬关键词: {keyword}")
return True
return False
@staticmethod
def get_uk_from_app_id(app_id, use_proxy=False, proxy_api_url=None):
"""从baijiahao主页提取uk参数"""
url = f"https://baijiahao.baidu.com/u?app_id={app_id}"
headers = {
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
}
print(f"正在从主页获取uk: {url}")
max_retries = 3 # 最大重试次数
retry_count = 0
while retry_count < max_retries:
# 准备代理配置
kwargs = {'headers': headers, 'timeout': 30}
if use_proxy:
proxy_url = proxy_api_url or 'http://api.tianqiip.com/getip?secret=lu29e593&num=1&type=txt&port=1&mr=1&sign=4b81a62eaed89ba802a8f34053e2c964'
try:
print(f"{'重新' if retry_count > 0 else ''}使用代理获取uk（第{retry_count + 1}次）")
proxy_response = requests.get(proxy_url, timeout=5) # 优化超时为5秒
content = proxy_response.content.decode("utf-8").strip()
print(f"代理API响应: {content}")
# 检查是否是JSON错误响应
if content.startswith('{'):
try:
import json
error_data = json.loads(content)
error_msg = error_data.get('msg', '未知错误')
raise Exception(f"代理IP池错误: {error_msg}")
except json.JSONDecodeError:
pass # 不是JSON，继续按IP处理
if ':' in content:
sj = content.strip().split(":", 1)
proxy_meta = f"http://{sj[0]}:{sj[1]}"
kwargs['proxies'] = {
'http': proxy_meta,
'https': proxy_meta
}
print(f"使用代理: {proxy_meta}")
else:
raise Exception(f"代理IP格式错误: {content}")
except Exception as e:
# 启用了代理但获取失败，不使用本机IP，直接抛出异常
raise Exception(f"启用了代理但获取代理IP失败，拒绝使用本机IP: {e}")
try:
response = requests.get(url, **kwargs)
if response.status_code != 200:
print(f"访问主页失败，状态码: {response.status_code}")
retry_count += 1
if retry_count < max_retries:
print(f"立即重试（无需等待）")
continue # 立即重试
else:
raise Exception(f"访问主页失败，状态码: {response.status_code}")
# 检查是否被反爬
content_lower = response.text.lower()
anti_spider_detected = False
# 检查is_need_foe字段
try:
data = response.json()
if data.get('data', {}).get('foe', {}).get('is_need_foe') == True:
print("⚠️ 检测到is_need_foe=True，需要切换IP")
anti_spider_detected = True
except:
pass # 不是JSON响应，继续检查其他特征
# 检查反爬关键词
if not anti_spider_detected:
anti_spider_keywords = ['验证码', 'captcha', '访问频繁', '异常访问', 'access denied']
for keyword in anti_spider_keywords:
if keyword in content_lower:
print(f"⚠️ 检测到反爬关键词: {keyword}")
anti_spider_detected = True
break
if anti_spider_detected and use_proxy:
retry_count += 1
if retry_count < max_retries:
print(f"检测到反爬，立即切换IP重试（无需等待）")
continue # 立即重试，不等待
else:
raise Exception("多次重试后仍被反爬拦截")
# 使用正则表达式提取uk
uk_match = re.search(r'"uk"\s*:\s*"([^"]+)"', response.text)
if not uk_match:
# 尝试另一种模式
uk_match = re.search(r'uk=([^&\s"]+)', response.text)
if uk_match:
uk = uk_match.group(1)
print(f">> 成功获取UK: {uk}")
# 获取Cookie
cookies = response.cookies
print(f">> 成功获取Cookie")
return uk, cookies
else:
raise Exception("无法从页面中提取UK参数")
except requests.exceptions.ProxyError as e:
print(f"代理错误: {e}，立即切换代理重试（无需等待）")
retry_count += 1
if retry_count < max_retries and use_proxy:
continue # 立即重试，不等待
else:
raise
except requests.exceptions.HTTPError as e:
# 检查是否是407错误（代理IP池限流）
if e.response and e.response.status_code == 407:
print(f"⚠️ 检测到407错误（代理IP池限流），等待3秒后重新获取IP...")
import time
time.sleep(3) # 等待3秒
retry_count += 1
if retry_count < max_retries and use_proxy:
print("重新获取代理IP，继续重试...")
continue
raise
else:
print(f"请求失败: {e}")
raise
except Exception as e:
if retry_count < max_retries - 1:
retry_count += 1
print(f"错误: {e}，立即重试（无需等待）")
continue # 立即重试，不等待
else:
print(f"获取UK失败: {e}")
raise
def get_articles(self, months=6, app_id=None, articles_only=True, task_id=None, on_page_fetched=None,
start_page=1, start_ctime=None):
"""使用API接口获取文章列表(调用baidu_api.py的get_baidu_data_sync)
Args:
months: 获取近几个月的数据
app_id: 百家号app_id
articles_only: 是否仅爬取文章（跳过视频）
task_id: 任务ID（用于数据库缓存）
on_page_fetched: 每页数据获取后的回调函数
start_page: 起始页码（断点续传）
start_ctime: 起始分页参数（断点续传）
Returns:
dict: {'last_page': 最后页码, 'last_ctime': 分页参数, 'completed': 是否完成}
"""
import sys
import os
# 导入baidu_api.py中的同步函数
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
from baidu_api import get_baidu_data_sync
from datetime import datetime, timedelta
# 支持小数月份(如0.33个月10天)
days = int(months * 30)
target_date = datetime.now() - timedelta(days=days)
print(f"\n{'=' * 50}")
print(f"开始获取百家号作者(uk={self.uk})的文章...")
if start_page > 1:
print(f"🔄 断点续传：从第{start_page}页开始")
print(f"{'=' * 50}\n")
# 定义处理每页数据的回调函数
def process_page(page, items, ctime):
"""处理每页的原始数据，提取文章并调用上层回调"""
processed_articles = []
for item in items:
item_data = item.get('itemData', {})
# 如果启用articles_only，过滤掉视频内容
if articles_only:
meta_type = item_data.get('meta_type', '')
if meta_type == 'video':
print(f" ✖ 跳过视频: {item_data.get('title', '[无标题]')[:50]}...")
continue
title = item_data.get('title', '')
article_url = item_data.get('url', '')
# 优先使用ctime(Unix时间戳),更准确
publish_time = '未知时间'
if 'ctime' in item_data and item_data['ctime']:
try:
timestamp = int(item_data['ctime'])
article_date = datetime.fromtimestamp(timestamp)
publish_time = article_date.strftime('%Y-%m-%d %H:%M:%S')
except Exception as e:
print(f" ✗ 解析ctime失败: {e}")
time_str = item_data.get('time', '未知时间')
if time_str != '未知时间':
article_date = self._parse_article_date(time_str)
if article_date:
publish_time = article_date.strftime('%Y-%m-%d %H:%M:%S')
else:
time_str = item_data.get('time', '未知时间')
if time_str != '未知时间':
article_date = self._parse_article_date(time_str)
if article_date:
publish_time = article_date.strftime('%Y-%m-%d %H:%M:%S')
processed_articles.append({
'标题': title.strip().split('\n')[0][:500] if title else '无标题',
'链接': article_url,
'发布时间': publish_time
})
print(f"第{page}页处理完成：{len(processed_articles)}篇文章")
# 调用上层回调（如果提供）
if on_page_fetched:
on_page_fetched(page, processed_articles, ctime)
try:
# 调用爬虫，传入回调函数
if months < 1:
print(f"调用get_baidu_data_sync获取数据(近{days}天)...\n")
else:
print(f"调用get_baidu_data_sync获取数据(近{int(months)}个月)...\n")
# 传递代理参数和回调函数
result = get_baidu_data_sync(
uk=self.uk,
months=months,
use_proxy=self.use_proxy,
proxy_api_url=self.proxy_api_url if self.proxy_api_url else None,
on_page_fetched=process_page,
start_page=start_page,
start_ctime=start_ctime
)
if not result:
print("\n✗ 未获取到数据")
return {'last_page': start_page, 'last_ctime': start_ctime, 'completed': False}
print(f"\n✓ 爬取完成！")
return result
except Exception as e:
print(f"\n✗ 调用get_baidu_data_sync失败: {e}")
import traceback
traceback.print_exc()
return {'last_page': start_page, 'last_ctime': start_ctime, 'completed': False}
def _parse_article_date(self, time_str):
"""解析文章时间字符串,返回datetime对象"""
from datetime import datetime, timedelta
import re
if not time_str or time_str == '未知':
return None
current_year = datetime.now().year
now = datetime.now()
# 格式1: 相对时间 "1天前", "2小时前", "30分钟前", "1个月前"
if '前' in time_str:
# 天前
match = re.search(r'(\d+)\s*天', time_str)
if match:
days = int(match.group(1))
return now - timedelta(days=days)
# 小时前
match = re.search(r'(\d+)\s*小时', time_str)
if match:
hours = int(match.group(1))
return now - timedelta(hours=hours)
# 分钟前
match = re.search(r'(\d+)\s*分钟', time_str)
if match:
minutes = int(match.group(1))
return now - timedelta(minutes=minutes)
# 月前
match = re.search(r'(\d+)\s*个?月', time_str)
if match:
months = int(match.group(1))
return now - timedelta(days=months * 30) # 近似计算
# 格式2: "11-29 07:23" 或 "11-29"
match = re.match(r'(\d{1,2})-(\d{1,2})', time_str)
if match:
month = int(match.group(1))
day = int(match.group(2))
return datetime(current_year, month, day)
# 格式3: "2024-11-29"
match = re.match(r'(\d{4})-(\d{1,2})-(\d{1,2})', time_str)
if match:
year = int(match.group(1))
month = int(match.group(2))
day = int(match.group(3))
return datetime(year, month, day)
return None
def get_articles_from_html(self, months=6, app_id=None):
"""直接解析HTML获取文章(不依赖Selenium)"""
if not app_id:
raise Exception("需要提供app_id")
articles = []
try:
url = f"https://baijiahao.baidu.com/u?app_id={app_id}"
print(f"访问页面: {url}")
headers = {
'User-Agent': self.headers['User-Agent'],
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
'Accept-Language': 'zh-CN,zh;q=0.9',
'Connection': 'keep-alive',
}
response = self.session.get(url, headers=headers, timeout=30)
if response.status_code != 200:
raise Exception(f"访问失败,状态码: {response.status_code}")
html = response.text
# 使用简单的字符串查找方式提取文章
# 查找所有包含 /s?id= 的链接
import re
links = re.findall(r'href="(/s\?id=[^"]+)"', html)
links += re.findall(r"href='(/s\?id=[^']+)'", html)
# 尝试提取标题
for link in set(links): # 去重
# 尝试找到这个链接对应的title
title_pattern = f'href="{link}"[^>]*title="([^"]+)"'
title_match = re.search(title_pattern, html)
if not title_match:
title_pattern = f'title="([^"]+)"[^>]*href="{link}"'
title_match = re.search(title_pattern, html)
title = title_match.group(1) if title_match else "未知标题"
full_link = 'https://baijiahao.baidu.com' + link
articles.append({
'标题': title.strip(),
'链接': full_link,
'发布时间': '未知'
})
print(f"成功提取 {len(articles)} 篇文章")
except Exception as e:
print(f"HTML解析失败: {e}")
raise
return articles
# ========== SocketIO 事件处理 ==========
@socketio.on('connect')
def handle_connect():
"""客户端连接时触发"""
logger.info(f"客户端已连接: {request.sid}")
emit('connected', {'message': '已连接到服务器'})
@socketio.on('disconnect')
def handle_disconnect():
"""客户端断开时触发"""
logger.info(f"客户端已断开: {request.sid}")
@socketio.on('subscribe_task')
def handle_subscribe_task(data):
"""客户端订阅任务进度"""
task_id = data.get('task_id')
logger.info(f"客户端 {request.sid} 订阅任务: {task_id}")
emit('subscribed', {'task_id': task_id})
def emit_task_log(task_id, message, level='info'):
"""发送任务日志到前端"""
socketio.emit('task_log', {
'task_id': task_id,
'message': message,
'level': level,
'timestamp': datetime.now().strftime('%Y-%m-%d %H:%M:%S')
})
def emit_task_progress(task_id, progress, current_step='', **kwargs):
"""发送任务进度到前端"""
data = {
'task_id': task_id,
'progress': progress,
'current_step': current_step,
'timestamp': datetime.now().strftime('%Y-%m-%d %H:%M:%S')
}
data.update(kwargs)
socketio.emit('task_progress', data)
# ========== Flask 路由 ==========
@app.route('/')
def index():
"""首页"""
# 如果未登录，重定向到登录页
if 'username' not in session:
return redirect(url_for('login_page'))
return render_template('index.html', username=session.get('username'))
@app.route('/queue')
def queue_page():
"""任务队列页面"""
# 如果未登录，重定向到登录页
if 'username' not in session:
return redirect(url_for('login_page'))
return render_template('queue.html', username=session.get('username'))
@app.route('/login')
def login_page():
"""登录页面"""
# 如果已登录，重定向到首页
if 'username' in session:
return redirect(url_for('index'))
return render_template('login.html')
@app.route('/api/login', methods=['POST'])
def login():
"""登录接口"""
try:
data = request.get_json()
username = data.get('username', '').strip()
password = data.get('password', '').strip()
if not username or not password:
return jsonify({'success': False, 'message': '请输入用户名和密码'})
# 验证用户名和密码
if username in USERS and USERS[username] == password:
session['username'] = username
return jsonify({'success': True, 'message': '登录成功'})
else:
return jsonify({'success': False, 'message': '用户名或密码错误'})
except Exception as e:
return jsonify({'success': False, 'message': f'登录失败: {str(e)}'})
@app.route('/api/logout', methods=['POST'])
def logout():
"""登出接口"""
session.pop('username', None)
return jsonify({'success': True, 'message': '已登出'})
@app.route('/api/export', methods=['POST'])
@login_required
def export_articles():
"""导出文章到Excel"""
try:
data = request.get_json()
url = data.get('url', '')
cookies_str = data.get('cookies', '') # 从前端获取Cookie
months = data.get('months', 6) # 获取时间范围，默认6个月
# 从URL中提取app_id，并转换为uk参数
app_id_match = re.search(r'app_id=(\d+)', url)
if not app_id_match:
return jsonify({'success': False, 'message': 'URL格式不正确，无法提取app_id'})
app_id = app_id_match.group(1)
print(f"开始导出，app_id={app_id}")
print(f"Cookie长度: {len(cookies_str) if cookies_str else 0}")
# 检查是否使用代理
use_proxy = data.get('use_proxy', False)
proxy_api_url = data.get('proxy_api_url', '')
articles_only = data.get('articles_only', True) # 获取是否仅爬取文章
# 从主页获取uk和Cookie
uk, auto_cookies = BaijiahaoScraper.get_uk_from_app_id(app_id, use_proxy=use_proxy, proxy_api_url=proxy_api_url if proxy_api_url else None)
if not uk:
return jsonify({'success': False, 'message': '无法获取用户UK，请检查URL是否正确'})
print(f"成功获取uk={uk}")
# 如果用户提供了Cookie,使用用户的Cookie
if cookies_str:
print("使用用户提供的Cookie")
# 解析Cookie字符串
cookies_dict = {}
for item in cookies_str.split(';'):
item = item.strip()
if '=' in item:
key, value = item.split('=', 1)
cookies_dict[key.strip()] = value.strip()
# 转换为requests.cookies.RequestsCookieJar
from requests.cookies import cookiejar_from_dict
user_cookies = cookiejar_from_dict(cookies_dict)
scraper = BaijiahaoScraper(uk, user_cookies, use_proxy=use_proxy, proxy_api_url=proxy_api_url if proxy_api_url else None)
else:
print("使用自动获取的Cookie")
scraper = BaijiahaoScraper(uk, auto_cookies, use_proxy=use_proxy, proxy_api_url=proxy_api_url if proxy_api_url else None)
# 使用 API 方式获取文章（仅保留代理IP池 + API 方式）
print(f"使用 API 方式获取文章（近{months}个月）...")
try:
articles = scraper.get_articles(months=months, app_id=app_id, articles_only=articles_only)
except Exception as e:
print(f"API 方式失败: {e}")
articles = []
if not articles:
return jsonify({
'success': False,
'message': (
'未能获取到文章数据。\n\n'
'请确保:\n'
'1. URL正确且该作者有发布过文章\n'
'2. 网络连接正常\n'
'3. 如需使用代理，请配置代理IP池'
)
})
# 创建Excel文件
df = pd.DataFrame(articles)
# 确保输出目录存在
output_dir = os.path.join(os.path.dirname(__file__), 'exports')
os.makedirs(output_dir, exist_ok=True)
# 生成文件名
timestamp = datetime.now().strftime('%Y%m%d_%H%M%S')
filename = f'baijiahao_articles_{app_id}_{timestamp}.xlsx'
filepath = os.path.join(output_dir, filename)
# 保存Excel
with pd.ExcelWriter(filepath, engine='openpyxl') as writer:
df.to_excel(writer, index=False, sheet_name='文章列表')
# 调整列宽
worksheet = writer.sheets['文章列表']
worksheet.column_dimensions['A'].width = 80 # 标题列
worksheet.column_dimensions['B'].width = 20 # 时间列
return jsonify({
'success': True,
'message': f'成功导出{len(articles)}篇文章',
'filename': filename,
'count': len(articles),
'articles': articles[:100] # 返回前100篇文章用于预览
})
except Exception as e:
import traceback
traceback.print_exc()
return jsonify({'success': False, 'message': f'导出失败: {str(e)}'})
@app.route('/api/download/<filename>')
@login_required
def download_file(filename):
"""下载Excel文件"""
try:
filepath = os.path.join(os.path.dirname(__file__), 'exports', filename)
if os.path.exists(filepath):
return send_file(filepath, as_attachment=True, download_name=filename)
else:
return jsonify({'success': False, 'message': '文件不存在'})
except Exception as e:
return jsonify({'success': False, 'message': f'下载失败: {str(e)}'})
# ==================== 任务队列相关API ====================
@app.route('/api/queue/add', methods=['POST'])
@login_required
def add_task_to_queue():
"""添加任务到队列"""
try:
data = request.get_json()
url = data.get('url', '')
months = data.get('months', 6)
use_proxy = data.get('use_proxy', False)
proxy_api_url = data.get('proxy_api_url', '')
articles_only = data.get('articles_only', True) # 获取是否仅爬取文章
if not url:
return jsonify({'success': False, 'message': 'URL不能为空'})
# 添加任务到队列
queue = get_task_queue()
task_id = queue.add_task(
url=url,
months=months,
use_proxy=use_proxy,
proxy_api_url=proxy_api_url if proxy_api_url else None,
username=session.get('username'),
articles_only=articles_only
)
return jsonify({
'success': True,
'message': '任务已添加到队列',
'task_id': task_id
})
except Exception as e:
import traceback
traceback.print_exc()
return jsonify({'success': False, 'message': f'添加任务失败: {str(e)}'})
@app.route('/api/queue/tasks', methods=['GET'])
@login_required
def get_tasks():
"""获取任务列表"""
try:
queue = get_task_queue()
username = session.get('username')
# 获取用户的所有任务
tasks = queue.get_all_tasks(username=username)
# 按创建时间倒序排列
tasks.sort(key=lambda x: x.get('created_at', ''), reverse=True)
return jsonify({
'success': True,
'tasks': tasks
})
except Exception as e:
return jsonify({'success': False, 'message': f'获取任务列表失败: {str(e)}'})
@app.route('/api/queue/task/<task_id>', methods=['GET'])
@login_required
def get_task_detail(task_id):
"""获取任务详情"""
try:
queue = get_task_queue()
task = queue.get_task(task_id)
if not task:
return jsonify({'success': False, 'message': '任务不存在'})
# 检查权限
if task.get('username') != session.get('username'):
return jsonify({'success': False, 'message': '无权查看此任务'})
return jsonify({
'success': True,
'task': task
})
except Exception as e:
return jsonify({'success': False, 'message': f'获取任务详情失败: {str(e)}'})
@app.route('/api/queue/stats', methods=['GET'])
@login_required
def get_queue_stats():
"""获取队列统计信息"""
try:
queue = get_task_queue()
username = session.get('username')
stats = queue.get_queue_stats(username=username)
return jsonify({
'success': True,
'stats': stats
})
except Exception as e:
return jsonify({'success': False, 'message': f'获取统计信息失败: {str(e)}'})
@app.route('/api/queue/download/<task_id>', methods=['GET'])
@login_required
def download_task_result(task_id):
"""下载任务结果文件（通过任务ID）"""
try:
queue = get_task_queue()
task = queue.get_task(task_id)
if not task:
return jsonify({'success': False, 'message': '任务不存在'})
# 检查权限
if task.get('username') != session.get('username'):
return jsonify({'success': False, 'message': '无权下载此文件'})
if task.get('status') != 'completed':
return jsonify({'success': False, 'message': '任务未完成'})
result_file = task.get('result_file')
if not result_file:
return jsonify({'success': False, 'message': '结果文件不存在'})
filepath = os.path.join(queue.results_dir, result_file)
if not os.path.exists(filepath):
return jsonify({'success': False, 'message': '文件不存在'})
return send_file(filepath, as_attachment=True, download_name=result_file)
except Exception as e:
return jsonify({'success': False, 'message': f'下载失败: {str(e)}'})
@app.route('/api/queue/task/<task_id>/delete', methods=['POST'])
@login_required
def delete_task(task_id):
"""删除任务（自动终止后删除）"""
queue = get_task_queue()
task = queue.get_task(task_id)
if not task:
return jsonify({'success': False, 'message': '任务不存在'})
if task.get('username') != session.get('username'):
return jsonify({'success': False, 'message': '无权删除此任务'})
ok = queue.delete_task(task_id)
return jsonify({'success': ok})
@app.route('/api/queue/task/<task_id>/cancel', methods=['POST'])
@login_required
def cancel_task(task_id):
"""终止任务（等待中或处理中）"""
queue = get_task_queue()
task = queue.get_task(task_id)
if not task:
return jsonify({'success': False, 'message': '任务不存在'})
if task.get('username') != session.get('username'):
return jsonify({'success': False, 'message': '无权终止此任务'})
if task.get('status') not in ['pending', 'processing']:
return jsonify({'success': False, 'message': '仅可终止等待中或处理中任务'})
ok = queue.cancel_task(task_id)
return jsonify({'success': ok})
@app.route('/api/queue/task/<task_id>/logs', methods=['GET'])
@login_required
def get_task_logs(task_id):
"""获取任务的历史日志"""
try:
queue = get_task_queue()
task = queue.get_task(task_id)
if not task:
return jsonify({'success': False, 'message': '任务不存在'})
# 检查权限
if task.get('username') != session.get('username'):
return jsonify({'success': False, 'message': '无权查看此任务日志'})
# 从数据库获取日志
from database import get_database
db = get_database()
logs = db.get_task_logs(task_id)
return jsonify({
'success': True,
'logs': logs
})
except Exception as e:
return jsonify({'success': False, 'message': f'获取日志失败: {str(e)}'})
@app.route('/health/taskworker')
def health_taskworker():
"""TaskWorker 健康检查接口"""
try:
from task_worker import get_task_worker
from task_queue import get_task_queue
worker = get_task_worker()
queue = get_task_queue()
# 统计信息
tasks = queue.get_all_tasks()
pending_count = len([t for t in tasks if t.get('status') == 'pending'])
processing_count = len([t for t in tasks if t.get('status') == 'processing'])
# Worker 状态
alive_threads = sum(1 for t in worker.worker_threads if t and t.is_alive())
status = 'healthy' if worker.running and alive_threads > 0 else 'unhealthy'
return jsonify({
'status': status,
'worker': {
'running': worker.running,
'alive_threads': alive_threads,
'current_workers': worker.current_workers,
'max_workers': worker.max_workers,
'processing_tasks': len(worker.processing_tasks)
},
'queue': {
'pending': pending_count,
'processing': processing_count,
'total': len(tasks)
},
'timestamp': datetime.now().strftime('%Y-%m-%d %H:%M:%S')
})
except Exception as e:
return jsonify({
'status': 'error',
'message': str(e)
}), 500
if __name__ == '__main__':
import sys
# 检查并安装依赖
def check_dependencies():
"""检查关键依赖是否安装"""
missing = []
try:
import flask
except ImportError:
missing.append('flask')
try:
import pandas
except ImportError:
missing.append('pandas')
try:
import openpyxl
except ImportError:
missing.append('openpyxl')
if missing:
print(f"\n⚠️ 缺少依赖: {', '.join(missing)}")
print("请运行: pip install -r requirements.txt\n")
return False
return True
if not check_dependencies():
sys.exit(1)
# 创建必要的目录
os.makedirs('exports', exist_ok=True)
os.makedirs('templates', exist_ok=True)
os.makedirs('static/css', exist_ok=True)
os.makedirs('static/js', exist_ok=True)
os.makedirs('data', exist_ok=True)
os.makedirs('data/results', exist_ok=True)
# 启动任务处理器
print('🔧 启动任务处理器...')
# 设置 SocketIO 实例到 task_worker
from task_worker import set_socketio
set_socketio(socketio)
start_task_worker()
print('✅ 任务处理器已启动')
# 检测是否是生产环境
is_production = os.environ.get('FLASK_ENV') == 'production'
if is_production:
print('✅ 生产环境启动')
print('请使用 gunicorn 或 uwsgi 运行:')
print(' gunicorn -w 4 -b 0.0.0.0:5001 app:app')
print('\n如果要直接运行,请使用: python app.py --dev')
if '--dev' not in sys.argv:
sys.exit(1)
print('🚀 服务器启动成功！')
print('请访问: http://127.0.0.1:8030')
# 开发环境使用 SocketIO 服务器
socketio.run(
app,
debug=not is_production,
host='0.0.0.0',
port=8030,
allow_unsafe_werkzeug=True
)
Reference in New Issue Copy Permalink