2025-12-25 upload
This commit is contained in:
“shengyudong”
285
bjh_data_to_database.py
Normal file
285
bjh_data_to_database.py
Normal file
@@ -0,0 +1,285 @@
|
||||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
"""
|
||||
百家号数据抓取并生成CSV文件的整合脚本
|
||||
|
||||
功能：
|
||||
1. 从百家号API抓取最新数据（发文统计 + 收益数据 + 当周收益）
|
||||
2. 保存到 bjh_integrated_data.json
|
||||
3. 生成三个CSV文件：
|
||||
- ai_statistics.csv - 账号汇总统计表
|
||||
- ai_statistics_day.csv - 每日明细统计表（最新一天）
|
||||
- ai_statistics_days.csv - 核心指标统计表（最新一天，含发文量、收益、环比）
|
||||
|
||||
特点：
|
||||
- author_id 将从 ai_authors 表中根据 author_name 查询获取
|
||||
- 支持从数据库加载Cookie，无需手动维护
|
||||
- 支持代理模式，防止被限流
|
||||
- 支持错误重试机制，自动处理代理连接异常
|
||||
"""
|
||||
|
||||
import sys
|
||||
import os
|
||||
import time
|
||||
|
||||
# 设置UTF-8编码
|
||||
if sys.platform == 'win32':
|
||||
import io
|
||||
if not isinstance(sys.stdout, io.TextIOWrapper) or sys.stdout.encoding != 'utf-8':
|
||||
sys.stdout = io.TextIOWrapper(sys.stdout.buffer, encoding='utf-8')
|
||||
if not isinstance(sys.stderr, io.TextIOWrapper) or sys.stderr.encoding != 'utf-8':
|
||||
sys.stderr = io.TextIOWrapper(sys.stderr.buffer, encoding='utf-8')
|
||||
|
||||
from bjh_analytics import BaijiahaoAnalytics
|
||||
from export_to_csv import DataExporter
|
||||
from log_config import setup_data_to_db_logger
|
||||
|
||||
|
||||
def main():
|
||||
# 初始化日志记录器
|
||||
logger = setup_data_to_db_logger()
|
||||
|
||||
logger.info("="*70)
|
||||
logger.info("百家号数据抓取并生成CSV文件 - 启动")
|
||||
logger.info("="*70)
|
||||
|
||||
print("\n" + "="*70)
|
||||
print("百家号数据抓取并生成CSV文件")
|
||||
print("="*70)
|
||||
|
||||
# 步骤1：配置数据抓取
|
||||
print("\n【步骤1/3】配置数据抓取")
|
||||
print("="*70)
|
||||
logger.info("步骤1/3：配置数据抓取")
|
||||
|
||||
# 询问数据源
|
||||
print("\n请选择账号数据源：")
|
||||
print(" 1. 从本地文件加载Cookie (captured_account_cookies.json)")
|
||||
print(" 2. 从数据库加载Cookie")
|
||||
|
||||
source = input("\n输入选项 (1/2, 默认1): ").strip() or '1'
|
||||
load_from_db = (source == '2')
|
||||
logger.info(f"数据源选择: {'数据库' if load_from_db else '本地文件'}")
|
||||
|
||||
# 询问是否使用代理
|
||||
print("\n是否启用代理？")
|
||||
use_proxy_input = input("(y/n, 默认n): ").strip().lower() or 'n'
|
||||
use_proxy = (use_proxy_input == 'y')
|
||||
logger.info(f"代理模式: {'启用' if use_proxy else '禁用'}")
|
||||
|
||||
# 询问抓取天数
|
||||
print("\n抓取最近多少天的数据？")
|
||||
days_input = input("(默认7天): ").strip() or '7'
|
||||
try:
|
||||
days = int(days_input)
|
||||
except ValueError:
|
||||
days = 7
|
||||
logger.info(f"抓取天数: {days}天")
|
||||
|
||||
# 自动启用错误重试机制（不询问）
|
||||
enable_retry = True
|
||||
max_retries = 3
|
||||
print(f"\n[配置] 已自动启用错误重试机制（最大{max_retries}次）")
|
||||
logger.info(f"错误重试机制: 启用，最大重试{max_retries}次")
|
||||
|
||||
# 步骤2：抓取数据
|
||||
print("\n【步骤2/3】抓取数据")
|
||||
print("="*70)
|
||||
logger.info("步骤2/3：抓取数据")
|
||||
|
||||
# 初始化分析器
|
||||
analytics = BaijiahaoAnalytics(use_proxy=use_proxy, load_from_db=load_from_db)
|
||||
|
||||
if not analytics.account_cookies:
|
||||
print("\n[X] 未找到可用的账号Cookie，无法继续")
|
||||
logger.error("未找到可用的账号Cookie")
|
||||
return
|
||||
|
||||
print(f"\n找到 {len(analytics.account_cookies)} 个账号")
|
||||
logger.info(f"找到 {len(analytics.account_cookies)} 个账号")
|
||||
|
||||
# 询问是否抓取所有账号
|
||||
print("\n请选择抓取模式：")
|
||||
print(" 1. 抓取所有账号")
|
||||
print(" 2. 只抓取第一个账号（测试用）")
|
||||
|
||||
mode = input("\n输入选项 (1/2, 默认1): ").strip() or '1'
|
||||
logger.info(f"抓取模式: {'所有账号' if mode == '1' else '测试单账号'}")
|
||||
|
||||
if mode == '2':
|
||||
# 只抓取第一个账号
|
||||
account_ids = [list(analytics.account_cookies.keys())[0]]
|
||||
else:
|
||||
# 抓取所有账号
|
||||
account_ids = list(analytics.account_cookies.keys())
|
||||
|
||||
print(f"\n将抓取 {len(account_ids)} 个账号的数据")
|
||||
logger.info(f"将抓取 {len(account_ids)} 个账号: {', '.join(account_ids)}")
|
||||
|
||||
confirm = input("\n是否继续? (y/n): ").strip().lower()
|
||||
if confirm != 'y':
|
||||
print("\n已取消")
|
||||
logger.info("用户取消操作")
|
||||
return
|
||||
|
||||
# 执行抓取
|
||||
print("\n开始抓取数据...")
|
||||
print(f" 错误重试策略：遇到代理连接错误时自动重试，最大{max_retries}次\n")
|
||||
logger.info("="*70)
|
||||
logger.info("开始执行数据抓取任务")
|
||||
logger.info(f"总账号数: {len(account_ids)}，错误重试: 最大{max_retries}次")
|
||||
logger.info("="*70)
|
||||
|
||||
success_count = 0
|
||||
failed_accounts = []
|
||||
partial_success_accounts = [] # 部分成功的账号（只获取到部分数据）
|
||||
results = []
|
||||
|
||||
for i, account_id in enumerate(account_ids, 1):
|
||||
print(f"\n[{i}/{len(account_ids)}] 抓取账号: {account_id}")
|
||||
print("-"*70)
|
||||
|
||||
# 重试机制
|
||||
retry_count = 0
|
||||
result = None
|
||||
last_error = None
|
||||
|
||||
while retry_count <= (max_retries if enable_retry else 0):
|
||||
try:
|
||||
# 如果是重试，先等待一段时间
|
||||
if retry_count > 0:
|
||||
wait_time = retry_count * 2 # 递增等待时间：2秒、4秒、6秒
|
||||
print(f" [重试 {retry_count}/{max_retries}] 等待 {wait_time} 秒后重试...")
|
||||
logger.info(f"账号 {account_id} 第{retry_count}次重试，等待{wait_time}秒")
|
||||
time.sleep(wait_time)
|
||||
print(f" [重试 {retry_count}/{max_retries}] 开始重新抓取...")
|
||||
|
||||
result = analytics.extract_integrated_data(account_id, days=days)
|
||||
|
||||
if result:
|
||||
results.append(result)
|
||||
status = result.get('status', '')
|
||||
|
||||
if status == 'success_all':
|
||||
success_count += 1
|
||||
print(f" [✓] 抓取成功")
|
||||
logger.info(f"账号 {account_id} 抓取成功 (完全成功)")
|
||||
break # 成功，跳出重试循环
|
||||
elif 'success' in status:
|
||||
# 部分成功（例如：只获取到发文数据，收入数据失败）
|
||||
partial_success_accounts.append(account_id)
|
||||
print(f" [!] 部分成功: {status}")
|
||||
logger.warning(f"账号 {account_id} 部分成功: {status}")
|
||||
break # 部分成功也跳出重试循环
|
||||
else:
|
||||
# 失败
|
||||
last_error = f"状态: {status}"
|
||||
if not enable_retry or retry_count >= max_retries:
|
||||
failed_accounts.append(account_id)
|
||||
print(f" [X] 抓取失败: {status}")
|
||||
logger.error(f"账号 {account_id} 抓取失败: {status}")
|
||||
break
|
||||
else:
|
||||
retry_count += 1
|
||||
continue
|
||||
else:
|
||||
# 返回None表示完全失败
|
||||
last_error = "返回结果为空"
|
||||
if not enable_retry or retry_count >= max_retries:
|
||||
failed_accounts.append(account_id)
|
||||
print(f" [X] 抓取失败: 无返回结果")
|
||||
logger.error(f"账号 {account_id} 抓取失败: 无返回结果")
|
||||
break
|
||||
else:
|
||||
retry_count += 1
|
||||
continue
|
||||
|
||||
except Exception as e:
|
||||
last_error = str(e)
|
||||
error_type = type(e).__name__
|
||||
|
||||
# 判断是否是代理相关错误
|
||||
is_proxy_error = any([
|
||||
'Connection' in error_type,
|
||||
'RemoteDisconnected' in error_type,
|
||||
'ProxyError' in error_type,
|
||||
'Timeout' in error_type,
|
||||
'ConnectionError' in str(e),
|
||||
'Connection aborted' in str(e),
|
||||
'Remote end closed' in str(e),
|
||||
])
|
||||
|
||||
if is_proxy_error:
|
||||
print(f" [!] 代理连接错误: {error_type}")
|
||||
logger.warning(f"账号 {account_id} 代理连接错误: {error_type} - {last_error}")
|
||||
else:
|
||||
print(f" [!] 发生异常: {error_type} - {e}")
|
||||
logger.error(f"账号 {account_id} 异常: {error_type} - {e}", exc_info=True)
|
||||
|
||||
if not enable_retry or retry_count >= max_retries:
|
||||
failed_accounts.append(account_id)
|
||||
print(f" [X] 抓取失败（已达最大重试次数）")
|
||||
logger.error(f"账号 {account_id} 最终失败，已达最大重试次数")
|
||||
break
|
||||
else:
|
||||
retry_count += 1
|
||||
continue
|
||||
|
||||
# 保存到文件
|
||||
if results:
|
||||
analytics.save_results(results)
|
||||
logger.info(f"数据已保存到 bjh_integrated_data.json，共 {len(results)} 个账号")
|
||||
|
||||
print(f"\n数据抓取完成:")
|
||||
print(f" 完全成功: {success_count}/{len(account_ids)}")
|
||||
logger.info(f"数据抓取完成: 完全成功 {success_count}/{len(account_ids)}")
|
||||
|
||||
if partial_success_accounts:
|
||||
print(f" 部分成功: {len(partial_success_accounts)} 个账号")
|
||||
print(f" 账号列表: {', '.join(partial_success_accounts)}")
|
||||
logger.info(f"部分成功: {len(partial_success_accounts)} 个账号 - {', '.join(partial_success_accounts)}")
|
||||
if failed_accounts:
|
||||
print(f" 完全失败: {len(failed_accounts)} 个账号")
|
||||
print(f" 账号列表: {', '.join(failed_accounts)}")
|
||||
logger.error(f"完全失败: {len(failed_accounts)} 个账号 - {', '.join(failed_accounts)}")
|
||||
|
||||
# 步骤3：生成CSV文件
|
||||
print("\n【步骤3/3】生成CSV文件")
|
||||
print("="*70)
|
||||
logger.info("步骤3/3：生成CSV文件")
|
||||
|
||||
confirm = input("\n是否生成CSV文件? (y/n): ").strip().lower()
|
||||
if confirm != 'y':
|
||||
print("\n已跳过CSV生成")
|
||||
logger.info("用户跳过CSV生成")
|
||||
return
|
||||
|
||||
# 初始化导出器（CSV模式，但启用数据库连接以查询author_id）
|
||||
exporter = DataExporter(use_database=False)
|
||||
logger.info("已初始化CSV导出器（CSV模式 + 数据库查询author_id）")
|
||||
|
||||
print("\n将生成以下三个CSV文件：")
|
||||
print(" 1. ai_statistics.csv - 账号汇总统计表")
|
||||
print(" 2. ai_statistics_day.csv - 每日明细统计表")
|
||||
print(" 3. ai_statistics_days.csv - 核心指标统计表（含发文量、收益、环比）")
|
||||
print("\n注意：author_id 将从 ai_authors 表中查询")
|
||||
|
||||
# 执行CSV导出
|
||||
success = exporter.export_all_tables()
|
||||
|
||||
if success:
|
||||
print("\n" + "="*70)
|
||||
print("✓ 所有步骤完成！")
|
||||
print("="*70)
|
||||
logger.info("="*70)
|
||||
logger.info("所有步骤完成，CSV文件已生成")
|
||||
logger.info("="*70)
|
||||
else:
|
||||
print("\n" + "="*70)
|
||||
print("✗ CSV生成失败")
|
||||
print("="*70)
|
||||
logger.error("CSV生成失败")
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
Reference in New Issue
Block a user