yunqueai/baijiahao_data_crawl
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: 完善代理重试机制，添加数据验证告警，新增README文档

Browse Source
This commit is contained in:
shengyudong@yunqueai.net
2026-01-16 18:36:52 +08:00
parent 322ac74336
commit b518e6aacf
55 changed files with 13202 additions and 34781 deletions
Download Patch File Download Diff File Expand all files Collapse all files

73
sync_cookies_to_db.py
View File

@@ -277,17 +277,44 @@ class CookieSyncToDB:
# 转换Cookie为字符串
cookie_string = self.cookie_dict_to_string(cookies)
# 提取其他信息
# 提取其他信息（使用username和nick作为author_name进行匹配）
username = account_info.get('username', '') # 优先用于与数据库author_name匹配
nick = account_info.get('nick', '') # 备用匹配字段
app_id = account_info.get('app_id', '')
nick = account_info.get('nick', '')
domain = account_info.get('domain', '')
level = account_info.get('level', '')
# 查找作者（使用 author_name + channel 作为唯一键）
# 验证username或nick至少有一个存在
if not username and not nick:
print(" [!] 该账号没有username和nick字段，跳过")
stats['skipped'] += 1
continue
print(f" Username: {username}")
print(f" 昵称: {nick}")
# 查找作者（使用双重匹配机制：先username，后nick）
channel = 1 # 百家号固定为channel=1
author = self.find_author_by_name(account_name, channel)
if author:
print(f" [√] 找到作者: {author['author_name']} (ID: {author['id']}, Channel: {author['channel']})")
author = None
matched_field = None
# 1. 首先尝试使用username匹配
if username:
author = self.find_author_by_name(username, channel)
if author:
matched_field = 'username'
print(f" [√] 通过username匹配到作者: {author['author_name']} (ID: {author['id']}, Channel: {author['channel']})")
# 2. 如果username匹配失败，尝试使用nick匹配
if not author and nick:
author = self.find_author_by_name(nick, channel)
if author:
matched_field = 'nick'
print(f" [√] 通过nick匹配到作者: {author['author_name']} (ID: {author['id']}, Channel: {author['channel']})")
# 3. 如果都没匹配到
if not author:
print(f" [!] 未找到匹配的作者（已尝试username和nick）")
# 更新或创建
if author:
@@ -300,15 +327,17 @@ class CookieSyncToDB:
)
if success:
print(f" [OK] Cookie已更新")
print(f" [OK] Cookie已更新（匹配字段: {matched_field}）")
stats['updated'] += 1
# 记录成功
success_records.append({
'account_name': account_name,
'app_id': app_id,
'username': username,
'nick': nick,
'app_id': app_id,
'domain': domain,
'action': 'updated',
'matched_field': matched_field,
'db_author_id': author['id'],
'db_author_name': author['author_name']
})
@@ -318,17 +347,21 @@ class CookieSyncToDB:
# 记录失败
failed_records.append({
'account_name': account_name,
'app_id': app_id,
'username': username,
'nick': nick,
'app_id': app_id,
'reason': '数据库更新失败',
'matched_field': matched_field,
'db_author_id': author['id']
})
else:
# 作者不存在
# 作者不存在，考虑创建
if auto_create:
print(f" [*] 作者不存在，创建新记录...")
# 优先使用username，如果没有则使用nick
author_name_to_create = username if username else nick
print(f" [*] 作者不存在，创建新记录（author_name: {author_name_to_create}）...")
success = self.insert_new_author(
account_name,
author_name_to_create, # 优先使用username，否则使用nick
cookie_string,
app_id,
nick,
@@ -337,15 +370,17 @@ class CookieSyncToDB:
)
if success:
print(f" [OK] 新作者已创建")
print(f" [OK] 新作者已创建 (author_name: {author_name_to_create})")
stats['created'] += 1
# 记录成功
success_records.append({
'account_name': account_name,
'app_id': app_id,
'username': username,
'nick': nick,
'app_id': app_id,
'domain': domain,
'action': 'created'
'action': 'created',
'created_with': 'username' if username else 'nick'
})
else:
print(f" [X] 创建作者失败")
@@ -353,8 +388,9 @@ class CookieSyncToDB:
# 记录失败
failed_records.append({
'account_name': account_name,
'app_id': app_id,
'username': username,
'nick': nick,
'app_id': app_id,
'reason': '数据库插入失败'
})
else:
@@ -363,9 +399,10 @@ class CookieSyncToDB:
# 记录失败（数据库中不存在）
failed_records.append({
'account_name': account_name,
'app_id': app_id,
'username': username,
'nick': nick,
'reason': '数据库中不存在该账号，且未开启自动创建'
'app_id': app_id,
'reason': '数据库中不存在该账号（已尝试username和nick），且未开启自动创建'
})
# 保存记录文件