542 lines
22 KiB
Python
542 lines
22 KiB
Python
#!/usr/bin/env python3
|
||
# -*- coding: utf-8 -*-
|
||
"""
|
||
文章图片智能匹配脚本
|
||
基于通义千问大模型，智能匹配文章与图片，将结果存储到ai_article_images表
|
||
"""
|
||
|
||
import os
|
||
import sys
|
||
import time
|
||
import json
|
||
import logging
|
||
import requests
|
||
import pymysql
|
||
import traceback
|
||
import threading
|
||
from datetime import datetime
|
||
from typing import Dict, List, Optional, Tuple
|
||
from concurrent.futures import ThreadPoolExecutor, as_completed
|
||
|
||
# 添加项目根目录到Python路径
|
||
sys.path.append(os.path.dirname(os.path.abspath(__file__)))
|
||
|
||
from database_config import db_manager
|
||
from log_config import setup_logger
|
||
|
||
# 配置日志记录器
|
||
logger = setup_logger(
|
||
name='article_image_matching',
|
||
log_file='logs/article_image_matching.log',
|
||
error_log_file='logs/article_image_matching_error.log',
|
||
level=logging.INFO,
|
||
console_output=True
|
||
)
|
||
|
||
# 配置常量
|
||
QWEN_API_KEY = "sk-e6a38204022a4b538b8954f0584712af"
|
||
QWEN_API_URL = "https://dashscope.aliyuncs.com/api/v1/services/aigc/text-generation/generation"
|
||
WORKER_COUNT = 4 # 并行处理worker数量
|
||
BATCH_SIZE = 50 # 每批处理的文章数量
|
||
MATCH_THRESHOLD = 0.6 # 匹配分数阈值（0-1）
|
||
|
||
class ArticleImageMatcher:
|
||
def __init__(self):
|
||
# 使用统一的数据库管理器
|
||
self.db_manager = db_manager
|
||
|
||
# 并行处理相关
|
||
self.processing_lock = threading.Lock()
|
||
self.processed_articles = set()
|
||
|
||
logger.info("ArticleImageMatcher 初始化完成")
|
||
self.log_to_database('INFO', 'ArticleImageMatcher 初始化完成')
|
||
|
||
def log_to_database(self, level: str, message: str, details: Optional[str] = None):
|
||
"""记录日志到数据库ai_logs表"""
|
||
try:
|
||
connection = self.db_manager.get_connection()
|
||
try:
|
||
with connection.cursor(pymysql.cursors.DictCursor) as cursor:
|
||
status_map = {
|
||
'INFO': 'success',
|
||
'WARNING': 'warning',
|
||
'ERROR': 'error'
|
||
}
|
||
status = status_map.get(level, 'success')
|
||
|
||
sql = """
|
||
INSERT INTO ai_logs (user_id, action, description, status, error_message, created_at)
|
||
VALUES (%s, %s, %s, %s, %s, NOW())
|
||
"""
|
||
cursor.execute(sql, (None, 'article_image_matching', message, status, details))
|
||
connection.commit()
|
||
logger.info(f"日志已记录到数据库: {level} - {message}")
|
||
finally:
|
||
connection.close()
|
||
except Exception as e:
|
||
logger.error(f"记录日志到数据库失败: {e}")
|
||
|
||
def get_articles_with_tags(self, limit: int = BATCH_SIZE) -> List[Dict]:
|
||
"""
|
||
从ai_article_tags表获取需要匹配图片的文章
|
||
|
||
Returns:
|
||
包含文章ID、标签等信息的列表
|
||
"""
|
||
try:
|
||
connection = self.db_manager.get_connection()
|
||
try:
|
||
with connection.cursor(pymysql.cursors.DictCursor) as cursor:
|
||
# 查询有标签但未匹配图片的文章
|
||
sql = """
|
||
SELECT
|
||
at.id,
|
||
at.article_id,
|
||
at.coze_tag,
|
||
a.title,
|
||
a.content
|
||
FROM ai_article_tags at
|
||
INNER JOIN ai_articles a ON at.article_id = a.id
|
||
WHERE at.coze_tag IS NOT NULL
|
||
AND at.coze_tag != ''
|
||
AND NOT EXISTS (
|
||
SELECT 1 FROM ai_article_images ai
|
||
WHERE ai.article_id = at.article_id
|
||
)
|
||
AND a.status = 'approved'
|
||
ORDER BY at.id DESC
|
||
LIMIT %s
|
||
"""
|
||
cursor.execute(sql, (limit,))
|
||
results = cursor.fetchall()
|
||
|
||
if results:
|
||
logger.info(f"查询到 {len(results)} 篇需要匹配图片的文章")
|
||
self.log_to_database('INFO', f"查询到需要匹配图片的文章", f"数量: {len(results)}")
|
||
else:
|
||
logger.info("未查询到需要匹配图片的文章")
|
||
|
||
return results
|
||
finally:
|
||
connection.close()
|
||
except Exception as e:
|
||
error_msg = f"查询文章标签异常: {e}"
|
||
logger.error(error_msg)
|
||
self.log_to_database('ERROR', error_msg, traceback.format_exc())
|
||
return []
|
||
|
||
def get_available_images_with_tags(self) -> List[Dict]:
|
||
"""
|
||
从ai_image_tags表获取可用的图片及其标签
|
||
|
||
Returns:
|
||
包含图片ID、标签等信息的列表
|
||
"""
|
||
try:
|
||
connection = self.db_manager.get_connection()
|
||
try:
|
||
with connection.cursor(pymysql.cursors.DictCursor) as cursor:
|
||
# 查询附加文章数量小于5的图片
|
||
sql = """
|
||
SELECT
|
||
id,
|
||
image_id,
|
||
image_name,
|
||
image_url,
|
||
image_thumb_url,
|
||
tag_id,
|
||
tag_name,
|
||
keywords_id,
|
||
keywords_name,
|
||
department_id,
|
||
department_name,
|
||
image_attached_article_count
|
||
FROM ai_image_tags
|
||
WHERE image_attached_article_count < 5
|
||
ORDER BY image_attached_article_count ASC, id DESC
|
||
"""
|
||
cursor.execute(sql)
|
||
results = cursor.fetchall()
|
||
|
||
if results:
|
||
logger.info(f"查询到 {len(results)} 张可用图片")
|
||
self.log_to_database('INFO', f"查询到可用图片", f"数量: {len(results)}")
|
||
else:
|
||
logger.info("未查询到可用图片")
|
||
|
||
return results
|
||
finally:
|
||
connection.close()
|
||
except Exception as e:
|
||
error_msg = f"查询图片标签异常: {e}"
|
||
logger.error(error_msg)
|
||
self.log_to_database('ERROR', error_msg, traceback.format_exc())
|
||
return []
|
||
|
||
def parse_article_tags(self, coze_tag: str) -> List[str]:
|
||
"""
|
||
解析文章标签（支持JSON格式和逗号分隔格式）
|
||
|
||
Args:
|
||
coze_tag: 标签字符串
|
||
|
||
Returns:
|
||
标签列表
|
||
"""
|
||
try:
|
||
if not coze_tag:
|
||
return []
|
||
|
||
# 尝试解析JSON格式
|
||
try:
|
||
tags_data = json.loads(coze_tag)
|
||
if isinstance(tags_data, list):
|
||
return tags_data
|
||
elif isinstance(tags_data, dict):
|
||
return list(tags_data.values())
|
||
except json.JSONDecodeError:
|
||
pass
|
||
|
||
# 按逗号分隔
|
||
tags = [tag.strip() for tag in str(coze_tag).split(',') if tag.strip()]
|
||
return tags
|
||
except Exception as e:
|
||
logger.error(f"解析文章标签异常: {e}")
|
||
return []
|
||
|
||
def call_qwen_for_matching(self, article_title: str, article_tags: List[str],
|
||
image_tags: List[str], image_keywords: str) -> Tuple[bool, float]:
|
||
"""
|
||
调用通义千问API评估文章与图片的匹配度
|
||
|
||
Args:
|
||
article_title: 文章标题
|
||
article_tags: 文章标签列表
|
||
image_tags: 图片标签列表
|
||
image_keywords: 图片关键词
|
||
|
||
Returns:
|
||
(是否匹配, 匹配分数)
|
||
"""
|
||
try:
|
||
# 构建提示词
|
||
prompt = f"""请评估以下文章与图片的匹配度：
|
||
|
||
文章标题：{article_title}
|
||
文章标签：{', '.join(article_tags)}
|
||
|
||
图片标签：{', '.join(image_tags)}
|
||
图片关键词：{image_keywords}
|
||
|
||
请根据标签的语义相关性，给出0-1之间的匹配分数，并说明是否适合匹配。
|
||
输出格式：{{"match": true/false, "score": 0.0-1.0, "reason": "匹配原因"}}
|
||
只输出JSON格式，不要其他内容。"""
|
||
|
||
headers = {
|
||
'Authorization': f'Bearer {QWEN_API_KEY}',
|
||
'Content-Type': 'application/json'
|
||
}
|
||
|
||
payload = {
|
||
"model": "qwen-max",
|
||
"input": {
|
||
"messages": [
|
||
{
|
||
"role": "user",
|
||
"content": prompt
|
||
}
|
||
]
|
||
},
|
||
"parameters": {
|
||
"result_format": "message"
|
||
}
|
||
}
|
||
|
||
response = requests.post(
|
||
QWEN_API_URL,
|
||
json=payload,
|
||
headers=headers,
|
||
timeout=30
|
||
)
|
||
|
||
if response.status_code == 200:
|
||
result = response.json()
|
||
if result.get('output') and result['output'].get('choices'):
|
||
message = result['output']['choices'][0].get('message', {})
|
||
content = message.get('content', '')
|
||
|
||
# 解析JSON响应
|
||
try:
|
||
match_result = json.loads(content)
|
||
is_match = match_result.get('match', False)
|
||
score = match_result.get('score', 0.0)
|
||
reason = match_result.get('reason', '')
|
||
|
||
logger.info(f"通义千问评估结果 - 匹配: {is_match}, 分数: {score}, 原因: {reason}")
|
||
return is_match, score
|
||
except json.JSONDecodeError:
|
||
logger.error(f"解析通义千问响应失败: {content}")
|
||
return False, 0.0
|
||
else:
|
||
logger.error(f"通义千问API返回格式异常: {result}")
|
||
return False, 0.0
|
||
else:
|
||
logger.error(f"通义千问API请求失败，状态码: {response.status_code}")
|
||
return False, 0.0
|
||
|
||
except requests.exceptions.Timeout:
|
||
logger.error("通义千问API请求超时")
|
||
return False, 0.0
|
||
except Exception as e:
|
||
error_msg = f"调用通义千问API异常: {e}"
|
||
logger.error(error_msg)
|
||
self.log_to_database('ERROR', error_msg, traceback.format_exc())
|
||
return False, 0.0
|
||
|
||
def insert_article_image_relation(self, article_id: int, image_data: Dict,
|
||
match_score: float) -> bool:
|
||
"""
|
||
将匹配结果插入ai_article_images表
|
||
|
||
Args:
|
||
article_id: 文章ID
|
||
image_data: 图片数据
|
||
match_score: 匹配分数
|
||
|
||
Returns:
|
||
是否插入成功
|
||
"""
|
||
try:
|
||
connection = self.db_manager.get_connection()
|
||
try:
|
||
with connection.cursor(pymysql.cursors.DictCursor) as cursor:
|
||
# 查询当前文章下已有图片的最大sort_order
|
||
query_max_sort = """
|
||
SELECT COALESCE(MAX(sort_order), 0) as max_sort_order
|
||
FROM ai_article_images
|
||
WHERE article_id = %s
|
||
"""
|
||
cursor.execute(query_max_sort, (article_id,))
|
||
result = cursor.fetchone()
|
||
max_sort_order = result.get('max_sort_order', 0)
|
||
new_sort_order = max_sort_order + 1
|
||
|
||
# 插入关联记录
|
||
insert_sql = """
|
||
INSERT INTO ai_article_images
|
||
(article_id, image_id, image_url, image_thumb_url, image_tag_id,
|
||
sort_order, keywords_id, keywords_name, department_id, department_name,
|
||
image_source, created_at)
|
||
VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, NOW())
|
||
"""
|
||
cursor.execute(insert_sql, (
|
||
article_id,
|
||
image_data['image_id'],
|
||
image_data['image_url'],
|
||
image_data['image_thumb_url'],
|
||
image_data['id'], # image_tag_id
|
||
new_sort_order,
|
||
image_data['keywords_id'],
|
||
image_data['keywords_name'],
|
||
image_data['department_id'],
|
||
image_data['department_name'],
|
||
1 # image_source: 1表示tag匹配
|
||
))
|
||
|
||
# 更新图片附加文章计数
|
||
update_sql = """
|
||
UPDATE ai_image_tags
|
||
SET image_attached_article_count = image_attached_article_count + 1
|
||
WHERE id = %s
|
||
"""
|
||
cursor.execute(update_sql, (image_data['id'],))
|
||
|
||
connection.commit()
|
||
logger.info(f"成功插入文章图片关联 - 文章ID: {article_id}, 图片ID: {image_data['image_id']}, 分数: {match_score}")
|
||
return True
|
||
finally:
|
||
connection.close()
|
||
|
||
except Exception as e:
|
||
error_msg = f"插入文章图片关联异常: {e}"
|
||
logger.error(error_msg)
|
||
self.log_to_database('ERROR', error_msg, traceback.format_exc())
|
||
return False
|
||
|
||
def match_article_with_images(self, article_data: Dict, available_images: List[Dict]) -> bool:
|
||
"""
|
||
为单篇文章匹配图片
|
||
|
||
Args:
|
||
article_data: 文章数据
|
||
available_images: 可用图片列表
|
||
|
||
Returns:
|
||
是否匹配成功
|
||
"""
|
||
article_id = article_data['article_id']
|
||
article_title = article_data.get('title', '')
|
||
coze_tag = article_data.get('coze_tag', '')
|
||
|
||
try:
|
||
# 解析文章标签
|
||
article_tags = self.parse_article_tags(coze_tag)
|
||
if not article_tags:
|
||
logger.warning(f"文章 {article_id} 没有有效标签，跳过")
|
||
return False
|
||
|
||
logger.info(f"开始为文章 {article_id} 匹配图片 - 标题: {article_title}, 标签: {article_tags}")
|
||
|
||
best_match = None
|
||
best_score = 0.0
|
||
|
||
# 遍历可用图片，找到最佳匹配
|
||
for image_data in available_images:
|
||
image_tags = [image_data['tag_name']]
|
||
image_keywords = image_data.get('keywords_name', '')
|
||
|
||
# 调用通义千问评估匹配度
|
||
is_match, score = self.call_qwen_for_matching(
|
||
article_title, article_tags, image_tags, image_keywords
|
||
)
|
||
|
||
# 记录评估结果
|
||
logger.info(f"文章 {article_id} vs 图片 {image_data['image_id']} - 匹配: {is_match}, 分数: {score}")
|
||
|
||
# 更新最佳匹配
|
||
if is_match and score > best_score and score >= MATCH_THRESHOLD:
|
||
best_score = score
|
||
best_match = image_data
|
||
|
||
# 如果找到匹配的图片，插入关联记录
|
||
if best_match:
|
||
if self.insert_article_image_relation(article_id, best_match, best_score):
|
||
logger.info(f"文章 {article_id} 成功匹配图片 {best_match['image_id']}, 分数: {best_score}")
|
||
self.log_to_database('INFO', f"文章匹配成功",
|
||
f"文章ID: {article_id}, 图片ID: {best_match['image_id']}, 分数: {best_score}")
|
||
return True
|
||
else:
|
||
return False
|
||
else:
|
||
logger.info(f"文章 {article_id} 未找到合适的匹配图片")
|
||
self.log_to_database('WARNING', f"文章未找到匹配图片", f"文章ID: {article_id}")
|
||
return False
|
||
|
||
except Exception as e:
|
||
error_msg = f"匹配文章图片异常 - 文章ID: {article_id}, 错误: {e}"
|
||
logger.error(error_msg)
|
||
self.log_to_database('ERROR', error_msg, traceback.format_exc())
|
||
return False
|
||
|
||
def get_next_available_article(self, pending_articles: List[Dict]) -> Optional[Dict]:
|
||
"""线程安全地获取下一篇待处理文章"""
|
||
with self.processing_lock:
|
||
for article_data in pending_articles:
|
||
article_id = article_data['article_id']
|
||
if article_id not in self.processed_articles:
|
||
self.processed_articles.add(article_id)
|
||
return article_data
|
||
return None
|
||
|
||
def worker_process_articles(self, pending_articles: List[Dict],
|
||
available_images: List[Dict], worker_id: int) -> int:
|
||
"""Worker线程处理文章匹配"""
|
||
processed_count = 0
|
||
thread_name = f"Worker-{worker_id}"
|
||
threading.current_thread().name = thread_name
|
||
|
||
logger.info(f"[{thread_name}] 启动，准备处理文章匹配")
|
||
|
||
while True:
|
||
# 线程安全地获取下一篇待处理文章
|
||
article_data = self.get_next_available_article(pending_articles)
|
||
if not article_data:
|
||
logger.info(f"[{thread_name}] 没有更多待处理文章，退出")
|
||
break
|
||
|
||
# 匹配文章与图片
|
||
if self.match_article_with_images(article_data, available_images):
|
||
processed_count += 1
|
||
logger.info(f"[{thread_name}] 成功处理文章: {article_data['article_id']}")
|
||
else:
|
||
logger.warning(f"[{thread_name}] 文章处理失败: {article_data['article_id']}")
|
||
|
||
logger.info(f"[{thread_name}] 完成，共处理 {processed_count} 篇文章")
|
||
return processed_count
|
||
|
||
def run_matching(self):
|
||
"""运行文章图片匹配流程"""
|
||
logger.info("开始文章图片智能匹配...")
|
||
self.log_to_database('INFO', '启动文章图片智能匹配服务', f'worker数量: {WORKER_COUNT}')
|
||
|
||
try:
|
||
# 获取需要匹配的文章
|
||
pending_articles = self.get_articles_with_tags()
|
||
if not pending_articles:
|
||
logger.info("没有需要匹配的文章")
|
||
return
|
||
|
||
# 获取可用图片
|
||
available_images = self.get_available_images_with_tags()
|
||
if not available_images:
|
||
logger.warning("没有可用图片，无法进行匹配")
|
||
self.log_to_database('WARNING', '没有可用图片')
|
||
return
|
||
|
||
logger.info(f"开始匹配 {len(pending_articles)} 篇文章与 {len(available_images)} 张图片")
|
||
self.log_to_database('INFO', '开始批量匹配',
|
||
f'文章数: {len(pending_articles)}, 图片数: {len(available_images)}')
|
||
|
||
# 清空已处理记录集合
|
||
with self.processing_lock:
|
||
self.processed_articles.clear()
|
||
|
||
# 使用线程池并行处理
|
||
with ThreadPoolExecutor(max_workers=WORKER_COUNT, thread_name_prefix="MatchWorker") as executor:
|
||
# 提交worker任务
|
||
future_to_worker = {}
|
||
for worker_id in range(1, WORKER_COUNT + 1):
|
||
future = executor.submit(
|
||
self.worker_process_articles,
|
||
pending_articles,
|
||
available_images,
|
||
worker_id
|
||
)
|
||
future_to_worker[future] = worker_id
|
||
|
||
# 等待所有worker完成
|
||
total_processed = 0
|
||
for future in as_completed(future_to_worker):
|
||
worker_id = future_to_worker[future]
|
||
try:
|
||
processed_count = future.result()
|
||
total_processed += processed_count
|
||
logger.info(f"Worker-{worker_id} 完成，处理了 {processed_count} 篇文章")
|
||
except Exception as e:
|
||
logger.error(f"Worker-{worker_id} 执行异常: {e}")
|
||
self.log_to_database('ERROR', f'Worker-{worker_id} 执行异常', str(e))
|
||
|
||
logger.info(f"匹配完成，共处理 {total_processed} 篇文章")
|
||
self.log_to_database('INFO', '匹配任务完成', f'共处理 {total_processed} 篇文章')
|
||
|
||
except Exception as e:
|
||
error_msg = f"匹配流程异常: {e}"
|
||
logger.error(error_msg)
|
||
self.log_to_database('ERROR', error_msg, traceback.format_exc())
|
||
|
||
def main():
|
||
"""主函数"""
|
||
matcher = ArticleImageMatcher()
|
||
|
||
try:
|
||
# 运行匹配
|
||
matcher.run_matching()
|
||
|
||
except Exception as e:
|
||
logger.error(f"程序运行异常: {e}")
|
||
matcher.log_to_database('ERROR', f'程序运行异常: {e}', traceback.format_exc())
|
||
|
||
if __name__ == "__main__":
|
||
main()
|