| ID | +站点名称 | +URL | +状态 | +点击数 | +回复数 | +创建时间 | +
|---|---|---|---|---|---|---|
| 加载中... | +||||||
数据概览
+实时查看系统运行统计数据
+| ID | +站点名称 | +URL | +状态 | +点击数 | +回复数 | +创建时间 | +
|---|---|---|---|---|---|---|
|
+
+ 加载中...
+ |
+ ||||||
diff --git a/.env.example b/.env.example index f9c891d..d55534c 100644 --- a/.env.example +++ b/.env.example @@ -24,3 +24,13 @@ LOG_DIR=./logs # 调试模式 DEBUG=True + +# 测试配置 +AUTO_CLOSE_BROWSER=False + +# MySQL数据库配置 +MYSQL_HOST=localhost +MYSQL_PORT=3306 +MYSQL_USER=root +MYSQL_PASSWORD=your_password +MYSQL_DATABASE=ai_article diff --git a/VENV_GUIDE.txt b/VENV_GUIDE.txt deleted file mode 100644 index 9900dd5..0000000 --- a/VENV_GUIDE.txt +++ /dev/null @@ -1,81 +0,0 @@ -# 虚拟环境使用指南 - -## 快速开始(推荐) - -直接双击运行 `start.bat`,脚本会自动: -1. 检查并创建虚拟环境 -2. 激活虚拟环境 -3. 安装依赖包 -4. 启动服务 - -## 手动使用虚拟环境 - -### Windows 系统 - -```bash -# 1. 创建虚拟环境 -python -m venv venv - -# 2. 激活虚拟环境 -venv\Scripts\activate - -# 3. 安装依赖 -pip install -r requirements.txt - -# 4. 运行服务 -python app.py - -# 5. 退出虚拟环境(使用完毕后) -deactivate -``` - -### Linux/Mac 系统 - -```bash -# 1. 创建虚拟环境 -python3 -m venv venv - -# 2. 激活虚拟环境 -source venv/bin/activate - -# 3. 安装依赖 -pip install -r requirements.txt - -# 4. 运行服务 -python app.py - -# 5. 退出虚拟环境 -deactivate -``` - -## 配置说明 - -在启动服务前,需要配置 `.env` 文件: - -**必须配置项:** -- `ADSPOWER_USER_ID`: 在 AdsPower 中创建的用户 ID - -**可选配置项:** -- 其他配置项保持默认值即可 - -## 验证虚拟环境 - -激活虚拟环境后,命令行提示符前会显示 `(venv)`: - -``` -(venv) D:\project\Work\ai_mip> -``` - -## 常见问题 - -**Q: 如何确认已在虚拟环境中?** -A: 命令行提示符前有 `(venv)` 标识 - -**Q: 虚拟环境在哪里?** -A: 项目根目录下的 `venv` 文件夹 - -**Q: 需要每次都激活虚拟环境吗?** -A: 使用 `start.bat` 会自动激活,手动运行需要先激活 - -**Q: 如何重建虚拟环境?** -A: 删除 `venv` 文件夹,重新运行 `start.bat` 或手动创建 diff --git a/ad_automation.py b/ad_automation.py index c61dddd..5a94896 100644 --- a/ad_automation.py +++ b/ad_automation.py @@ -4,28 +4,78 @@ from typing import Optional, Tuple from playwright.sync_api import Page, ElementHandle from loguru import logger from config import Config +from pathlib import Path +from datetime import datetime class MIPAdAutomation: """MIP页面广告自动化操作""" - def __init__(self, page: Page): + # 预设的咨询语句 + CONSULTATION_MESSAGES = [ + "我想要预约一个医生,有什么推荐吗?", + "我现在本人不在当地,医生什么时候有空,是随时能去吗?有没有推荐的医生。", + "咱们医院是周六日是否上班,随时去吗?", + "想找医生看看,有没有推荐的区生", + "最近很不舒服,也说不出来全部的症状,能不能直接对话医生?" + ] + + def __init__(self, page: Page, task_index: int = None): self.page = page + self.site_id = None # 当前站点ID + self.click_id = None # 当前点击ID + self.task_folder = None # 任务日志目录 - def check_and_click_ad(self, url: str) -> Tuple[bool, bool]: + # 创建任务日志目录 + if task_index: + timestamp = datetime.now().strftime('%Y%m%d_%H%M%S') + self.task_folder = Path("./test") / f"task_{task_index}_{timestamp}" + self.task_folder.mkdir(parents=True, exist_ok=True) + logger.info(f"任务日志目录: {self.task_folder}") + + def check_and_click_ad(self, url: str, site_id: int = None) -> Tuple[bool, bool]: """ 检查并点击广告 Args: url: MIP页面链接 + site_id: 站点ID(用于数据库记录) Returns: (是否点击成功, 是否获得回复) """ + self.site_id = site_id + try: - # 访问链接 - logger.info(f"访问链接: {url}") - self.page.goto(url, wait_until='domcontentloaded') + # 访问链接(带重试机制) + max_retries = 2 + page_loaded = False + for attempt in range(max_retries): + try: + logger.info(f"访问链接: {url} (第{attempt+1}次尝试)") + self.page.goto(url, wait_until='domcontentloaded', timeout=30000) + page_loaded = True + break + except Exception as goto_err: + if attempt < max_retries - 1: + logger.warning(f"访问超时,尝试刷新页面...") + try: + self.page.reload(wait_until='domcontentloaded', timeout=30000) + logger.info("✅ 页面刷新成功") + page_loaded = True + break + except: + logger.warning(f"刷新失败,等待2秒后重试...") + time.sleep(2) + else: + logger.error(f"访问链接失败: {str(goto_err)}") + # 记录访问失败 + self._record_click_failure(url, f"访问超时: {str(goto_err)}") + return False, False + + if not page_loaded: + self._record_click_failure(url, "页面加载失败") + return False, False # 等待页面加载 time.sleep(3) @@ -43,9 +93,19 @@ class MIPAdAutomation: logger.warning("点击广告失败") return False, False + # 记录点击到数据库 + self._record_click(url) + + # 发送咨询消息 + message_sent = self._send_consultation_message() + # 等待并检查回复 has_reply = self._wait_for_reply() + # 记录互动到数据库 + if message_sent: + self._record_interaction(has_reply) + return True, has_reply except Exception as e: @@ -99,7 +159,7 @@ class MIPAdAutomation: def _click_advertisement(self, ad_element: ElementHandle) -> bool: """ - 点击广告元素 + 点击广告元素(当前页面导航) Args: ad_element: 广告元素 @@ -108,26 +168,35 @@ class MIPAdAutomation: 是否点击成功 """ try: - # 获取当前页面 - context = self.page.context + original_url = self.page.url # 滚动到广告元素可见 ad_element.scroll_into_view_if_needed() time.sleep(1) - # 监听新页面打开 - with context.expect_page() as new_page_info: - # 点击广告 - ad_element.click() - logger.info("已点击广告") + # 直接点击广告(当前页面导航) + logger.info("点击广告...") + ad_element.click() + logger.info("已点击广告") - # 等待新页面 - new_page = new_page_info.value - new_page.wait_for_load_state('domcontentloaded') + # 等待页面导航(增加等待时间,支持慢速电脑) + logger.info("等待页面跳转...") + max_wait = 10 # 最多等待10秒 + check_interval = 1 # 每秒检查一次 - # 切换到新页面 - self.page = new_page - logger.info("已切换到广告页面") + for i in range(max_wait): + time.sleep(check_interval) + if self.page.url != original_url: + logger.info(f"✅ 页面已导航(耗时{i+1}秒): {original_url} -> {self.page.url}") + self.page.wait_for_load_state('domcontentloaded') + break + else: + # 循环正常结束(未跳转) + logger.error(f"❌ 页面URL未变化(等待{max_wait}秒后),广告点击失败: {self.page.url}") + return False + + # 等待聊天页面加载 + time.sleep(2) return True @@ -135,6 +204,294 @@ class MIPAdAutomation: logger.error(f"点击广告异常: {str(e)}") return False + def _send_consultation_message(self) -> bool: + """ + 在聊天页面发送随机咨询消息 + + Returns: + 是否发送成功 + """ + try: + logger.info("准备发送咨询消息...") + + # 随机选择一条消息 + message = random.choice(self.CONSULTATION_MESSAGES) + logger.info(f"选择的消息: {message}") + + # 等待页面加载完成 + time.sleep(2) + + # 打印当前页面URL + logger.info(f"当前页面: {self.page.url}") + + # 常见的输入框选择器(优先通过placeholder查找) + input_selectors = [ + # 优先:通过placeholder查找 + "textarea[placeholder*='消息']", + "textarea[placeholder*='问题']", + "input[type='text'][placeholder*='消息']", + "input[type='text'][placeholder*='问题']", + "textarea[placeholder*='输入']", + "textarea[placeholder*='发送']", + "input[type='text'][placeholder*='输入']", + "input[type='text'][placeholder*='发送']", + # 次选:通过class查找 + "textarea[class*='input']", + # 兜底:通用选择器 + "div[contenteditable='true']", + "textarea", + "input[type='text']" + ] + + input_element = None + logger.info("开始查找输入框...") + for selector in input_selectors: + try: + elements = self.page.locator(selector).all() + logger.debug(f"选择器 {selector} 找到 {len(elements)} 个元素") + for elem in elements: + if elem.is_visible(): + input_element = elem + logger.info(f"✅ 找到可见输入框: {selector}") + break + if input_element: + break + except Exception as e: + logger.debug(f"选择器 {selector} 失败: {str(e)}") + continue + + if not input_element: + logger.warning("❌ 未找到输入框") + # 尝试截图便于调试 + try: + if self.task_folder: + screenshot_path = self.task_folder / "debug_no_input.png" + else: + screenshot_path = Path(f"./logs/debug_no_input_{int(time.time())}.png") + self.page.screenshot(path=str(screenshot_path)) + logger.info(f"已保存调试截图: {screenshot_path}") + except Exception as e: + logger.warning(f"截图失败: {str(e)}") + + # 兜底方案:尝试查找并点击任何可能的输入区域 + logger.warning("尝试兜底方案:查找所有可能的输入区域...") + try: + # 先滚动到页面最底部 + self.page.evaluate("window.scrollTo(0, document.body.scrollHeight)") + time.sleep(1) + + # 尝试查找所有可能的输入相关元素并点击 + fallback_selectors = [ + "textarea", + "input[type='text']", + "div[contenteditable='true']", + "div[class*='input']", + "div[class*='textarea']", + "div[class*='message']", + "div[class*='chat']", + "div[id*='input']", + "div[id*='message']" + ] + + clicked = False + for selector in fallback_selectors: + try: + elements = self.page.locator(selector).all() + logger.debug(f"兜底选择器 {selector} 找到 {len(elements)} 个元素") + for elem in elements: + if elem.is_visible(): + # 滚动到元素位置 + elem.scroll_into_view_if_needed() + time.sleep(0.5) + # 点击元素 + elem.click() + time.sleep(1) + logger.info(f"已点击元素: {selector}") + clicked = True + break + if clicked: + break + except Exception as e: + logger.debug(f"兜底选择器 {selector} 失败: {str(e)}") + continue + + if clicked: + # 直接输入消息 + self.page.keyboard.type(message, delay=50) + logger.info("✅ 已输入消息(兜底)") + + # 直接按回车发送 + self.page.keyboard.press('Enter') + logger.info("✅ 已按回车键发送(兜底)") + + # 保存已发送的消息内容 + self.sent_message = message + time.sleep(2) + return True + else: + logger.error("❌ 兜底方案未找到任何可点击的输入区域") + return False + + except Exception as fallback_err: + logger.error(f"兜底方案失败: {str(fallback_err)}") + return False + + # 正常流程:点击输入框获取焦点 + input_element.click() + time.sleep(0.5) + + # 输入消息 + input_element.fill(message) + logger.info("✅ 已输入消息") + time.sleep(1) + + # 尝试发送消息(优先回车,再尝试按钮) + sent = False + + # 方法1(优先):按回车键发送 + try: + logger.info("尝试按回车键发送...") + input_element.press('Enter') + logger.info("✅ 已按回车键发送") + sent = True + time.sleep(1) + except Exception as e: + logger.warning(f"❌ 按回车键失败: {str(e)}") + + # 方法2(兜底): 尝试找到发送按钮并点击 + if not sent: + send_button_selectors = [ + "button:has-text('发送')", + "button[class*='send']", + "button[type='submit']", + "div[class*='send']", + "span:has-text('发送')" + ] + + logger.info("开始查找发送按钮...") + for selector in send_button_selectors: + try: + buttons = self.page.locator(selector).all() + logger.debug(f"选择器 {selector} 找到 {len(buttons)} 个按钮") + for btn in buttons: + if btn.is_visible() and btn.is_enabled(): + btn.click() + logger.info(f"✅ 已点击发送按钮: {selector}") + sent = True + break + if sent: + break + except Exception as e: + logger.debug(f"选择器 {selector} 失败: {str(e)}") + continue + + if sent: + logger.info("✅ 消息发送成功") + # 保存已发送的消息内容 + self.sent_message = message + time.sleep(2) # 等待消息发送完成 + return True + else: + logger.warning("❌ 未能发送消息") + # 截图调试 + try: + if self.task_folder: + screenshot_path = self.task_folder / "debug_send_failed.png" + else: + screenshot_path = Path(f"./logs/debug_send_failed_{int(time.time())}.png") + self.page.screenshot(path=str(screenshot_path)) + logger.info(f"已保存调试截图: {screenshot_path}") + except: + pass + return False + + except Exception as e: + logger.error(f"发送消息异常: {str(e)}") + import traceback + traceback.print_exc() + return False + + def _record_click(self, site_url: str): + """记录点击到数据库""" + try: + if not self.site_id: + logger.warning("未设置 site_id,跳过点击记录") + return + + from db_manager import ClickManager + click_mgr = ClickManager() + self.click_id = click_mgr.record_click( + site_id=self.site_id, + site_url=site_url, + user_ip=None, # 可以后续添加代理IP + device_type='pc' + ) + logger.info(f"已记录点击: click_id={self.click_id}") + except Exception as e: + logger.error(f"记录点击失败: {str(e)}") + + def _record_click_failure(self, site_url: str, error_message: str): + """ + 记录点击失败到数据库 + + Args: + site_url: 站点URL + error_message: 错误信息 + """ + try: + if not self.site_id: + logger.warning("未设置 site_id,跳过失败记录") + return + + from db_manager import ClickManager + click_mgr = ClickManager() + # 记录点击(失败也计数) + self.click_id = click_mgr.record_click( + site_id=self.site_id, + site_url=site_url, + user_ip=None, + device_type='pc' + ) + + # 记录互动失败 + from db_manager import InteractionManager + interaction_mgr = InteractionManager() + interaction_mgr.record_interaction( + site_id=self.site_id, + click_id=self.click_id, + interaction_type='reply', + reply_content=None, + is_successful=False, + response_received=False, + error_message=error_message + ) + logger.info(f"已记录失败: {error_message}") + except Exception as e: + logger.error(f"记录失败异常: {str(e)}") + + def _record_interaction(self, response_received: bool): + """记录互动到数据库""" + try: + if not self.site_id: + logger.warning("未设置 site_id,跳过互动记录") + return + + from db_manager import InteractionManager + interaction_mgr = InteractionManager() + + interaction_id = interaction_mgr.record_interaction( + site_id=self.site_id, + click_id=self.click_id, + interaction_type='message', # 符合数据库ENUM定义:reply/comment/message/form_submit/follow/like/share + reply_content=getattr(self, 'sent_message', None), + is_successful=True, + response_received=response_received, + response_content=None # 可以后续添加提取回复内容 + ) + logger.info(f"已记录互动: interaction_id={interaction_id}, response={response_received}") + except Exception as e: + logger.error(f"记录互动失败: {str(e)}") + def _wait_for_reply(self) -> bool: """ 等待广告主回复 diff --git a/adspower_client.py b/adspower_client.py index 3c5e2aa..fff5562 100644 --- a/adspower_client.py +++ b/adspower_client.py @@ -1,9 +1,9 @@ import requests +import json from typing import Dict, Optional from loguru import logger from playwright.sync_api import sync_playwright, Browser, Page from config import Config -import json class AdsPowerClient: @@ -28,6 +28,72 @@ class AdsPowerClient: logger.debug(f"AdsPowerClient 初始化 - API Key: {self.api_key[:8]}... (len={len(self.api_key)})") else: logger.debug("AdsPowerClient 初始化 - 未配置 API Key") + + def _make_request(self, method: str, endpoint: str, json: Dict = None, params: Dict = None) -> Optional[Dict]: + """ + 通用 API 请求方法 + + Args: + method: HTTP方法 (GET/POST/PUT/DELETE) + endpoint: API端点,如 '/api/v2/browser-profile/create' + json: JSON请求体 + params: URL查询参数 + + Returns: + 响应JSON或None + """ + try: + import json as json_lib # 避免参数名冲突 + + url = f"{self.api_url}{endpoint}" + + headers = { + 'Content-Type': 'application/json' + } + if self.api_key: + headers['Authorization'] = f'Bearer {self.api_key}' + + # 记录请求 + logger.info("\n" + "="*70) + logger.info(f"API请求: {method} {endpoint}") + logger.info("="*70) + logger.info(f"URL: {url}") + if params: + logger.info(f"Params:\n{json_lib.dumps(params, indent=2, ensure_ascii=False)}") + if json: + logger.info(f"Body:\n{json_lib.dumps(json, indent=2, ensure_ascii=False)}") + + response = requests.request( + method=method, + url=url, + json=json, + params=params, + headers=headers, + timeout=30 + ) + + # 记录响应 + logger.info("\n" + "-"*70) + logger.info("API响应") + logger.info("-"*70) + logger.info(f"Status: {response.status_code}") + + try: + response_data = response.json() + logger.info(f"Body:\n{json_lib.dumps(response_data, indent=2, ensure_ascii=False)}") + except: + logger.info(f"Body (Raw):\n{response.text}") + response_data = None + + logger.info("="*70 + "\n") + + return response_data + + except Exception as e: + logger.error(f"API请求异常 [{method} {endpoint}]: {str(e)}") + import traceback + traceback.print_exc() + return None def update_profile_proxy(self, profile_id: str, proxy_id: str) -> bool: """ @@ -553,21 +619,70 @@ class AdsPowerClient: Playwright Browser 实例 """ try: + import asyncio + import sys + + # 检测是否在 asyncio 事件循环中 + try: + loop = asyncio.get_running_loop() + logger.warning("检测到 asyncio 事件循环,将在新线程中执行 Playwright") + + # 在新线程中执行 Playwright 同步 API + import threading + result_container = {'browser': None, 'error': None} + + def run_playwright(): + try: + # 获取 CDP WebSocket 端点 + ws_endpoint = browser_info['data']['ws']['puppeteer'] + + # 创建新的 Playwright 实例 + playwright = sync_playwright().start() + + # 通过 CDP 连接到浏览器 + browser = playwright.chromium.connect_over_cdp(ws_endpoint) + logger.info("成功通过 CDP 连接到 AdsPower 浏览器") + + # 保存引用 + self.playwright = playwright + self.browser = browser + result_container['browser'] = browser + except Exception as e: + result_container['error'] = str(e) + + thread = threading.Thread(target=run_playwright) + thread.start() + thread.join(timeout=30) + + if result_container['error']: + raise Exception(result_container['error']) + + return result_container['browser'] + + except RuntimeError: + # 没有运行中的事件循环,正常执行 + pass + # 获取 CDP WebSocket 端点 ws_endpoint = browser_info['data']['ws']['puppeteer'] - # 初始化 Playwright - if not self.playwright: - self.playwright = sync_playwright().start() + # 创建新的 Playwright 实例 + playwright = sync_playwright().start() # 通过 CDP 连接到浏览器 - self.browser = self.playwright.chromium.connect_over_cdp(ws_endpoint) + browser = playwright.chromium.connect_over_cdp(ws_endpoint) logger.info("成功通过 CDP 连接到 AdsPower 浏览器") - return self.browser + # 保存引用 + self.playwright = playwright + self.browser = browser + + return browser except Exception as e: logger.error(f"CDP 连接失败: {str(e)}") + import traceback + traceback.print_exc() return None def get_page(self, browser: Browser) -> Optional[Page]: @@ -912,7 +1027,119 @@ class AdsPowerClient: logger.error(f"检查浏览器状态异常: {str(e)}") return None - def list_profiles(self, group_id: str = None, page: int = 1, page_size: int = 100) -> Optional[Dict]: + def list_groups(self, group_name: str = None, page: int = 1, page_size: int = 2000) -> Optional[Dict]: + """ + 查询分组列表 + 使用 AdsPower API v1 + + Args: + group_name: 分组名称(可选) + page: 页码 + page_size: 每页数量(范围 1 ~ 2000) + + Returns: + 分组列表信息 + """ + try: + url = f"{self.api_url}/api/v1/group/list" + + # 准备请求头 + headers = {} + if self.api_key: + headers['Authorization'] = f'Bearer {self.api_key}' + + # 准备请求参数 + params = { + "page": page, + "page_size": page_size + } + if group_name: + params["group_name"] = group_name + + logger.info("\n" + "="*70) + logger.info("📂 查询分组列表") + logger.info("="*70) + logger.info(f"URL: {url}") + logger.info(f"Method: GET") + logger.info(f"Params: {params}") + + response = requests.get(url, params=params, headers=headers, timeout=30) + + logger.info("\n" + "-"*70) + logger.info("📥 响应信息") + logger.info("-"*70) + logger.info(f"Status Code: {response.status_code}") + + try: + response_json = response.json() + logger.info(f"Response Body:") + logger.info(json.dumps(response_json, indent=2, ensure_ascii=False)) + except: + logger.info(f"Response Body (Raw):") + logger.info(response.text) + + logger.info("="*70 + "\n") + + result = response_json if 'response_json' in locals() else response.json() + + if result.get('code') == 0: + groups = result.get('data', {}).get('list', []) + logger.success(f"查询成功,找到 {len(groups)} 个分组") + + if groups: + logger.info("\n分组列表:") + for idx, group in enumerate(groups, 1): + group_id = group.get('group_id', 'N/A') + group_name = group.get('group_name', 'N/A') + remark = group.get('remark', '') + logger.info(f" {idx}. ID: {group_id} | 名称: {group_name} | 备注: {remark}") + + return result + else: + logger.error(f"查询分组失败: {result.get('msg')}") + return None + + except Exception as e: + logger.error(f"查询分组异常: {str(e)}") + return None + + def get_group_by_env(self) -> Optional[str]: + """ + 根据当前运行环境获取对应的分组ID + dev环境查询 group_name=dev + 生产环境查询 group_name=prod + + Returns: + 分组ID,失败返回None + """ + # 根据环境决定分组名 + group_name = 'dev' if Config.ENV == 'development' else 'prod' + + logger.info(f"当前运行环境: {Config.ENV},查询分组: {group_name}") + + # 等待一下避免请求过于频繁 + import time + time.sleep(1.0) + + # 查询分组 + result = self.list_groups(group_name=group_name) + + if result and result.get('code') == 0: + groups = result.get('data', {}).get('list', []) + if groups: + group_id = groups[0].get('group_id') + logger.success(f"获取到分组ID: {group_id}") + return group_id + else: + logger.warning(f"未找到名为 '{group_name}' 的分组") + return None + else: + logger.error(f"查询分组失败") + return None + + def list_profiles(self, group_id: str = None, page: int = 1, page_size: int = 100, + profile_id: list = None, profile_no: list = None, + limit: int = None, sort_type: str = None, sort_order: str = None) -> Optional[Dict]: """ 查询 Profile 列表 使用 AdsPower API v2 @@ -920,7 +1147,12 @@ class AdsPowerClient: Args: group_id: 组ID(可选) page: 页码 - page_size: 每页数量 + page_size: 每页数量(当limit未指定时使用) + profile_id: 环境ID数组(可选) + profile_no: 环境编号数组(可选) + limit: 每页大小(范围 1 ~ 200) + sort_type: 排序类型 (profile_no/last_open_time/created_time) + sort_order: 排序顺序 (asc/desc) Returns: Profile 列表信息 @@ -938,10 +1170,18 @@ class AdsPowerClient: # 准备请求体 payload = { "page": page, - "page_size": page_size + "page_size": limit if limit else page_size } if group_id: payload["group_id"] = group_id + if profile_id: + payload["profile_id"] = profile_id + if profile_no: + payload["profile_no"] = profile_no + if sort_type: + payload["sort_type"] = sort_type + if sort_order: + payload["sort_order"] = sort_order # 打印请求信息 logger.info("\n" + "="*70) @@ -1018,6 +1258,35 @@ class AdsPowerClient: logger.error(f"查询 Profile 异常: {str(e)}") return None + def delete_profile(self, profile_id: str) -> bool: + """ + 删除 Profile + 使用 AdsPower API v2 + + Args: + profile_id: Profile ID + + Returns: + 是否成功删除 + """ + try: + result = self._make_request( + 'POST', + '/api/v2/browser-profile/delete', + json={"profile_id": [profile_id]} + ) + + if result and result.get('code') == 0: + logger.success(f"✅ 成功删除 Profile: {profile_id}") + return True + else: + logger.error(f"删除 Profile 失败: {result.get('msg') if result else '请求失败'}") + return False + + except Exception as e: + logger.error(f"删除 Profile 异常: {str(e)}") + return False + def __del__(self): """析构函数,确保资源清理""" try: diff --git a/ai_mip.zip b/ai_mip.zip new file mode 100644 index 0000000..f672ac8 Binary files /dev/null and b/ai_mip.zip differ diff --git a/app.py b/app.py index 41fed35..e0e8e32 100644 --- a/app.py +++ b/app.py @@ -33,8 +33,8 @@ scheduler = ClickScheduler() @app.route('/') def index(): - """首页 - 重定向到数据概览""" - return redirect('/dashboard.html') + """首页 - 重定向到新的单页应用""" + return redirect('/app.html') @app.route('/health', methods=['GET']) @@ -196,14 +196,103 @@ def get_scheduler_status(): # AdsPower 接口调试 -@app.route('/api/adspower/profiles', methods=['GET']) -def adspower_list_profiles(): - """查询Profile列表""" +@app.route('/api/adspower/groups', methods=['GET']) +def adspower_list_groups(): + """查询分组列表""" try: from adspower_client import AdsPowerClient + + group_name = request.args.get('group_name') + page = request.args.get('page', 1, type=int) + page_size = request.args.get('page_size', 2000, type=int) + client = AdsPowerClient() - result = client.list_profiles() - return jsonify({'success': True, 'data': result}) + result = client.list_groups(group_name=group_name, page=page, page_size=page_size) + + if result: + return jsonify({'success': True, 'data': result}) + else: + return jsonify({'success': False, 'message': '查询分组列表失败'}), 500 + except Exception as e: + logger.error(f"AdsPower查询分组异常: {str(e)}") + return jsonify({'success': False, 'message': str(e)}), 500 + + +@app.route('/api/adspower/group/env', methods=['GET']) +def adspower_get_group_by_env(): + """根据当前运行环境获取对应的分组ID""" + try: + from adspower_client import AdsPowerClient + + client = AdsPowerClient() + group_id = client.get_group_by_env() + + if group_id: + return jsonify({'success': True, 'data': {'group_id': group_id, 'env': Config.ENV}}) + else: + return jsonify({'success': False, 'message': f'未找到对应环境的分组'}), 404 + except Exception as e: + logger.error(f"AdsPower获取环境分组异常: {str(e)}") + return jsonify({'success': False, 'message': str(e)}), 500 + + +@app.route('/api/adspower/profiles', methods=['GET']) +def adspower_list_profiles(): + """查询Profile列表(支持多个查询参数)""" + try: + from adspower_client import AdsPowerClient + import json as json_module + + # 获取查询参数 + group_id = request.args.get('group_id') + page = request.args.get('page', 1, type=int) + limit = request.args.get('limit', type=int) # 可选 + page_size = request.args.get('page_size', type=int) # 可选 + + # 数组参数(JSON格式) + profile_id = request.args.get('profile_id') + profile_no = request.args.get('profile_no') + + # 解析JSON数组 + if profile_id: + try: + profile_id = json_module.loads(profile_id) + except: + profile_id = None + + if profile_no: + try: + profile_no = json_module.loads(profile_no) + except: + profile_no = None + + # 排序参数 + sort_type = request.args.get('sort_type') + sort_order = request.args.get('sort_order') + + # 如果没有指定group_id,尝试根据环境自动获取 + client = AdsPowerClient() + if not group_id: + group_id = client.get_group_by_env() + if group_id: + logger.info(f"自动获取到分组ID: {group_id}") + + # 查询Profile列表 + result = client.list_profiles( + group_id=group_id, + page=page, + page_size=page_size if page_size else 100, + profile_id=profile_id, + profile_no=profile_no, + limit=limit, + sort_type=sort_type, + sort_order=sort_order + ) + + if result: + return jsonify({'success': True, 'data': result}) + else: + return jsonify({'success': False, 'message': '查询Profile列表失败'}), 500 except Exception as e: logger.error(f"AdsPower查询Profile异常: {str(e)}") return jsonify({'success': False, 'message': str(e)}), 500 @@ -338,6 +427,71 @@ def adspower_update_profile_v1(): return jsonify({'success': False, 'message': str(e)}), 500 +# 数据库查询接口 +@app.route('/api/clicks', methods=['GET']) +def get_all_clicks(): + """获取所有点击记录""" + try: + from db_manager import ClickManager + + site_id = request.args.get('site_id', type=int) + limit = request.args.get('limit', 100, type=int) + + click_mgr = ClickManager() + + if site_id: + clicks = click_mgr.get_clicks_by_site(site_id, limit=limit) + else: + # 获取所有点击记录 + conn = click_mgr.get_connection() + cursor = conn.cursor() + cursor.execute(f""" + SELECT * FROM ai_mip_click + ORDER BY click_time DESC + LIMIT ? + """, (limit,)) + rows = cursor.fetchall() + conn.close() + clicks = [click_mgr._dict_from_row(row) for row in rows] + + return jsonify({'success': True, 'data': clicks}) + except Exception as e: + logger.error(f"查询点击记录异常: {str(e)}") + return jsonify({'success': False, 'message': str(e)}), 500 + + +@app.route('/api/interactions', methods=['GET']) +def get_all_interactions(): + """获取所有互动记录""" + try: + from db_manager import InteractionManager + + site_id = request.args.get('site_id', type=int) + limit = request.args.get('limit', 100, type=int) + + interaction_mgr = InteractionManager() + + if site_id: + interactions = interaction_mgr.get_interactions_by_site(site_id, limit=limit) + else: + # 获取所有互动记录 + conn = interaction_mgr.get_connection() + cursor = conn.cursor() + cursor.execute(f""" + SELECT * FROM ai_mip_interaction + ORDER BY interaction_time DESC + LIMIT ? + """, (limit,)) + rows = cursor.fetchall() + conn.close() + interactions = [interaction_mgr._dict_from_row(row) for row in rows] + + return jsonify({'success': True, 'data': interactions}) + except Exception as e: + logger.error(f"查询互动记录异常: {str(e)}") + return jsonify({'success': False, 'message': str(e)}), 500 + + if __name__ == '__main__': logger.info(f"启动MIP广告点击服务 - 环境: {Config.ENV}") logger.info(f"服务地址: http://{Config.SERVER_HOST}:{Config.SERVER_PORT}") diff --git a/batch_insert_urls.py b/batch_insert_urls.py new file mode 100644 index 0000000..23dba72 --- /dev/null +++ b/batch_insert_urls.py @@ -0,0 +1,62 @@ +"""批量插入URL到数据库""" +from data_manager import DataManager + +# 要插入的URL列表 +urls = [ + "https://health.baidu.com/m/detail/ar_1763832104063502612", + "https://health.baidu.com/m/detail/ar_3234161746463547514", + "https://health.baidu.com/m/detail/ar_2979413891570169996", + "https://health.baidu.com/m/detail/ar_2956015846029041423", + "https://health.baidu.com/m/detail/ar_168792171069657865", + "https://health.baidu.com/m/detail/ar_6465728881863076989", + "https://health.baidu.com/m/detail/ar_5239302258777444788", + "https://health.baidu.com/m/detail/ar_4713935339392349406", + "https://health.baidu.com/m/detail/ar_5279303492380349045", + "https://health.baidu.com/m/detail/ar_3049436766450657685", + "https://health.baidu.com/m/detail/ar_2014490668952387433", +] + +print("=" * 60) +print("批量插入URL到数据库") +print("=" * 60) + +# 创建数据管理器 +dm = DataManager() +print(f"\n存储方式: {'SQLite数据库' if dm.use_database else 'JSON文件'}") +print(f"总URL数: {len(urls)}\n") + +# 批量插入 +success_count = 0 +failed_count = 0 + +for idx, url in enumerate(urls, 1): + print(f"[{idx}/{len(urls)}] 插入: {url}") + result = dm.add_url(url) + + if result: + success_count += 1 + print(f" ✓ 成功") + else: + failed_count += 1 + print(f" × 失败(可能已存在)") + +# 统计结果 +print("\n" + "=" * 60) +print("插入完成") +print("=" * 60) +print(f"成功: {success_count} 个") +print(f"失败: {failed_count} 个") + +# 显示当前数据库统计 +print("\n数据库统计:") +stats = dm.get_statistics() +for key, value in stats.items(): + print(f" {key}: {value}") + +# 显示所有活跃URL +print("\n活跃URL列表:") +active_urls = dm.get_active_urls() +for idx, site in enumerate(active_urls[:15], 1): + site_url = site.get('site_url', site.get('url')) + click_count = site.get('click_count', 0) + print(f" {idx}. {site_url} (点击: {click_count}次)") diff --git a/config.py b/config.py index 282e425..5d48f1e 100644 --- a/config.py +++ b/config.py @@ -57,6 +57,16 @@ class BaseConfig: # 调试模式 DEBUG = os.getenv('DEBUG', 'False').lower() == 'true' + # 测试配置 + AUTO_CLOSE_BROWSER = os.getenv('AUTO_CLOSE_BROWSER', 'False').lower() == 'true' # 测试完成后是否自动关闭浏览器 + + # MySQL数据库配置 + MYSQL_HOST = os.getenv('MYSQL_HOST', 'localhost') + MYSQL_PORT = int(os.getenv('MYSQL_PORT', 3306)) + MYSQL_USER = os.getenv('MYSQL_USER', 'root') + MYSQL_PASSWORD = os.getenv('MYSQL_PASSWORD', '') + MYSQL_DATABASE = os.getenv('MYSQL_DATABASE', 'ai_article') + @classmethod def ensure_dirs(cls): """确保必要的目录存在""" diff --git a/data_manager.py b/data_manager.py index 5b0c0b7..91bfcd2 100644 --- a/data_manager.py +++ b/data_manager.py @@ -1,280 +1,84 @@ -import json -import random +"""数据管理器 - MySQL数据库存储""" + +import warnings +warnings.filterwarnings('ignore', category=DeprecationWarning) + from datetime import datetime from typing import List, Dict, Optional -from pathlib import Path from loguru import logger from config import Config +# 导入MySQL数据库管理器 +try: + from db_manager import SiteManager, ClickManager, InteractionManager, StatisticsManager + logger.info("使用MySQL数据库存储") +except Exception as e: + logger.error(f"MySQL数据库初始化失败: {str(e)}") + raise + class DataManager: - """数据管理器,负责URL和统计数据的存储与管理""" + """数据管理器 - MySQL数据库存储""" def __init__(self, data_file: str = None): - self.data_file = data_file or str(Path(Config.DATA_DIR) / 'urls_data.json') - self._ensure_data_file() - self.data = self._load_data() - - def _ensure_data_file(self): - """确保数据文件存在""" - Config.ensure_dirs() - if not Path(self.data_file).exists(): - self._save_data({'urls': {}}) - - def _load_data(self) -> Dict: - """加载数据""" - try: - with open(self.data_file, 'r', encoding='utf-8') as f: - return json.load(f) - except Exception as e: - logger.error(f"加载数据文件失败: {str(e)}") - return {'urls': {}} - - def _save_data(self, data: Dict = None): - """保存数据""" - try: - save_data = data if data is not None else self.data - with open(self.data_file, 'w', encoding='utf-8') as f: - json.dump(save_data, f, ensure_ascii=False, indent=2) - except Exception as e: - logger.error(f"保存数据文件失败: {str(e)}") + """初始化MySQL数据库管理器""" + self.site_mgr = SiteManager() + self.click_mgr = ClickManager() + self.interaction_mgr = InteractionManager() + self.stats_mgr = StatisticsManager() + logger.info("初始化MySQL数据库管理器") + def add_url(self, url: str) -> bool: - """ - 添加新URL - - Args: - url: MIP页面链接 - - Returns: - 是否添加成功 - """ - try: - if url in self.data['urls']: - logger.warning(f"URL已存在: {url}") - return False - - # 生成随机目标点击次数 - target_clicks = random.randint(Config.MIN_CLICK_COUNT, Config.MAX_CLICK_COUNT) - - self.data['urls'][url] = { - 'url': url, - 'status': 'active', # active, completed, failed - 'target_clicks': target_clicks, - 'click_count': 0, - 'reply_count': 0, - 'created_time': datetime.now().isoformat(), - 'last_click_time': None, - 'click_history': [] - } - - self._save_data() - logger.info(f"成功添加URL,目标点击次数: {target_clicks}") - return True - - except Exception as e: - logger.error(f"添加URL失败: {str(e)}") - return False + """添加新URL到MySQL数据库""" + site_id = self.site_mgr.add_site( + site_url=url, + site_name=url, + site_dimension='MIP广告' + ) + return site_id is not None def record_click(self, url: str, has_reply: bool = False): - """ - 记录一次点击 - - Args: - url: URL - has_reply: 是否获得回复 - """ + """记录一次点击到MySQL数据库""" try: - if url not in self.data['urls']: + site = self.site_mgr.get_site_by_url(url) + if not site: logger.error(f"URL不存在: {url}") return - url_data = self.data['urls'][url] - url_data['click_count'] += 1 - url_data['last_click_time'] = datetime.now().isoformat() + click_id = self.click_mgr.record_click( + site_id=site['id'], + site_url=url + ) - if has_reply: - url_data['reply_count'] += 1 + if click_id and has_reply: + self.interaction_mgr.record_interaction( + site_id=site['id'], + click_id=click_id, + interaction_type='reply', + is_successful=True, + response_received=True + ) - # 记录点击历史 - url_data['click_history'].append({ - 'time': datetime.now().isoformat(), - 'has_reply': has_reply - }) - - self._save_data() - logger.info(f"记录点击成功,总点击次数: {url_data['click_count']}/{url_data['target_clicks']}") + logger.info(f"记录点击成功: {url}, has_reply={has_reply}") except Exception as e: logger.error(f"记录点击失败: {str(e)}") - - def mark_url_completed(self, url: str): - """ - 标记URL为已完成 - - Args: - url: URL - """ - try: - if url not in self.data['urls']: - logger.error(f"URL不存在: {url}") - return - - self.data['urls'][url]['status'] = 'completed' - self.data['urls'][url]['completed_time'] = datetime.now().isoformat() - self._save_data() - - logger.info(f"URL已完成: {url}") - - except Exception as e: - logger.error(f"标记URL完成失败: {str(e)}") - - def mark_url_failed(self, url: str, reason: str = ""): - """ - 标记URL为失败 - - Args: - url: URL - reason: 失败原因 - """ - try: - if url not in self.data['urls']: - logger.error(f"URL不存在: {url}") - return - - self.data['urls'][url]['status'] = 'failed' - self.data['urls'][url]['failed_reason'] = reason - self.data['urls'][url]['failed_time'] = datetime.now().isoformat() - self._save_data() - - logger.warning(f"URL标记为失败: {url}, 原因: {reason}") - - except Exception as e: - logger.error(f"标记URL失败状态失败: {str(e)}") + def get_active_urls(self) -> List[Dict]: - """ - 获取所有活跃的URL - - Returns: - 活跃URL列表 - """ - try: - active_urls = [ - data for data in self.data['urls'].values() - if data['status'] == 'active' - ] - return active_urls - except Exception as e: - logger.error(f"获取活跃URL失败: {str(e)}") - return [] + """获取所有活跃的URL""" + return self.site_mgr.get_active_sites() def get_url_info(self, url: str) -> Optional[Dict]: - """ - 获取URL详细信息 - - Args: - url: URL - - Returns: - URL信息 - """ - return self.data['urls'].get(url) + """获取URL详细信息""" + return self.site_mgr.get_site_by_url(url) def get_all_urls(self) -> List[Dict]: - """ - 获取所有URL - - Returns: - 所有URL列表 - """ - return list(self.data['urls'].values()) + """获取所有URL""" + return self.site_mgr.get_all_sites() def get_statistics(self) -> Dict: - """ - 获取统计数据 - - Returns: - 统计数据 - """ - try: - total_urls = len(self.data['urls']) - active_urls = sum(1 for data in self.data['urls'].values() if data['status'] == 'active') - completed_urls = sum(1 for data in self.data['urls'].values() if data['status'] == 'completed') - failed_urls = sum(1 for data in self.data['urls'].values() if data['status'] == 'failed') - - total_clicks = sum(data['click_count'] for data in self.data['urls'].values()) - total_replies = sum(data['reply_count'] for data in self.data['urls'].values()) - - stats = { - 'total_urls': total_urls, - 'active_urls': active_urls, - 'completed_urls': completed_urls, - 'failed_urls': failed_urls, - 'total_clicks': total_clicks, - 'total_replies': total_replies, - 'reply_rate': f"{(total_replies / total_clicks * 100) if total_clicks > 0 else 0:.2f}%" - } - - return stats - - except Exception as e: - logger.error(f"获取统计数据失败: {str(e)}") - return {} - - def delete_url(self, url: str) -> bool: - """ - 删除URL - - Args: - url: URL - - Returns: - 是否删除成功 - """ - try: - if url in self.data['urls']: - del self.data['urls'][url] - self._save_data() - logger.info(f"已删除URL: {url}") - return True - else: - logger.warning(f"URL不存在: {url}") - return False - except Exception as e: - logger.error(f"删除URL失败: {str(e)}") - return False - - def reset_url(self, url: str) -> bool: - """ - 重置URL状态(重新开始点击) - - Args: - url: URL - - Returns: - 是否重置成功 - """ - try: - if url not in self.data['urls']: - logger.warning(f"URL不存在: {url}") - return False - - # 生成新的随机目标点击次数 - target_clicks = random.randint(Config.MIN_CLICK_COUNT, Config.MAX_CLICK_COUNT) - - self.data['urls'][url]['status'] = 'active' - self.data['urls'][url]['target_clicks'] = target_clicks - self.data['urls'][url]['click_count'] = 0 - self.data['urls'][url]['reply_count'] = 0 - self.data['urls'][url]['last_click_time'] = None - self.data['urls'][url]['click_history'] = [] - self.data['urls'][url]['reset_time'] = datetime.now().isoformat() - - self._save_data() - logger.info(f"已重置URL: {url}") - return True - - except Exception as e: - logger.error(f"重置URL失败: {str(e)}") - return False + """获取统计数据""" + return self.stats_mgr.get_statistics() + diff --git a/db/ai_mip_click.sql b/db/ai_mip_click.sql new file mode 100644 index 0000000..c4804ba --- /dev/null +++ b/db/ai_mip_click.sql @@ -0,0 +1,51 @@ +/* + Navicat Premium Dump SQL + + Source Server : mixue + Source Server Type : MySQL + Source Server Version : 90001 (9.0.1) + Source Host : localhost:3306 + Source Schema : ai_article + + Target Server Type : MySQL + Target Server Version : 90001 (9.0.1) + File Encoding : 65001 + + Date: 12/01/2026 20:31:43 +*/ + +SET NAMES utf8mb4; +SET FOREIGN_KEY_CHECKS = 0; + +-- ---------------------------- +-- Table structure for ai_mip_click +-- ---------------------------- +DROP TABLE IF EXISTS `ai_mip_click`; +CREATE TABLE `ai_mip_click` ( + `id` bigint NOT NULL AUTO_INCREMENT COMMENT '主键ID', + `site_id` bigint NOT NULL COMMENT '关联站点ID(外键指向 ai_mip_site.id)', + `site_url` varchar(512) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci NOT NULL COMMENT '网站URL(冗余字段,便于查询优化)', + `click_time` datetime NOT NULL COMMENT '点击发生时间', + `user_ip` varchar(64) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci NULL DEFAULT NULL COMMENT '用户IP地址', + `user_agent` text CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci NULL COMMENT '浏览器/设备信息', + `referer_url` varchar(512) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci NULL DEFAULT NULL COMMENT '来源页面URL', + `device_type` enum('mobile','pc','tablet') CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci NULL DEFAULT NULL COMMENT '设备类型', + `click_count` int NULL DEFAULT 1 COMMENT '本次点击事件的计数(一般为1,可用于批量插入)', + `is_valid` tinyint(1) NULL DEFAULT 1 COMMENT '是否有效点击(防刷)', + `task_id` varchar(64) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci NULL DEFAULT NULL COMMENT 'RPA任务ID(可选)', + `operator` varchar(64) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci NULL DEFAULT NULL COMMENT '操作者(如自动系统)', + `created_at` timestamp NULL DEFAULT CURRENT_TIMESTAMP COMMENT '记录创建时间', + PRIMARY KEY (`id`) USING BTREE, + INDEX `idx_site_id`(`site_id` ASC) USING BTREE, + INDEX `idx_click_time`(`click_time` ASC) USING BTREE, + INDEX `idx_site_url`(`site_url` ASC) USING BTREE, + INDEX `idx_click_time_site`(`click_time` ASC, `site_id` ASC) USING BTREE, + INDEX `idx_task_id`(`task_id` ASC) USING BTREE +) ENGINE = InnoDB AUTO_INCREMENT = 2 CHARACTER SET = utf8mb4 COLLATE = utf8mb4_0900_ai_ci COMMENT = 'MIP页广告点击日志表' ROW_FORMAT = Dynamic; + +-- ---------------------------- +-- Records of ai_mip_click +-- ---------------------------- +INSERT INTO `ai_mip_click` VALUES (1, 1, 'https://example.com', '2026-01-12 20:25:09', NULL, NULL, NULL, NULL, 1, 1, 'TASK20260112001', 'RPA_SYSTEM', '2026-01-12 20:25:09'); + +SET FOREIGN_KEY_CHECKS = 1; diff --git a/db/ai_mip_interaction.sql b/db/ai_mip_interaction.sql new file mode 100644 index 0000000..85a9c76 --- /dev/null +++ b/db/ai_mip_interaction.sql @@ -0,0 +1,75 @@ +/* + Navicat Premium Dump SQL + + Source Server : mixue + Source Server Type : MySQL + Source Server Version : 90001 (9.0.1) + Source Host : localhost:3306 + Source Schema : ai_article + + Target Server Type : MySQL + Target Server Version : 90001 (9.0.1) + File Encoding : 65001 + + Date: 12/01/2026 20:31:30 +*/ + +SET NAMES utf8mb4; +SET FOREIGN_KEY_CHECKS = 0; + +-- ---------------------------- +-- Table structure for ai_mip_interaction +-- ---------------------------- +DROP TABLE IF EXISTS `ai_mip_interaction`; +CREATE TABLE `ai_mip_interaction` ( + `id` bigint NOT NULL AUTO_INCREMENT COMMENT '主键ID', + `site_id` bigint NOT NULL COMMENT '关联站点ID', + `click_id` bigint NULL DEFAULT NULL COMMENT '关联点击记录ID', + `task_id` varchar(64) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci NULL DEFAULT NULL COMMENT 'RPA任务ID', + `interaction_type` enum('reply','comment','message','form_submit','follow','like','share') CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci NOT NULL COMMENT '互动类型', + `interaction_time` datetime NOT NULL COMMENT '互动发生时间', + `interaction_status` enum('pending','success','failed','skipped') CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci NULL DEFAULT 'pending' COMMENT '互动状态', + `reply_content` text CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci NULL COMMENT '回复/评论的内容', + `reply_template_id` varchar(64) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci NULL DEFAULT NULL COMMENT '使用的回复模板ID', + `ad_element_xpath` varchar(512) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci NULL DEFAULT NULL COMMENT '广告元素的XPath定位', + `ad_element_selector` varchar(512) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci NULL DEFAULT NULL COMMENT '广告元素的CSS选择器', + `ad_text_content` text CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci NULL COMMENT '广告的文本内容', + `execution_mode` enum('auto','manual','semi_auto') CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci NULL DEFAULT 'auto' COMMENT '执行方式', + `rpa_script` varchar(255) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci NULL DEFAULT NULL COMMENT '使用的RPA脚本名称', + `browser_type` enum('headless','headed','playwright','selenium') CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci NULL DEFAULT NULL COMMENT '浏览器类型', + `anti_detection_method` json NULL COMMENT '万金油技术方案', + `proxy_ip` varchar(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci NULL DEFAULT NULL COMMENT '使用的代理IP', + `user_agent` text CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci NULL COMMENT '使用的User-Agent', + `custom_headers` json NULL COMMENT '自定义HTTP头', + `fingerprint_id` varchar(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci NULL DEFAULT NULL COMMENT '浏览器指纹ID', + `response_received` tinyint(1) NULL DEFAULT 0 COMMENT '是否收到回复', + `response_content` text CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci NULL COMMENT '对方回复的内容', + `response_time` datetime NULL DEFAULT NULL COMMENT '收到回复的时间', + `response_delay_seconds` int NULL DEFAULT NULL COMMENT '回复延迟(秒)', + `is_successful` tinyint(1) NULL DEFAULT 0 COMMENT '是否成功互动', + `error_message` text CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci NULL COMMENT '失败原因/错误信息', + `retry_count` int NULL DEFAULT 0 COMMENT '重试次数', + `conversion_flag` tinyint(1) NULL DEFAULT 0 COMMENT '是否产生转化', + `site_dimension` varchar(255) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci NULL DEFAULT NULL COMMENT '网址维度标签', + `campaign_id` varchar(64) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci NULL DEFAULT NULL COMMENT '广告活动ID', + `operator` varchar(64) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci NULL DEFAULT NULL COMMENT '操作者', + `created_at` timestamp NULL DEFAULT CURRENT_TIMESTAMP COMMENT '记录创建时间', + `updated_at` timestamp NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT '记录更新时间', + `remark` varchar(512) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci NULL DEFAULT NULL COMMENT '备注信息', + PRIMARY KEY (`id`) USING BTREE, + INDEX `idx_site_id`(`site_id` ASC) USING BTREE, + INDEX `idx_click_id`(`click_id` ASC) USING BTREE, + INDEX `idx_task_id`(`task_id` ASC) USING BTREE, + INDEX `idx_interaction_time`(`interaction_time` ASC) USING BTREE, + INDEX `idx_interaction_status`(`interaction_status` ASC) USING BTREE, + INDEX `idx_composite`(`site_id` ASC, `interaction_time` ASC, `interaction_status` ASC) USING BTREE, + INDEX `idx_response_received`(`response_received` ASC) USING BTREE, + INDEX `idx_conversion`(`conversion_flag` ASC) USING BTREE +) ENGINE = InnoDB AUTO_INCREMENT = 2 CHARACTER SET = utf8mb4 COLLATE = utf8mb4_0900_ai_ci COMMENT = 'MIP页广告互动回复日志表' ROW_FORMAT = DYNAMIC; + +-- ---------------------------- +-- Records of ai_mip_interaction +-- ---------------------------- +INSERT INTO `ai_mip_interaction` VALUES (1, 1, 1, 'TASK20260112001', 'reply', '2026-01-12 20:25:09', 'success', '您好,请问有什么可以帮助您的吗?', NULL, NULL, NULL, NULL, 'auto', NULL, NULL, NULL, NULL, NULL, NULL, NULL, 0, NULL, NULL, NULL, 1, NULL, 0, 0, NULL, NULL, NULL, '2026-01-12 20:25:09', '2026-01-12 20:25:09', NULL); + +SET FOREIGN_KEY_CHECKS = 1; diff --git a/db/ai_mip_site.sql b/db/ai_mip_site.sql new file mode 100644 index 0000000..c632d3f --- /dev/null +++ b/db/ai_mip_site.sql @@ -0,0 +1,55 @@ +/* + Navicat Premium Dump SQL + + Source Server : mixue + Source Server Type : MySQL + Source Server Version : 90001 (9.0.1) + Source Host : localhost:3306 + Source Schema : ai_article + + Target Server Type : MySQL + Target Server Version : 90001 (9.0.1) + File Encoding : 65001 + + Date: 12/01/2026 20:31:23 +*/ + +SET NAMES utf8mb4; +SET FOREIGN_KEY_CHECKS = 0; + +-- ---------------------------- +-- Table structure for ai_mip_site +-- ---------------------------- +DROP TABLE IF EXISTS `ai_mip_site`; +CREATE TABLE `ai_mip_site` ( + `id` bigint NOT NULL AUTO_INCREMENT COMMENT '主键ID', + `site_url` varchar(512) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci NOT NULL COMMENT '网站URL,唯一', + `site_name` varchar(255) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci NULL DEFAULT NULL COMMENT '网站名称(可选)', + `status` enum('active','inactive','pending') CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci NULL DEFAULT 'active' COMMENT '状态:激活/停用/待审核', + `frequency` int NULL DEFAULT 1 COMMENT '频次(如每小时发几次)', + `time_start` time NULL DEFAULT '00:00:00' COMMENT '开始时间(HH:MM:SS)', + `time_end` time NULL DEFAULT '23:59:59' COMMENT '结束时间(HH:MM:SS)', + `interval_minutes` int NULL DEFAULT 60 COMMENT '执行间隔(分钟)', + `ad_feature` varchar(1024) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci NULL DEFAULT NULL COMMENT '广告特征描述(JSON格式,如:{\"color\":\"red\", \"position\":\"top\"})', + `click_count` bigint NULL DEFAULT 0 COMMENT '累计点击次数', + `reply_count` bigint NULL DEFAULT 0 COMMENT '累计回复次数', + `site_dimension` varchar(255) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci NULL DEFAULT NULL COMMENT '网址维度标签(如:教育、医疗等)', + `created_at` timestamp NULL DEFAULT CURRENT_TIMESTAMP COMMENT '创建时间', + `updated_at` timestamp NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT '更新时间', + `created_by` varchar(64) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci NULL DEFAULT NULL COMMENT '创建人', + `updated_by` varchar(64) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci NULL DEFAULT NULL COMMENT '更新人', + `remark` varchar(255) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci NULL DEFAULT NULL COMMENT '备注信息', + PRIMARY KEY (`id`) USING BTREE, + UNIQUE INDEX `site_url`(`site_url` ASC) USING BTREE, + UNIQUE INDEX `idx_site_url`(`site_url`(191) ASC) USING BTREE, + INDEX `idx_status`(`status` ASC) USING BTREE, + INDEX `idx_created_at`(`created_at` ASC) USING BTREE +) ENGINE = InnoDB AUTO_INCREMENT = 3 CHARACTER SET = utf8mb4 COLLATE = utf8mb4_0900_ai_ci COMMENT = 'MIP页广告网址管理表' ROW_FORMAT = Dynamic; + +-- ---------------------------- +-- Records of ai_mip_site +-- ---------------------------- +INSERT INTO `ai_mip_site` VALUES (1, 'https://example.com', '示例网站1', 'active', 1, '00:00:00', '23:59:59', 60, NULL, 0, 0, '教育', '2026-01-12 20:24:18', '2026-01-12 20:24:18', 'admin', NULL, NULL); +INSERT INTO `ai_mip_site` VALUES (2, 'https://test.com', '测试网站2', 'active', 1, '00:00:00', '23:59:59', 60, NULL, 0, 0, '医疗', '2026-01-12 20:24:18', '2026-01-12 20:24:18', 'admin', NULL, NULL); + +SET FOREIGN_KEY_CHECKS = 1; diff --git a/db/init_databases.py b/db/init_databases.py new file mode 100644 index 0000000..2b25574 --- /dev/null +++ b/db/init_databases.py @@ -0,0 +1,100 @@ +#!/usr/bin/env python3 +""" +SQLite数据库初始化脚本 +自动创建开发环境(ai_mip_dev.db)和生产环境(ai_mip_prod.db)数据库 +""" + +import sqlite3 +import os +from pathlib import Path + +# 数据库文件路径 +DB_DIR = Path(__file__).parent +DEV_DB = DB_DIR / "ai_mip_dev.db" +PROD_DB = DB_DIR / "ai_mip_prod.db" + +# SQL脚本路径 +INIT_SQL = DB_DIR / "init_sqlite.sql" +SEED_DEV_SQL = DB_DIR / "seed_dev.sql" + + +def execute_sql_file(conn, sql_file): + """执行SQL文件""" + with open(sql_file, 'r', encoding='utf-8') as f: + sql_script = f.read() + + # SQLite需要逐条执行语句 + conn.executescript(sql_script) + conn.commit() + print(f"✓ 已执行: {sql_file.name}") + + +def init_database(db_path, with_seed=False): + """初始化数据库""" + # 如果数据库已存在,询问是否覆盖 + if db_path.exists(): + response = input(f"\n数据库 {db_path.name} 已存在,是否覆盖? (y/n): ").strip().lower() + if response != 'y': + print(f"跳过 {db_path.name}") + return + os.remove(db_path) + print(f"已删除旧数据库: {db_path.name}") + + print(f"\n创建数据库: {db_path.name}") + + # 连接数据库(自动创建) + conn = sqlite3.connect(db_path) + + try: + # 执行初始化SQL + execute_sql_file(conn, INIT_SQL) + + # 如果需要,执行种子数据 + if with_seed: + execute_sql_file(conn, SEED_DEV_SQL) + + print(f"✓ 数据库 {db_path.name} 创建成功") + + # 验证表是否创建成功 + cursor = conn.cursor() + cursor.execute("SELECT name FROM sqlite_master WHERE type='table'") + tables = cursor.fetchall() + print(f" 创建的表: {', '.join([t[0] for t in tables])}") + + except Exception as e: + print(f"✗ 创建数据库失败: {str(e)}") + raise + finally: + conn.close() + + +def main(): + print("=" * 60) + print("SQLite数据库初始化工具") + print("=" * 60) + + # 检查SQL文件是否存在 + if not INIT_SQL.exists(): + print(f"错误: 找不到初始化脚本 {INIT_SQL}") + return + + # 初始化开发数据库(带测试数据) + print("\n[1] 初始化开发环境数据库") + init_database(DEV_DB, with_seed=True) + + # 初始化生产数据库(不带测试数据) + print("\n[2] 初始化生产环境数据库") + init_database(PROD_DB, with_seed=False) + + print("\n" + "=" * 60) + print("数据库初始化完成") + print("=" * 60) + print(f"开发数据库: {DEV_DB}") + print(f"生产数据库: {PROD_DB}") + print("\n使用方法:") + print(" 开发环境: 在 .env.development 中设置 DATABASE_PATH=db/ai_mip_dev.db") + print(" 生产环境: 在 .env.production 中设置 DATABASE_PATH=db/ai_mip_prod.db") + + +if __name__ == "__main__": + main() diff --git a/db/init_sqlite.sql b/db/init_sqlite.sql new file mode 100644 index 0000000..fe44978 --- /dev/null +++ b/db/init_sqlite.sql @@ -0,0 +1,125 @@ +-- SQLite数据库初始化脚本 +-- 适用于开发环境(ai_mip_dev.db)和生产环境(ai_mip_prod.db) + +-- ---------------------------- +-- Table structure for ai_mip_site +-- ---------------------------- +DROP TABLE IF EXISTS ai_mip_site; +CREATE TABLE ai_mip_site ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + site_url TEXT NOT NULL UNIQUE, + site_name TEXT, + status TEXT CHECK(status IN ('active', 'inactive', 'pending')) DEFAULT 'active', + frequency INTEGER DEFAULT 1, + time_start TEXT DEFAULT '00:00:00', + time_end TEXT DEFAULT '23:59:59', + interval_minutes INTEGER DEFAULT 60, + ad_feature TEXT, + click_count INTEGER DEFAULT 0, + reply_count INTEGER DEFAULT 0, + site_dimension TEXT, + created_at DATETIME DEFAULT CURRENT_TIMESTAMP, + updated_at DATETIME DEFAULT CURRENT_TIMESTAMP, + created_by TEXT, + updated_by TEXT, + remark TEXT +); + +-- 创建索引 +CREATE UNIQUE INDEX idx_site_url ON ai_mip_site(site_url); +CREATE INDEX idx_status ON ai_mip_site(status); +CREATE INDEX idx_created_at ON ai_mip_site(created_at); + +-- ---------------------------- +-- Table structure for ai_mip_click +-- ---------------------------- +DROP TABLE IF EXISTS ai_mip_click; +CREATE TABLE ai_mip_click ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + site_id INTEGER NOT NULL, + site_url TEXT NOT NULL, + click_time DATETIME NOT NULL, + user_ip TEXT, + user_agent TEXT, + referer_url TEXT, + device_type TEXT CHECK(device_type IN ('mobile', 'pc', 'tablet')), + click_count INTEGER DEFAULT 1, + is_valid INTEGER DEFAULT 1, + task_id TEXT, + operator TEXT, + created_at DATETIME DEFAULT CURRENT_TIMESTAMP, + FOREIGN KEY (site_id) REFERENCES ai_mip_site(id) +); + +-- 创建索引 +CREATE INDEX idx_site_id ON ai_mip_click(site_id); +CREATE INDEX idx_click_time ON ai_mip_click(click_time); +CREATE INDEX idx_site_url_click ON ai_mip_click(site_url); +CREATE INDEX idx_click_time_site ON ai_mip_click(click_time, site_id); +CREATE INDEX idx_task_id ON ai_mip_click(task_id); + +-- ---------------------------- +-- Table structure for ai_mip_interaction +-- ---------------------------- +DROP TABLE IF EXISTS ai_mip_interaction; +CREATE TABLE ai_mip_interaction ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + site_id INTEGER NOT NULL, + click_id INTEGER, + task_id TEXT, + interaction_type TEXT CHECK(interaction_type IN ('reply', 'comment', 'message', 'form_submit', 'follow', 'like', 'share')) NOT NULL, + interaction_time DATETIME NOT NULL, + interaction_status TEXT CHECK(interaction_status IN ('pending', 'success', 'failed', 'skipped')) DEFAULT 'pending', + reply_content TEXT, + reply_template_id TEXT, + ad_element_xpath TEXT, + ad_element_selector TEXT, + ad_text_content TEXT, + execution_mode TEXT CHECK(execution_mode IN ('auto', 'manual', 'semi_auto')) DEFAULT 'auto', + rpa_script TEXT, + browser_type TEXT CHECK(browser_type IN ('headless', 'headed', 'playwright', 'selenium')), + anti_detection_method TEXT, + proxy_ip TEXT, + user_agent TEXT, + custom_headers TEXT, + fingerprint_id TEXT, + response_received INTEGER DEFAULT 0, + response_content TEXT, + response_time DATETIME, + response_delay_seconds INTEGER, + is_successful INTEGER DEFAULT 0, + error_message TEXT, + retry_count INTEGER DEFAULT 0, + conversion_flag INTEGER DEFAULT 0, + site_dimension TEXT, + campaign_id TEXT, + operator TEXT, + created_at DATETIME DEFAULT CURRENT_TIMESTAMP, + updated_at DATETIME DEFAULT CURRENT_TIMESTAMP, + remark TEXT, + FOREIGN KEY (site_id) REFERENCES ai_mip_site(id), + FOREIGN KEY (click_id) REFERENCES ai_mip_click(id) +); + +-- 创建索引 +CREATE INDEX idx_site_id_interaction ON ai_mip_interaction(site_id); +CREATE INDEX idx_click_id_interaction ON ai_mip_interaction(click_id); +CREATE INDEX idx_task_id_interaction ON ai_mip_interaction(task_id); +CREATE INDEX idx_interaction_time ON ai_mip_interaction(interaction_time); +CREATE INDEX idx_interaction_status ON ai_mip_interaction(interaction_status); +CREATE INDEX idx_composite ON ai_mip_interaction(site_id, interaction_time, interaction_status); +CREATE INDEX idx_response_received ON ai_mip_interaction(response_received); +CREATE INDEX idx_conversion ON ai_mip_interaction(conversion_flag); + +-- 创建触发器:自动更新 updated_at +CREATE TRIGGER update_ai_mip_site_timestamp +AFTER UPDATE ON ai_mip_site +BEGIN + UPDATE ai_mip_site SET updated_at = CURRENT_TIMESTAMP WHERE id = NEW.id; +END; + +CREATE TRIGGER update_ai_mip_interaction_timestamp +AFTER UPDATE ON ai_mip_interaction +BEGIN + UPDATE ai_mip_interaction SET updated_at = CURRENT_TIMESTAMP WHERE id = NEW.id; +END; diff --git a/db/mip_table.txt b/db/mip_table.txt new file mode 100644 index 0000000..c6c1c35 --- /dev/null +++ b/db/mip_table.txt @@ -0,0 +1,145 @@ + +-- ---------------------------- +-- Table structure for ai_mip_click +-- ---------------------------- +DROP TABLE IF EXISTS `ai_mip_click`; +CREATE TABLE `ai_mip_click` ( + `id` bigint NOT NULL AUTO_INCREMENT COMMENT '主键ID', + `site_id` bigint NOT NULL COMMENT '关联站点ID(外键指向 ai_mip_site.id)', + `site_url` varchar(512) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci NOT NULL COMMENT '网站URL(冗余字段,便于查询优化)', + `click_time` datetime NOT NULL COMMENT '点击发生时间', + `user_ip` varchar(64) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci NULL DEFAULT NULL COMMENT '用户IP地址', + `user_agent` text CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci NULL COMMENT '浏览器/设备信息', + `referer_url` varchar(512) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci NULL DEFAULT NULL COMMENT '来源页面URL', + `device_type` enum('mobile','pc','tablet') CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci NULL DEFAULT NULL COMMENT '设备类型', + `click_count` int NULL DEFAULT 1 COMMENT '本次点击事件的计数(一般为1,可用于批量插入)', + `is_valid` tinyint(1) NULL DEFAULT 1 COMMENT '是否有效点击(防刷)', + `task_id` varchar(64) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci NULL DEFAULT NULL COMMENT 'RPA任务ID(可选)', + `operator` varchar(64) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci NULL DEFAULT NULL COMMENT '操作者(如自动系统)', + `created_at` timestamp NULL DEFAULT CURRENT_TIMESTAMP COMMENT '记录创建时间', + PRIMARY KEY (`id`) USING BTREE, + INDEX `idx_site_id`(`site_id` ASC) USING BTREE, + INDEX `idx_click_time`(`click_time` ASC) USING BTREE, + INDEX `idx_site_url`(`site_url` ASC) USING BTREE, + INDEX `idx_click_time_site`(`click_time` ASC, `site_id` ASC) USING BTREE, + INDEX `idx_task_id`(`task_id` ASC) USING BTREE +) ENGINE = InnoDB AUTO_INCREMENT = 2 CHARACTER SET = utf8mb4 COLLATE = utf8mb4_0900_ai_ci COMMENT = 'MIP页广告点击日志表' ROW_FORMAT = Dynamic; + +-- ---------------------------- +-- Table structure for ai_mip_interaction +-- ---------------------------- +DROP TABLE IF EXISTS `ai_mip_interaction`; +CREATE TABLE `ai_mip_interaction` ( + `id` bigint NOT NULL AUTO_INCREMENT COMMENT '主键ID', + `site_id` bigint NOT NULL COMMENT '关联站点ID', + `click_id` bigint NULL DEFAULT NULL COMMENT '关联点击记录ID', + `task_id` varchar(64) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci NULL DEFAULT NULL COMMENT 'RPA任务ID', + `interaction_type` enum('reply','comment','message','form_submit','follow','like','share') CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci NOT NULL COMMENT '互动类型', + `interaction_time` datetime NOT NULL COMMENT '互动发生时间', + `interaction_status` enum('pending','success','failed','skipped') CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci NULL DEFAULT 'pending' COMMENT '互动状态', + `reply_content` text CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci NULL COMMENT '回复/评论的内容', + `reply_template_id` varchar(64) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci NULL DEFAULT NULL COMMENT '使用的回复模板ID', + `ad_element_xpath` varchar(512) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci NULL DEFAULT NULL COMMENT '广告元素的XPath定位', + `ad_element_selector` varchar(512) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci NULL DEFAULT NULL COMMENT '广告元素的CSS选择器', + `ad_text_content` text CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci NULL COMMENT '广告的文本内容', + `execution_mode` enum('auto','manual','semi_auto') CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci NULL DEFAULT 'auto' COMMENT '执行方式', + `rpa_script` varchar(255) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci NULL DEFAULT NULL COMMENT '使用的RPA脚本名称', + `browser_type` enum('headless','headed','playwright','selenium') CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci NULL DEFAULT NULL COMMENT '浏览器类型', + `anti_detection_method` json NULL COMMENT '万金油技术方案', + `proxy_ip` varchar(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci NULL DEFAULT NULL COMMENT '使用的代理IP', + `user_agent` text CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci NULL COMMENT '使用的User-Agent', + `custom_headers` json NULL COMMENT '自定义HTTP头', + `fingerprint_id` varchar(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci NULL DEFAULT NULL COMMENT '浏览器指纹ID', + `response_received` tinyint(1) NULL DEFAULT 0 COMMENT '是否收到回复', + `response_content` text CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci NULL COMMENT '对方回复的内容', + `response_time` datetime NULL DEFAULT NULL COMMENT '收到回复的时间', + `response_delay_seconds` int NULL DEFAULT NULL COMMENT '回复延迟(秒)', + `is_successful` tinyint(1) NULL DEFAULT 0 COMMENT '是否成功互动', + `error_message` text CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci NULL COMMENT '失败原因/错误信息', + `retry_count` int NULL DEFAULT 0 COMMENT '重试次数', + `conversion_flag` tinyint(1) NULL DEFAULT 0 COMMENT '是否产生转化', + `site_dimension` varchar(255) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci NULL DEFAULT NULL COMMENT '网址维度标签', + `campaign_id` varchar(64) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci NULL DEFAULT NULL COMMENT '广告活动ID', + `operator` varchar(64) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci NULL DEFAULT NULL COMMENT '操作者', + `created_at` timestamp NULL DEFAULT CURRENT_TIMESTAMP COMMENT '记录创建时间', + `updated_at` timestamp NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT '记录更新时间', + `remark` varchar(512) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci NULL DEFAULT NULL COMMENT '备注信息', + PRIMARY KEY (`id`) USING BTREE, + INDEX `idx_site_id`(`site_id` ASC) USING BTREE, + INDEX `idx_click_id`(`click_id` ASC) USING BTREE, + INDEX `idx_task_id`(`task_id` ASC) USING BTREE, + INDEX `idx_interaction_time`(`interaction_time` ASC) USING BTREE, + INDEX `idx_interaction_status`(`interaction_status` ASC) USING BTREE, + INDEX `idx_composite`(`site_id` ASC, `interaction_time` ASC, `interaction_status` ASC) USING BTREE, + INDEX `idx_response_received`(`response_received` ASC) USING BTREE, + INDEX `idx_conversion`(`conversion_flag` ASC) USING BTREE +) ENGINE = InnoDB AUTO_INCREMENT = 2 CHARACTER SET = utf8mb4 COLLATE = utf8mb4_0900_ai_ci COMMENT = 'MIP页广告互动回复日志表' ROW_FORMAT = DYNAMIC; + +-- ---------------------------- +-- Table structure for ai_mip_site +-- ---------------------------- +DROP TABLE IF EXISTS `ai_mip_site`; +CREATE TABLE `ai_mip_site` ( + `id` bigint NOT NULL AUTO_INCREMENT COMMENT '主键ID', + `site_url` varchar(512) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci NOT NULL COMMENT '网站URL,唯一', + `site_name` varchar(255) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci NULL DEFAULT NULL COMMENT '网站名称(可选)', + `status` enum('active','inactive','pending') CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci NULL DEFAULT 'active' COMMENT '状态:激活/停用/待审核', + `frequency` int NULL DEFAULT 1 COMMENT '频次(如每小时发几次)', + `time_start` time NULL DEFAULT '00:00:00' COMMENT '开始时间(HH:MM:SS)', + `time_end` time NULL DEFAULT '23:59:59' COMMENT '结束时间(HH:MM:SS)', + `interval_minutes` int NULL DEFAULT 60 COMMENT '执行间隔(分钟)', + `ad_feature` varchar(1024) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci NULL DEFAULT NULL COMMENT '广告特征描述(JSON格式,如:{\"color\":\"red\", \"position\":\"top\"})', + `click_count` bigint NULL DEFAULT 0 COMMENT '累计点击次数', + `reply_count` bigint NULL DEFAULT 0 COMMENT '累计回复次数', + `site_dimension` varchar(255) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci NULL DEFAULT NULL COMMENT '网址维度标签(如:教育、医疗等)', + `created_at` timestamp NULL DEFAULT CURRENT_TIMESTAMP COMMENT '创建时间', + `updated_at` timestamp NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT '更新时间', + `created_by` varchar(64) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci NULL DEFAULT NULL COMMENT '创建人', + `updated_by` varchar(64) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci NULL DEFAULT NULL COMMENT '更新人', + `remark` varchar(255) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci NULL DEFAULT NULL COMMENT '备注信息', + PRIMARY KEY (`id`) USING BTREE, + UNIQUE INDEX `site_url`(`site_url` ASC) USING BTREE, + UNIQUE INDEX `idx_site_url`(`site_url`(191) ASC) USING BTREE, + INDEX `idx_status`(`status` ASC) USING BTREE, + INDEX `idx_created_at`(`created_at` ASC) USING BTREE +) ENGINE = InnoDB AUTO_INCREMENT = 3 CHARACTER SET = utf8mb4 COLLATE = utf8mb4_0900_ai_ci COMMENT = 'MIP页广告网址管理表' ROW_FORMAT = Dynamic; + +-- ---------------------------- +-- Table structure for ai_mip_task_log +-- ---------------------------- +DROP TABLE IF EXISTS `ai_mip_task_log`; +CREATE TABLE `ai_mip_task_log` ( + `id` bigint NOT NULL AUTO_INCREMENT COMMENT '主键ID', + `task_id` varchar(64) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci NOT NULL COMMENT 'RPA任务唯一ID', + `site_id` bigint NOT NULL COMMENT '关联站点ID', + `step_1_visit_time` datetime NULL DEFAULT NULL COMMENT '步骤1:访问网址时间', + `step_1_status` enum('success','failed','skipped') CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci NULL DEFAULT NULL COMMENT '步骤1状态', + `step_2_antibot_time` datetime NULL DEFAULT NULL COMMENT '步骤2:万金油技术方案执行时间', + `step_2_status` enum('success','failed','skipped') CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci NULL DEFAULT NULL COMMENT '步骤2状态', + `step_3_ad_detection_time` datetime NULL DEFAULT NULL COMMENT '步骤3:广告检测时间', + `step_3_has_ad` tinyint(1) NULL DEFAULT NULL COMMENT '是否检测到广告', + `step_3_ad_count` int NULL DEFAULT 0 COMMENT '检测到的广告数量', + `step_4_click_time` datetime NULL DEFAULT NULL COMMENT '步骤4:点击广告时间', + `step_4_status` enum('success','failed','skipped') CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci NULL DEFAULT NULL COMMENT '步骤4状态', + `step_5_reply_time` datetime NULL DEFAULT NULL COMMENT '步骤5:获取回复时间', + `step_5_status` enum('success','failed','skipped') CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci NULL DEFAULT NULL COMMENT '步骤5状态', + `task_start_time` datetime NOT NULL COMMENT '任务开始时间', + `task_end_time` datetime NULL DEFAULT NULL COMMENT '任务结束时间', + `task_duration_seconds` int NULL DEFAULT NULL COMMENT '任务执行时长(秒)', + `task_status` enum('running','completed','failed','timeout') CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci NULL DEFAULT 'running' COMMENT '任务整体状态', + `total_clicks` int NULL DEFAULT 0 COMMENT '本次任务总点击次数', + `total_interactions` int NULL DEFAULT 0 COMMENT '本次任务总互动次数', + `successful_interactions` int NULL DEFAULT 0 COMMENT '成功互动次数', + `failed_interactions` int NULL DEFAULT 0 COMMENT '失败互动次数', + `execution_mode` enum('auto','manual','scheduled') CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci NULL DEFAULT 'auto' COMMENT '执行模式', + `triggered_by` varchar(64) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci NULL DEFAULT NULL COMMENT '触发者(定时任务/手动触发/队列)', + `error_log` text CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci NULL COMMENT '错误日志', + `created_at` timestamp NULL DEFAULT CURRENT_TIMESTAMP COMMENT '记录创建时间', + `updated_at` timestamp NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT '记录更新时间', + PRIMARY KEY (`id`) USING BTREE, + UNIQUE INDEX `task_id`(`task_id` ASC) USING BTREE, + UNIQUE INDEX `uk_task_id`(`task_id` ASC) USING BTREE, + INDEX `idx_site_id`(`site_id` ASC) USING BTREE, + INDEX `idx_task_status`(`task_status` ASC) USING BTREE, + INDEX `idx_start_time`(`task_start_time` ASC) USING BTREE +) ENGINE = InnoDB AUTO_INCREMENT = 2 CHARACTER SET = utf8mb4 COLLATE = utf8mb4_0900_ai_ci COMMENT = 'RPA任务执行日志表' ROW_FORMAT = DYNAMIC; diff --git a/db/seed_dev.sql b/db/seed_dev.sql new file mode 100644 index 0000000..d4233c2 --- /dev/null +++ b/db/seed_dev.sql @@ -0,0 +1,20 @@ +-- 开发环境测试数据 +-- 用于 ai_mip_dev.db + +-- 插入测试站点 +INSERT INTO ai_mip_site (site_url, site_name, status, frequency, time_start, time_end, interval_minutes, click_count, reply_count, site_dimension, created_by) VALUES +('https://health.baidu.com/m/detail/ar_2366617956693492811', '百度健康测试页面', 'active', 3, '09:00:00', '21:00:00', 45, 0, 0, '医疗健康', 'admin'), +('https://example.com/test', '测试网站1', 'active', 2, '10:00:00', '20:00:00', 60, 0, 0, '教育', 'admin'), +('https://demo.com/page', '演示网站', 'inactive', 1, '00:00:00', '23:59:59', 120, 0, 0, '商业', 'admin'); + +-- 插入测试点击记录 +INSERT INTO ai_mip_click (site_id, site_url, click_time, user_ip, device_type, task_id, operator) VALUES +(1, 'https://health.baidu.com/m/detail/ar_2366617956693492811', datetime('now'), '192.168.1.100', 'pc', 'TASK_DEV_001', 'RPA_SYSTEM'), +(1, 'https://health.baidu.com/m/detail/ar_2366617956693492811', datetime('now', '-1 hour'), '192.168.1.101', 'mobile', 'TASK_DEV_002', 'RPA_SYSTEM'), +(2, 'https://example.com/test', datetime('now', '-2 hours'), '192.168.1.102', 'pc', 'TASK_DEV_003', 'RPA_SYSTEM'); + +-- 插入测试互动记录 +INSERT INTO ai_mip_interaction (site_id, click_id, task_id, interaction_type, interaction_time, interaction_status, reply_content, execution_mode, browser_type, is_successful, operator) VALUES +(1, 1, 'TASK_DEV_001', 'reply', datetime('now'), 'success', '测试回复内容', 'auto', 'playwright', 1, 'RPA_SYSTEM'), +(1, 2, 'TASK_DEV_002', 'comment', datetime('now', '-1 hour'), 'success', '测试评论内容', 'auto', 'playwright', 1, 'RPA_SYSTEM'), +(2, 3, 'TASK_DEV_003', 'reply', datetime('now', '-2 hours'), 'pending', NULL, 'auto', 'playwright', 0, 'RPA_SYSTEM'); diff --git a/db_manager.py b/db_manager.py new file mode 100644 index 0000000..06d2c1c --- /dev/null +++ b/db_manager.py @@ -0,0 +1,547 @@ +""" +数据库管理器 +使用 MySQL 数据库 +""" + +import json +import random +from datetime import datetime +from typing import List, Dict, Optional, Union +from pathlib import Path +from loguru import logger +from config import Config + +try: + import pymysql + MYSQL_AVAILABLE = True +except ImportError: + MYSQL_AVAILABLE = False + logger.error("pymysql 未安装,请安装: pip install pymysql") + raise ImportError("使用 MySQL 需要安装 pymysql: pip install pymysql") + + +class DatabaseManager: + """数据库管理器,使用 MySQL""" + + def __init__(self, db_path: str = None): + """ + 初始化数据库连接 + + Args: + db_path: 忽略,仅为了兼容性 + """ + if not MYSQL_AVAILABLE: + raise ImportError("使用 MySQL 需要安装 pymysql: pip install pymysql") + + self.db_config = { + 'host': Config.MYSQL_HOST, + 'port': Config.MYSQL_PORT, + 'user': Config.MYSQL_USER, + 'password': Config.MYSQL_PASSWORD, + 'database': Config.MYSQL_DATABASE, + 'charset': 'utf8mb4' + } + logger.info(f"MySQL数据库初始化: {Config.MYSQL_HOST}:{Config.MYSQL_PORT}/{Config.MYSQL_DATABASE}") + + def get_connection(self) -> 'pymysql.Connection': + """获取MySQL数据库连接""" + conn = pymysql.connect(**self.db_config) + return conn + + def _dict_from_row(self, row) -> Dict: + """将数据库行转换为字典""" + if row is None: + return None + return dict(row) if isinstance(row, dict) else row + + def _get_placeholder(self) -> str: + """获取SQL占位符,MySQL使用 %s""" + return '%s' + + def _execute_query(self, conn, sql: str, params: tuple = None): + """执行SQL查询,使用DictCursor""" + cursor = conn.cursor(pymysql.cursors.DictCursor) + + if params: + cursor.execute(sql, params) + else: + cursor.execute(sql) + + return cursor + + +class SiteManager(DatabaseManager): + """站点管理""" + + def add_site(self, site_url: str, site_name: str = None, + site_dimension: str = None, frequency: int = None, + time_start: str = None, time_end: str = None, + interval_minutes: int = None) -> Optional[int]: + """ + 添加新站点 + + Args: + site_url: 网站URL + site_name: 网站名称 + site_dimension: 网站维度标签 + frequency: 频次 + time_start: 开始时间 + time_end: 结束时间 + interval_minutes: 执行间隔(分钟) + + Returns: + 站点ID,失败返回None + """ + try: + conn = self.get_connection() + ph = self._get_placeholder() + + cursor = conn.cursor(pymysql.cursors.DictCursor) + + # 生成随机目标点击次数(兼容原有逻辑) + target_clicks = random.randint( + getattr(Config, 'MIN_CLICK_COUNT', 1), + getattr(Config, 'MAX_CLICK_COUNT', 10) + ) + + sql = f""" + INSERT INTO ai_mip_site ( + site_url, site_name, status, frequency, + time_start, time_end, interval_minutes, + site_dimension, created_by + ) VALUES ({ph}, {ph}, {ph}, {ph}, {ph}, {ph}, {ph}, {ph}, {ph}) + """ + + cursor.execute(sql, ( + site_url, + site_name or site_url, + 'active', + frequency or 1, + time_start or '09:00:00', + time_end or '21:00:00', + interval_minutes or 60, + site_dimension, + 'system' + )) + + site_id = cursor.lastrowid + conn.commit() + conn.close() + + logger.info(f"成功添加站点: {site_url} (ID: {site_id})") + return site_id + + except pymysql.IntegrityError: + logger.warning(f"站点URL已存在: {site_url}") + return None + except Exception as e: + logger.error(f"添加站点失败: {str(e)}") + return None + + def get_site_by_url(self, site_url: str) -> Optional[Dict]: + """根据URL获取站点信息""" + try: + conn = self.get_connection() + ph = self._get_placeholder() + cursor = self._execute_query(conn, f"SELECT * FROM ai_mip_site WHERE site_url = {ph}", (site_url,)) + row = cursor.fetchone() + conn.close() + return self._dict_from_row(row) if row else None + except Exception as e: + logger.error(f"查询站点失败: {str(e)}") + return None + + def get_site_by_id(self, site_id: int) -> Optional[Dict]: + """根据ID获取站点信息""" + try: + conn = self.get_connection() + ph = self._get_placeholder() + cursor = self._execute_query(conn, f"SELECT * FROM ai_mip_site WHERE id = {ph}", (site_id,)) + row = cursor.fetchone() + conn.close() + return self._dict_from_row(row) if row else None + except Exception as e: + logger.error(f"查询站点失败: {str(e)}") + return None + + def get_active_sites(self) -> List[Dict]: + """获取所有活跃站点""" + try: + conn = self.get_connection() + cursor = self._execute_query(conn, "SELECT * FROM ai_mip_site WHERE status = 'active' ORDER BY created_at DESC") + rows = cursor.fetchall() + conn.close() + return [self._dict_from_row(row) for row in rows] + except Exception as e: + logger.error(f"查询活跃站点失败: {str(e)}") + return [] + + def get_all_sites(self) -> List[Dict]: + """获取所有站点""" + try: + conn = self.get_connection() + cursor = self._execute_query(conn, "SELECT * FROM ai_mip_site ORDER BY created_at DESC") + rows = cursor.fetchall() + conn.close() + return [self._dict_from_row(row) for row in rows] + except Exception as e: + logger.error(f"查询所有站点失败: {str(e)}") + return [] + + def update_site_status(self, site_id: int, status: str) -> bool: + """更新站点状态""" + try: + conn = self.get_connection() + ph = self._get_placeholder() + cursor = self._execute_query( + conn, + f"UPDATE ai_mip_site SET status = {ph}, updated_by = {ph} WHERE id = {ph}", + (status, 'system', site_id) + ) + conn.commit() + conn.close() + logger.info(f"更新站点状态: ID={site_id}, status={status}") + return True + except Exception as e: + logger.error(f"更新站点状态失败: {str(e)}") + return False + + def increment_click_count(self, site_id: int, count: int = 1) -> bool: + """增加点击次数""" + try: + conn = self.get_connection() + ph = self._get_placeholder() + cursor = self._execute_query( + conn, + f"UPDATE ai_mip_site SET click_count = click_count + {ph} WHERE id = {ph}", + (count, site_id) + ) + conn.commit() + conn.close() + return True + except Exception as e: + logger.error(f"更新点击次数失败: {str(e)}") + return False + + def increment_reply_count(self, site_id: int, count: int = 1) -> bool: + """增加回复次数""" + try: + conn = self.get_connection() + ph = self._get_placeholder() + cursor = self._execute_query( + conn, + f"UPDATE ai_mip_site SET reply_count = reply_count + {ph} WHERE id = {ph}", + (count, site_id) + ) + conn.commit() + conn.close() + return True + except Exception as e: + logger.error(f"更新回复次数失败: {str(e)}") + return False + + def delete_site(self, site_id: int) -> bool: + """删除站点""" + try: + conn = self.get_connection() + ph = self._get_placeholder() + cursor = self._execute_query(conn, f"DELETE FROM ai_mip_site WHERE id = {ph}", (site_id,)) + conn.commit() + conn.close() + logger.info(f"已删除站点: ID={site_id}") + return True + except Exception as e: + logger.error(f"删除站点失败: {str(e)}") + return False + + +class ClickManager(DatabaseManager): + """点击记录管理""" + + def record_click(self, site_id: int, site_url: str, + user_ip: str = None, device_type: str = 'pc', + task_id: str = None) -> Optional[int]: + """ + 记录一次点击 + + Args: + site_id: 站点ID + site_url: 站点URL + user_ip: 用户IP(代理IP) + device_type: 设备类型 + task_id: 任务ID + + Returns: + 点击记录ID + """ + try: + conn = self.get_connection() + ph = self._get_placeholder() + + cursor = conn.cursor(pymysql.cursors.DictCursor) + + sql = f""" + INSERT INTO ai_mip_click ( + site_id, site_url, click_time, user_ip, + device_type, task_id, operator + ) VALUES ({ph}, {ph}, {ph}, {ph}, {ph}, {ph}, {ph}) + """ + + cursor.execute(sql, ( + site_id, + site_url, + datetime.now().strftime('%Y-%m-%d %H:%M:%S'), + user_ip, + device_type, + task_id or f'TASK_{datetime.now().strftime("%Y%m%d%H%M%S")}', + 'RPA_SYSTEM' + )) + + click_id = cursor.lastrowid + conn.commit() + conn.close() + + # 更新站点点击次数 + site_mgr = SiteManager() + site_mgr.increment_click_count(site_id) + + logger.info(f"记录点击: site_id={site_id}, click_id={click_id}") + return click_id + + except Exception as e: + logger.error(f"记录点击失败: {str(e)}") + return None + + def get_clicks_by_site(self, site_id: int, limit: int = 100) -> List[Dict]: + """获取站点的点击记录""" + try: + conn = self.get_connection() + cursor = conn.cursor() + cursor.execute(""" + SELECT * FROM ai_mip_click + WHERE site_id = ? + ORDER BY click_time DESC + LIMIT ? + """, (site_id, limit)) + rows = cursor.fetchall() + conn.close() + return [self._dict_from_row(row) for row in rows] + except Exception as e: + logger.error(f"查询点击记录失败: {str(e)}") + return [] + + def get_click_count_by_site(self, site_id: int) -> int: + """获取站点的总点击次数""" + try: + conn = self.get_connection() + cursor = conn.cursor() + cursor.execute("SELECT COUNT(*) as count FROM ai_mip_click WHERE site_id = ?", (site_id,)) + row = cursor.fetchone() + conn.close() + return row['count'] if row else 0 + except Exception as e: + logger.error(f"查询点击次数失败: {str(e)}") + return 0 + + +class InteractionManager(DatabaseManager): + """互动记录管理""" + + def record_interaction(self, site_id: int, click_id: int = None, + task_id: str = None, interaction_type: str = 'reply', + reply_content: str = None, is_successful: bool = False, + response_received: bool = False, response_content: str = None, + proxy_ip: str = None, fingerprint_id: str = None, + error_message: str = None) -> Optional[int]: + """ + 记录一次互动 + + Args: + site_id: 站点ID + click_id: 关联的点击记录ID + task_id: 任务ID + interaction_type: 互动类型(reply/comment等) + reply_content: 回复内容 + is_successful: 是否成功 + response_received: 是否收到回复 + response_content: 对方回复内容 + proxy_ip: 使用的代理IP + fingerprint_id: 浏览器指纹ID + error_message: 错误信息 + + Returns: + 互动记录ID + """ + try: + conn = self.get_connection() + ph = self._get_placeholder() + + cursor = conn.cursor(pymysql.cursors.DictCursor) + + now = datetime.now().strftime('%Y-%m-%d %H:%M:%S') + + sql = f""" + INSERT INTO ai_mip_interaction ( + site_id, click_id, task_id, interaction_type, + interaction_time, interaction_status, reply_content, + execution_mode, browser_type, proxy_ip, fingerprint_id, + response_received, response_content, is_successful, + error_message, operator + ) VALUES ({ph}, {ph}, {ph}, {ph}, {ph}, {ph}, {ph}, {ph}, {ph}, {ph}, {ph}, {ph}, {ph}, {ph}, {ph}, {ph}) + """ + + cursor.execute(sql, ( + site_id, + click_id, + task_id or f'TASK_{datetime.now().strftime("%Y%m%d%H%M%S")}', + interaction_type, + now, + 'success' if is_successful else 'failed', + reply_content, + 'auto', + 'playwright', + proxy_ip, + fingerprint_id, + 1 if response_received else 0, + response_content, + 1 if is_successful else 0, + error_message, + 'RPA_SYSTEM' + )) + + interaction_id = cursor.lastrowid + conn.commit() + conn.close() + + # 如果收到回复,更新站点回复次数 + if response_received: + site_mgr = SiteManager() + site_mgr.increment_reply_count(site_id) + + logger.info(f"记录互动: site_id={site_id}, interaction_id={interaction_id}, success={is_successful}") + return interaction_id + + except Exception as e: + logger.error(f"记录互动失败: {str(e)}") + return None + + def get_interactions_by_site(self, site_id: int, limit: int = 100) -> List[Dict]: + """获取站点的互动记录""" + try: + conn = self.get_connection() + cursor = conn.cursor() + cursor.execute(""" + SELECT * FROM ai_mip_interaction + WHERE site_id = ? + ORDER BY interaction_time DESC + LIMIT ? + """, (site_id, limit)) + rows = cursor.fetchall() + conn.close() + return [self._dict_from_row(row) for row in rows] + except Exception as e: + logger.error(f"查询互动记录失败: {str(e)}") + return [] + + def get_successful_interactions_count(self, site_id: int) -> int: + """获取站点的成功互动次数""" + try: + conn = self.get_connection() + cursor = conn.cursor() + cursor.execute(""" + SELECT COUNT(*) as count FROM ai_mip_interaction + WHERE site_id = ? AND is_successful = 1 + """, (site_id,)) + row = cursor.fetchone() + conn.close() + return row['count'] if row else 0 + except Exception as e: + logger.error(f"查询成功互动次数失败: {str(e)}") + return 0 + + +class StatisticsManager(DatabaseManager): + """统计数据管理""" + + def get_statistics(self) -> Dict: + """ + 获取全局统计数据 + + Returns: + 统计数据字典 + """ + try: + conn = self.get_connection() + + # 站点统计 + cursor = self._execute_query(conn, "SELECT COUNT(*) as total FROM ai_mip_site") + total_sites = cursor.fetchone()['total'] + + cursor = self._execute_query(conn, "SELECT COUNT(*) as total FROM ai_mip_site WHERE status = 'active'") + active_sites = cursor.fetchone()['total'] + + # 点击统计 + cursor = self._execute_query(conn, "SELECT COUNT(*) as total FROM ai_mip_click") + total_clicks = cursor.fetchone()['total'] + + # 互动统计 + cursor = self._execute_query(conn, "SELECT COUNT(*) as total FROM ai_mip_interaction WHERE response_received = 1") + total_replies = cursor.fetchone()['total'] + + cursor = self._execute_query(conn, "SELECT COUNT(*) as total FROM ai_mip_interaction WHERE is_successful = 1") + successful_interactions = cursor.fetchone()['total'] + + conn.close() + + reply_rate = (total_replies / total_clicks * 100) if total_clicks > 0 else 0 + success_rate = (successful_interactions / total_clicks * 100) if total_clicks > 0 else 0 + + return { + 'total_sites': total_sites, + 'active_sites': active_sites, + 'total_clicks': total_clicks, + 'total_replies': total_replies, + 'successful_interactions': successful_interactions, + 'reply_rate': f"{reply_rate:.2f}%", + 'success_rate': f"{success_rate:.2f}%" + } + + except Exception as e: + logger.error(f"获取统计数据失败: {str(e)}") + return {} + + def get_site_statistics(self, site_id: int) -> Dict: + """ + 获取单个站点的统计数据 + + Args: + site_id: 站点ID + + Returns: + 站点统计数据 + """ + try: + site_mgr = SiteManager(self.db_path) + click_mgr = ClickManager(self.db_path) + interaction_mgr = InteractionManager(self.db_path) + + site = site_mgr.get_site_by_id(site_id) + if not site: + return {} + + click_count = click_mgr.get_click_count_by_site(site_id) + success_count = interaction_mgr.get_successful_interactions_count(site_id) + + return { + 'site_url': site['site_url'], + 'site_name': site['site_name'], + 'status': site['status'], + 'click_count': click_count, + 'reply_count': site['reply_count'], + 'successful_interactions': success_count, + 'reply_rate': f"{(site['reply_count'] / click_count * 100) if click_count > 0 else 0:.2f}%" + } + + except Exception as e: + logger.error(f"获取站点统计失败: {str(e)}") + return {} diff --git a/debug_no_input.png b/debug_no_input.png new file mode 100644 index 0000000..645743f Binary files /dev/null and b/debug_no_input.png differ diff --git a/open_database_manager.bat b/open_database_manager.bat new file mode 100644 index 0000000..bf886fd --- /dev/null +++ b/open_database_manager.bat @@ -0,0 +1,22 @@ +@echo off +chcp 65001 >nul +echo ==================================== +echo MIP广告自动化 - 数据库管理页面 +echo ==================================== +echo. + +echo 正在启动浏览器打开管理页面... +echo. + +REM 检测操作系统并打开浏览器 +start http://localhost:5000/static/database.html + +echo. +echo ✅ 已在浏览器中打开数据库管理页面 +echo. +echo 📌 如果浏览器未自动打开,请手动访问: +echo http://localhost:5000/static/database.html +echo. +echo 💡 提示: 请确保后端服务已启动 (python app.py) +echo. +pause diff --git a/requirements.txt b/requirements.txt index 1778041..a7f1580 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,9 +1,31 @@ -playwright>=1.35.0 -requests>=2.28.0 -urllib3<2.0 -flask>=2.0.0,<2.3.0 -flask-cors>=3.0.0 -apscheduler>=3.9.0 -python-dotenv>=0.19.0 -pyyaml>=5.4.0 -loguru>=0.6.0 +# MIP广告自动点击系统 - Python依赖包 + +# Web框架 +Flask==3.0.0 +Werkzeug==3.0.1 + +# 浏览器自动化 +playwright==1.40.0 + +# HTTP请求 +requests==2.31.0 + +# 日志处理 +loguru==0.7.2 + +# 任务调度 +APScheduler==3.10.4 + +# 环境变量管理 +python-dotenv==1.0.0 + +# 时区处理 +pytz==2023.3 +tzlocal==5.2 + +# 数据处理 +python-dateutil==2.8.2 + +# 数据库 +pymysql==1.1.0 +cryptography>=41.0.0 diff --git a/screenshots/task_1_20260116_150519/01_before_click.png b/screenshots/task_1_20260116_150519/01_before_click.png new file mode 100644 index 0000000..f004aa6 Binary files /dev/null and b/screenshots/task_1_20260116_150519/01_before_click.png differ diff --git a/screenshots/task_1_20260116_150519/02_after_click.png b/screenshots/task_1_20260116_150519/02_after_click.png new file mode 100644 index 0000000..d391d31 Binary files /dev/null and b/screenshots/task_1_20260116_150519/02_after_click.png differ diff --git a/screenshots/task_1_20260116_150732/01_before_click.png b/screenshots/task_1_20260116_150732/01_before_click.png new file mode 100644 index 0000000..153abf2 Binary files /dev/null and b/screenshots/task_1_20260116_150732/01_before_click.png differ diff --git a/screenshots/task_1_20260116_153405/01_before_click.png b/screenshots/task_1_20260116_153405/01_before_click.png new file mode 100644 index 0000000..ac5417b Binary files /dev/null and b/screenshots/task_1_20260116_153405/01_before_click.png differ diff --git a/screenshots/task_1_20260116_153405/02_after_click.png b/screenshots/task_1_20260116_153405/02_after_click.png new file mode 100644 index 0000000..02f709a Binary files /dev/null and b/screenshots/task_1_20260116_153405/02_after_click.png differ diff --git a/screenshots/task_1_20260116_155350/01_before_click.png b/screenshots/task_1_20260116_155350/01_before_click.png new file mode 100644 index 0000000..153abf2 Binary files /dev/null and b/screenshots/task_1_20260116_155350/01_before_click.png differ diff --git a/screenshots/task_1_20260116_155637/01_before_click.png b/screenshots/task_1_20260116_155637/01_before_click.png new file mode 100644 index 0000000..85c7bd0 Binary files /dev/null and b/screenshots/task_1_20260116_155637/01_before_click.png differ diff --git a/screenshots/task_1_20260116_155637/02_after_click.png b/screenshots/task_1_20260116_155637/02_after_click.png new file mode 100644 index 0000000..1e75dc2 Binary files /dev/null and b/screenshots/task_1_20260116_155637/02_after_click.png differ diff --git a/start_production.sh b/start_production.sh new file mode 100644 index 0000000..5e8bde2 --- /dev/null +++ b/start_production.sh @@ -0,0 +1,259 @@ +#!/bin/bash + +# MIP广告自动点击系统 - 生产环境启动脚本 +# 适用于 Ubuntu/Debian 系统 + +set -e # 遇到错误立即退出 + +echo "============================================================" +echo "MIP广告自动点击系统 - 生产环境启动" +echo "============================================================" + +# 颜色定义 +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +NC='\033[0m' # No Color + +# 项目目录(脚本所在目录) +PROJECT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +cd "$PROJECT_DIR" + +echo -e "${GREEN}项目目录: $PROJECT_DIR${NC}" + +# 检查Python版本 +echo "" +echo "============================================================" +echo "检查Python环境" +echo "============================================================" + +if ! command -v python3 &> /dev/null; then + echo -e "${RED}错误: 未找到 python3${NC}" + echo "请安装Python 3.8+: sudo apt-get install python3 python3-pip python3-venv" + exit 1 +fi + +PYTHON_VERSION=$(python3 --version | awk '{print $2}') +echo -e "${GREEN}Python版本: $PYTHON_VERSION${NC}" + +# 检查Python版本是否 >= 3.8 +PYTHON_MAJOR=$(echo $PYTHON_VERSION | cut -d. -f1) +PYTHON_MINOR=$(echo $PYTHON_VERSION | cut -d. -f2) + +if [ "$PYTHON_MAJOR" -lt 3 ] || ([ "$PYTHON_MAJOR" -eq 3 ] && [ "$PYTHON_MINOR" -lt 8 ]); then + echo -e "${RED}错误: Python版本过低,需要 3.8+${NC}" + echo "当前版本: $PYTHON_VERSION" + exit 1 +fi + +# 虚拟环境目录 +VENV_DIR="$PROJECT_DIR/venv" + +# 创建虚拟环境(如果不存在) +if [ ! -d "$VENV_DIR" ]; then + echo "" + echo "============================================================" + echo "创建Python虚拟环境" + echo "============================================================" + + # 检查是否安装了 venv 模块 + if ! python3 -m venv --help &> /dev/null; then + echo -e "${YELLOW}警告: python3-venv 未安装,正在尝试安装...${NC}" + + # 尝试安装(需要sudo权限) + if command -v apt-get &> /dev/null; then + sudo apt-get update + sudo apt-get install -y python3-venv + else + echo -e "${RED}错误: 无法自动安装 python3-venv${NC}" + echo "请手动执行: sudo apt-get install python3-venv" + exit 1 + fi + fi + + echo -e "${GREEN}正在创建虚拟环境...${NC}" + python3 -m venv "$VENV_DIR" + echo -e "${GREEN}✓ 虚拟环境创建成功${NC}" +else + echo "" + echo -e "${GREEN}✓ 虚拟环境已存在: $VENV_DIR${NC}" +fi + +# 激活虚拟环境 +echo "" +echo "============================================================" +echo "激活虚拟环境" +echo "============================================================" + +source "$VENV_DIR/bin/activate" + +if [ "$VIRTUAL_ENV" != "" ]; then + echo -e "${GREEN}✓ 虚拟环境已激活: $VIRTUAL_ENV${NC}" + echo -e "${GREEN}Python路径: $(which python)${NC}" +else + echo -e "${RED}错误: 虚拟环境激活失败${NC}" + exit 1 +fi + +# 升级pip +echo "" +echo "============================================================" +echo "升级pip" +echo "============================================================" +python -m pip install --upgrade pip -i https://pypi.tuna.tsinghua.edu.cn/simple + +# 安装依赖 +echo "" +echo "============================================================" +echo "安装项目依赖" +echo "============================================================" + +if [ -f "requirements.txt" ]; then + echo -e "${GREEN}从 requirements.txt 安装依赖...${NC}" + pip install -r requirements.txt -i https://pypi.tuna.tsinghua.edu.cn/simple +else + echo -e "${YELLOW}警告: requirements.txt 不存在${NC}" + echo "手动安装核心依赖..." + pip install flask playwright requests loguru apscheduler python-dotenv -i https://pypi.tuna.tsinghua.edu.cn/simple + + # 安装Playwright浏览器 + echo "安装Playwright浏览器驱动..." + playwright install chromium +fi + +# 检查配置文件 +echo "" +echo "============================================================" +echo "检查配置文件" +echo "============================================================" + +if [ ! -f ".env.production" ]; then + echo -e "${YELLOW}警告: .env.production 不存在${NC}" + + if [ -f ".env.example" ]; then + echo "从 .env.example 创建配置文件..." + cp .env.example .env.production + echo -e "${GREEN}✓ 已创建 .env.production${NC}" + echo -e "${RED}请编辑 .env.production 配置文件后重新启动${NC}" + exit 1 + else + echo -e "${RED}错误: 缺少配置文件模板${NC}" + exit 1 + fi +else + echo -e "${GREEN}✓ 配置文件存在: .env.production${NC}" +fi + +# 初始化数据库 +echo "" +echo "============================================================" +echo "初始化数据库" +echo "============================================================" + +if [ ! -f "db/ai_mip_prod.db" ]; then + echo -e "${YELLOW}数据库不存在,正在初始化...${NC}" + + if [ -f "db/init_databases.py" ]; then + # 自动创建生产数据库(跳过交互) + python << EOF +from pathlib import Path +import sqlite3 + +db_dir = Path('db') +db_path = db_dir / 'ai_mip_prod.db' + +if not db_path.exists(): + print(f"创建数据库: {db_path}") + + # 读取并执行SQL脚本 + init_sql = db_dir / 'init_sqlite.sql' + if init_sql.exists(): + with open(init_sql, 'r', encoding='utf-8') as f: + sql_script = f.read() + + conn = sqlite3.connect(str(db_path)) + conn.executescript(sql_script) + conn.commit() + conn.close() + print("✓ 数据库初始化完成") + else: + print("错误: 找不到 init_sqlite.sql") + exit(1) +EOF + echo -e "${GREEN}✓ 数据库初始化成功${NC}" + else + echo -e "${RED}错误: 找不到数据库初始化脚本${NC}" + exit 1 + fi +else + echo -e "${GREEN}✓ 数据库已存在: db/ai_mip_prod.db${NC}" +fi + +# 创建必要的目录 +echo "" +echo "============================================================" +echo "创建必要目录" +echo "============================================================" + +mkdir -p logs data +echo -e "${GREEN}✓ 目录创建完成${NC}" + +# 检查AdsPower连接 +echo "" +echo "============================================================" +echo "检查AdsPower连接" +echo "============================================================" + +python << EOF +import os +os.environ['ENV'] = 'production' + +try: + from adspower_client import AdsPowerClient + client = AdsPowerClient() + profiles = client.list_profiles() + + if profiles: + print("\033[0;32m✓ AdsPower连接正常\033[0m") + profile_count = len(profiles.get('data', {}).get('list', [])) + print(f"\033[0;32m Profile数量: {profile_count}\033[0m") + else: + print("\033[1;33m警告: AdsPower连接失败,请检查配置\033[0m") +except Exception as e: + print(f"\033[1;33m警告: AdsPower连接异常: {str(e)}\033[0m") + print("\033[1;33m 请确保AdsPower客户端已启动\033[0m") +EOF + +# 启动服务 +echo "" +echo "============================================================" +echo "启动Flask服务" +echo "============================================================" + +export ENV=production + +# 检查端口是否被占用 +PORT=5000 +if command -v netstat &> /dev/null; then + if netstat -tuln | grep ":$PORT " > /dev/null; then + echo -e "${YELLOW}警告: 端口 $PORT 已被占用${NC}" + echo "尝试查找占用进程..." + lsof -i :$PORT || true + fi +fi + +echo "" +echo -e "${GREEN}启动服务中...${NC}" +echo "访问地址: http://127.0.0.1:5000" +echo "按 Ctrl+C 停止服务" +echo "" + +# 使用nohup在后台运行(可选) +# nohup python app.py > logs/app.log 2>&1 & +# echo $! > app.pid +# echo -e "${GREEN}✓ 服务已启动(后台运行)${NC}" +# echo "PID: $(cat app.pid)" +# echo "日志: logs/app.log" + +# 前台运行(便于调试) +python app.py diff --git a/static/app.html b/static/app.html new file mode 100644 index 0000000..7794fb9 --- /dev/null +++ b/static/app.html @@ -0,0 +1,875 @@ + + +
+ + +| ID | +站点名称 | +URL | +状态 | +点击数 | +回复数 | +创建时间 | +
|---|---|---|---|---|---|---|
| 加载中... | +||||||