commit
This commit is contained in:
sjk
399
task_executor.py
399
task_executor.py
@@ -1,303 +1,250 @@
|
||||
#!/usr/bin/env python
|
||||
# -*- coding: utf-8 -*-
|
||||
"""
|
||||
任务执行器模块
|
||||
|
||||
提供广告点击任务的执行能力，包括：
|
||||
- 浏览器环境创建
|
||||
- 单个任务执行
|
||||
- 批量任务调度
|
||||
负责管理浏览器生命周期和执行点击任务
|
||||
"""
|
||||
|
||||
import time
|
||||
from typing import Dict, Optional
|
||||
from loguru import logger
|
||||
|
||||
from config import Config
|
||||
from adspower_client import AdsPowerClient
|
||||
from ad_automation import MIPAdAutomation
|
||||
from config import Config
|
||||
from data_manager import DataManager
|
||||
import time
|
||||
import threading
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
from typing import List, Dict, Optional
|
||||
|
||||
|
||||
class TaskExecutor:
|
||||
"""
|
||||
任务执行器
|
||||
|
||||
负责执行单个或批量广告点击任务。
|
||||
支持代理配置、浏览器环境管理、任务结果追踪。
|
||||
负责：
|
||||
1. 管理AdsPower浏览器环境
|
||||
2. 执行MIP广告点击任务
|
||||
3. 记录执行结果
|
||||
"""
|
||||
|
||||
_browser_start_lock = threading.Lock()
|
||||
|
||||
def __init__(self, max_workers: int = 1, use_proxy: bool = True):
|
||||
"""
|
||||
初始化任务执行器
|
||||
|
||||
Args:
|
||||
max_workers: 最大并发数（1=串行，>1=并发）
|
||||
max_workers: 最大并发数（当前仅支持1）
|
||||
use_proxy: 是否使用代理
|
||||
"""
|
||||
self.max_workers = max_workers
|
||||
self.use_proxy = use_proxy
|
||||
self.client = AdsPowerClient()
|
||||
self.dm = DataManager()
|
||||
self._browser_info = None
|
||||
self._proxy_id = None # 保存创建的代理ID，用于关闭时清理
|
||||
self._profile_id = None # 保存创建的Profile ID，用于关闭时清理
|
||||
|
||||
# 创建截图目录（按日期组织）
|
||||
timestamp = datetime.now().strftime('%Y%m%d')
|
||||
self.screenshot_dir = Path("./test") / f"batch_{timestamp}"
|
||||
self.screenshot_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
logger.debug(f"TaskExecutor initialized: workers={max_workers}, proxy={use_proxy}")
|
||||
logger.info(f"TaskExecutor 初始化: max_workers={max_workers}, use_proxy={use_proxy}")
|
||||
|
||||
def create_browser_profile(self, index: int) -> Optional[Dict]:
|
||||
def create_browser_profile(self, index: int = 1) -> Optional[Dict]:
|
||||
"""
|
||||
创建浏览器环境
|
||||
创建浏览器环境（启动AdsPower浏览器）
|
||||
|
||||
流程：
|
||||
1. 根据当前运行环境获取对应的分组ID（dev/prod）
|
||||
2. 获取代理并创建AdsPower代理
|
||||
3. 用代理ID创建新的profile
|
||||
4. 启动浏览器
|
||||
|
||||
Args:
|
||||
index: 环境编号
|
||||
index: 任务索引
|
||||
|
||||
Returns:
|
||||
环境信息字典，失败返回None
|
||||
包含 profile_id 的字典，失败返回 None
|
||||
"""
|
||||
try:
|
||||
# 获取分组ID
|
||||
logger.info(f"[Task {index}] 创建浏览器环境...")
|
||||
|
||||
# 1. 获取当前环境对应的分组ID (dev/prod)
|
||||
group_id = self.client.get_group_by_env()
|
||||
time.sleep(0.5)
|
||||
if not group_id:
|
||||
logger.error(f"[Task {index}] 获取分组ID失败，请确保AdsPower中存在dev或prod分组")
|
||||
return None
|
||||
|
||||
# 如果使用代理，获取代理配置
|
||||
proxy_config = {}
|
||||
logger.info(f"[Task {index}] 使用分组ID: {group_id}")
|
||||
|
||||
# 2. 获取大麦代理并创建AdsPower代理
|
||||
proxy_id = None
|
||||
proxy_info = None
|
||||
|
||||
if self.use_proxy:
|
||||
logger.info(f"[环境 {index}] 获取代理IP...")
|
||||
logger.info(f"[Task {index}] 获取大麦IP代理...")
|
||||
proxy_info = self.client.get_damai_proxy()
|
||||
time.sleep(0.5)
|
||||
|
||||
if proxy_info:
|
||||
logger.info(f"[环境 {index}] 代理IP: {proxy_info['host']}:{proxy_info['port']}")
|
||||
|
||||
proxy_data = {
|
||||
proxy_config = {
|
||||
"type": "http",
|
||||
"host": proxy_info["host"],
|
||||
"port": proxy_info["port"],
|
||||
"user": self.client.DAMAI_USER,
|
||||
"password": self.client.DAMAI_PASSWORD,
|
||||
"remark": f"任务代理_{index}"
|
||||
"ipchecker": "ip2location",
|
||||
"remark": "Damai Auto Proxy"
|
||||
}
|
||||
|
||||
proxy_id = self.client.create_proxy(proxy_data)
|
||||
time.sleep(0.5)
|
||||
|
||||
proxy_id = self.client.create_proxy(proxy_config)
|
||||
if proxy_id:
|
||||
logger.info(f"[环境 {index}] 创建代理: {proxy_id}")
|
||||
proxy_config = {"proxyid": proxy_id}
|
||||
self._proxy_id = proxy_id
|
||||
logger.info(f"[Task {index}] 创建代理成功: {proxy_id}")
|
||||
else:
|
||||
logger.warning(f"[Task {index}] 创建代理失败，将不使用代理")
|
||||
else:
|
||||
logger.warning(f"[Task {index}] 获取大麦代理失败，将不使用代理")
|
||||
|
||||
# 根据环境变量决定操作系统
|
||||
os_type = "Linux" if Config.ENV == "production" else "Windows"
|
||||
# 3. 创建新的profile（必须带proxy_id）
|
||||
if not proxy_id:
|
||||
logger.error(f"[Task {index}] 没有代理ID，无法创建profile")
|
||||
return None
|
||||
|
||||
profile_data = {
|
||||
"name": f"任务_{index}_{datetime.now().strftime('%H%M%S')}",
|
||||
"group_id": str(group_id) if group_id else "0",
|
||||
"platform": "health.baidu.com",
|
||||
"repeat_config": [],
|
||||
"ignore_cookie_error": "1",
|
||||
"country": "cn",
|
||||
"city": "beijing",
|
||||
"remark": f"任务环境 #{index}",
|
||||
"fingerprint_config": {
|
||||
"automatic_timezone": "1",
|
||||
"flash": "block",
|
||||
"scan_port_type": "1",
|
||||
"location": "ask",
|
||||
"location_switch": "1",
|
||||
"canvas": "0",
|
||||
"webgl": "0",
|
||||
"audio": "0",
|
||||
"webrtc": "local",
|
||||
"do_not_track": "true",
|
||||
"hardware_concurrency": "default",
|
||||
"device_memory": "default",
|
||||
"gpu": "2",
|
||||
"mac_address_config": {
|
||||
"model": "1",
|
||||
"address": ""
|
||||
},
|
||||
"browser_kernel_config": {
|
||||
"version": "latest",
|
||||
"type": "chrome"
|
||||
},
|
||||
"random_ua": {
|
||||
"ua_system_version": [os_type]
|
||||
}
|
||||
}
|
||||
import time
|
||||
profile_name = f"task_{index}_{int(time.time())}"
|
||||
profile_id = self.client.create_profile(group_id=group_id, name=profile_name, proxy_id=proxy_id)
|
||||
|
||||
if not profile_id:
|
||||
logger.error(f"[Task {index}] 创建profile失败")
|
||||
# 删除已创建的代理
|
||||
if self._proxy_id:
|
||||
self.client.delete_proxy(self._proxy_id)
|
||||
self._proxy_id = None
|
||||
return None
|
||||
|
||||
self._profile_id = profile_id
|
||||
logger.info(f"[Task {index}] 创建profile: {profile_id} (名称: {profile_name})")
|
||||
|
||||
# 4. 启动浏览器
|
||||
browser_info = self.client.start_browser(user_id=profile_id)
|
||||
|
||||
if not browser_info or browser_info.get('code') != 0:
|
||||
error_msg = browser_info.get('msg', '未知错误') if browser_info else '无响应'
|
||||
logger.error(f"[Task {index}] 启动浏览器失败: {error_msg}")
|
||||
# 清理资源
|
||||
self.client.delete_profile(profile_id)
|
||||
self._profile_id = None
|
||||
if self._proxy_id:
|
||||
self.client.delete_proxy(self._proxy_id)
|
||||
self._proxy_id = None
|
||||
return None
|
||||
|
||||
self._browser_info = browser_info
|
||||
self.client.user_id = profile_id
|
||||
|
||||
logger.info(f"[Task {index}] 浏览器已启动, profile_id: {profile_id}, proxy_id: {self._proxy_id}")
|
||||
|
||||
return {
|
||||
'profile_id': profile_id,
|
||||
'browser_info': browser_info,
|
||||
'proxy_id': self._proxy_id
|
||||
}
|
||||
|
||||
logger.debug(f"[环境 {index}] 操作系统: {os_type} (ENV={Config.ENV})")
|
||||
|
||||
if proxy_config:
|
||||
profile_data.update(proxy_config)
|
||||
|
||||
response = self.client._make_request(
|
||||
'POST',
|
||||
'/api/v2/browser-profile/create',
|
||||
json=profile_data
|
||||
)
|
||||
|
||||
if response and response.get('code') == 0:
|
||||
profile_id = response.get('data', {}).get('profile_id')
|
||||
logger.info(f"✅ 创建环境 #{index}: {profile_id}")
|
||||
return {
|
||||
'index': index,
|
||||
'profile_id': profile_id,
|
||||
'name': profile_data['name'],
|
||||
'proxy': proxy_info,
|
||||
'proxy_id': proxy_id
|
||||
}
|
||||
else:
|
||||
logger.error(f"❌ 创建环境 #{index} 失败: {response}")
|
||||
return None
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"❌ 创建环境 #{index} 异常: {str(e)}")
|
||||
logger.error(f"[Task {index}] 创建浏览器环境异常: {str(e)}")
|
||||
import traceback
|
||||
traceback.print_exc()
|
||||
return None
|
||||
|
||||
def execute_single_task(self, site_info: Dict, task_index: int, profile_id: str = None) -> Dict:
|
||||
def execute_single_task(self, site: Dict, index: int, profile_id: str) -> Dict:
|
||||
"""
|
||||
执行单个点击任务
|
||||
|
||||
Args:
|
||||
site_info: 站点信息
|
||||
task_index: 任务编号
|
||||
profile_id: 已创建的Profile ID（可选）
|
||||
site: 站点信息，包含 id, site_url 等
|
||||
index: 任务索引
|
||||
profile_id: 浏览器 Profile ID
|
||||
|
||||
Returns:
|
||||
执行结果字典
|
||||
执行结果字典 {'success': bool, 'error': str}
|
||||
"""
|
||||
# 设置线程名称
|
||||
threading.current_thread().name = f"Task-{task_index}"
|
||||
site_id = site.get('id')
|
||||
site_url = site.get('site_url')
|
||||
|
||||
site_id = site_info.get('id')
|
||||
site_url = site_info.get('site_url', site_info.get('url'))
|
||||
|
||||
result = {
|
||||
'task_index': task_index,
|
||||
'site_id': site_id,
|
||||
'site_url': site_url,
|
||||
'success': False,
|
||||
'click_count': 0,
|
||||
'has_ad': False,
|
||||
'has_reply': False,
|
||||
'error': None
|
||||
}
|
||||
|
||||
# 创建任务目录
|
||||
timestamp = datetime.now().strftime('%Y%m%d_%H%M%S')
|
||||
task_folder = self.screenshot_dir / f"task_{task_index}_{timestamp}"
|
||||
task_folder.mkdir(exist_ok=True)
|
||||
|
||||
# 每个线程创建自己的客户端实例
|
||||
client = AdsPowerClient()
|
||||
logger.info(f"[Task {index}] 开始执行: site_id={site_id}, url={site_url}")
|
||||
|
||||
try:
|
||||
logger.info(f"[任务 {task_index}] 开始执行: {site_url}")
|
||||
|
||||
# 如果没有传入profile_id，则创建新的
|
||||
if not profile_id:
|
||||
profiles_data = client.list_profiles()
|
||||
if not profiles_data:
|
||||
result['error'] = "获取Profile列表失败"
|
||||
return result
|
||||
|
||||
profiles = profiles_data.get('data', {}).get('list', [])
|
||||
if not profiles:
|
||||
result['error'] = "没有可用的Profile"
|
||||
return result
|
||||
|
||||
profile_id = profiles[0].get('profile_id')
|
||||
logger.info(f"[任务 {task_index}] 使用Profile: {profile_id}")
|
||||
|
||||
# 使用锁控制浏览器启动
|
||||
with self._browser_start_lock:
|
||||
logger.debug(f"[任务 {task_index}] 启动浏览器...")
|
||||
browser_info = client.start_browser(user_id=profile_id)
|
||||
if not browser_info:
|
||||
result['error'] = "启动浏览器失败"
|
||||
return result
|
||||
time.sleep(1.5)
|
||||
|
||||
time.sleep(1)
|
||||
|
||||
# 连接浏览器
|
||||
browser = client.connect_browser(browser_info)
|
||||
if not self._browser_info:
|
||||
return {'success': False, 'error': '浏览器未启动'}
|
||||
|
||||
browser = self.client.connect_browser(self._browser_info)
|
||||
if not browser:
|
||||
result['error'] = "CDP连接失败"
|
||||
return result
|
||||
return {'success': False, 'error': '连接浏览器失败'}
|
||||
|
||||
# 获取页面
|
||||
context = browser.contexts[0]
|
||||
all_pages = context.pages
|
||||
logger.debug(f"[任务 {task_index}] 当前标签页数: {len(all_pages)}")
|
||||
page = self.client.get_page(browser)
|
||||
if not page:
|
||||
return {'success': False, 'error': '获取页面失败'}
|
||||
|
||||
# 关闭AdsPower启动页
|
||||
for p in all_pages:
|
||||
try:
|
||||
if 'start.adspower.net' in p.url:
|
||||
p.close()
|
||||
except:
|
||||
pass
|
||||
# 清理多余标签页
|
||||
self._cleanup_tabs(browser)
|
||||
|
||||
# 获取或创建页面
|
||||
remaining_pages = context.pages
|
||||
page = remaining_pages[0] if remaining_pages else context.new_page()
|
||||
|
||||
# 执行广告点击和消息发送流程
|
||||
logger.info(f"[任务 {task_index}] 开始执行广告点击和咨询流程...")
|
||||
automation = MIPAdAutomation(page, task_index=task_index)
|
||||
click_success, has_reply = automation.check_and_click_ad(
|
||||
url=site_url,
|
||||
site_id=site_id
|
||||
)
|
||||
# 创建自动化实例并执行
|
||||
automation = MIPAdAutomation(page, task_index=index)
|
||||
click_success, has_reply = automation.check_and_click_ad(site_url, site_id=site_id)
|
||||
|
||||
if click_success:
|
||||
result['success'] = True
|
||||
result['click_count'] = 1
|
||||
result['has_ad'] = True
|
||||
result['has_reply'] = has_reply
|
||||
logger.info(f"[任务 {task_index}] ✅ 任务完成: 点击成功={click_success}, 收到回复={has_reply}")
|
||||
logger.info(f"[Task {index}] 点击成功, 收到回复: {has_reply}")
|
||||
return {'success': True, 'has_reply': has_reply}
|
||||
else:
|
||||
result['error'] = "广告点击失败"
|
||||
logger.warning(f"[任务 {task_index}] ❌ 广告点击失败")
|
||||
|
||||
# 关闭浏览器
|
||||
try:
|
||||
if browser:
|
||||
browser.close()
|
||||
time.sleep(0.5)
|
||||
except:
|
||||
pass
|
||||
|
||||
# 停止浏览器
|
||||
try:
|
||||
client.stop_browser(user_id=profile_id)
|
||||
logger.debug(f"[任务 {task_index}] 浏览器已关闭")
|
||||
time.sleep(1)
|
||||
except Exception as e:
|
||||
logger.warning(f"[任务 {task_index}] 停止浏览器失败: {str(e)}")
|
||||
|
||||
# 删除浏览器Profile（释放资源）
|
||||
try:
|
||||
logger.debug(f"[任务 {task_index}] 删除浏览器Profile: {profile_id}")
|
||||
client.delete_profile(profile_id)
|
||||
except Exception as e:
|
||||
logger.warning(f"[任务 {task_index}] 删除Profile异常: {str(e)}")
|
||||
|
||||
logger.warning(f"[Task {index}] 点击失败")
|
||||
return {'success': False, 'error': '点击广告失败'}
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"[任务 {task_index}] 执行异常: {str(e)}")
|
||||
result['error'] = str(e)
|
||||
logger.error(f"[Task {index}] 执行异常: {str(e)}")
|
||||
import traceback
|
||||
traceback.print_exc()
|
||||
return {'success': False, 'error': str(e)}
|
||||
|
||||
return result
|
||||
finally:
|
||||
# 关闭浏览器
|
||||
self.close_browser(profile_id)
|
||||
|
||||
def close_browser(self, profile_id: str = None):
|
||||
"""
|
||||
关闭浏览器并清理资源（代理和Profile）
|
||||
|
||||
Args:
|
||||
profile_id: Profile ID（可选）
|
||||
"""
|
||||
target_profile_id = profile_id or self._profile_id
|
||||
|
||||
try:
|
||||
# 1. 关闭浏览器
|
||||
logger.info(f"关闭浏览器: {target_profile_id or self.client.user_id}")
|
||||
self.client.stop_browser(user_id=target_profile_id)
|
||||
self._browser_info = None
|
||||
|
||||
# 2. 删除创建的代理
|
||||
if self._proxy_id:
|
||||
logger.info(f"删除代理: {self._proxy_id}")
|
||||
self.client.delete_proxy(self._proxy_id)
|
||||
self._proxy_id = None
|
||||
|
||||
# 3. 删除创建的Profile
|
||||
if self._profile_id:
|
||||
logger.info(f"删除Profile: {self._profile_id}")
|
||||
self.client.delete_profile(self._profile_id)
|
||||
self._profile_id = None
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"关闭浏览器异常: {str(e)}")
|
||||
|
||||
def _cleanup_tabs(self, browser):
|
||||
"""
|
||||
清理多余的标签页，只保留一个
|
||||
|
||||
Args:
|
||||
browser: Playwright Browser 实例
|
||||
"""
|
||||
try:
|
||||
if browser.contexts:
|
||||
context = browser.contexts[0]
|
||||
pages = context.pages
|
||||
|
||||
# 如果有多个标签页，关闭多余的
|
||||
if len(pages) > 1:
|
||||
logger.info(f"清理多余标签页: {len(pages)} -> 1")
|
||||
for page in pages[1:]:
|
||||
try:
|
||||
page.close()
|
||||
except:
|
||||
pass
|
||||
except Exception as e:
|
||||
logger.debug(f"清理标签页异常: {str(e)}")
|
||||
|
||||
Reference in New Issue
Block a user