") || strings.Contains(decoded, "<p>") ||
strings.Contains(decoded, "<div") || strings.Contains(decoded, "<br>")
// 计算解码后的中文密度
density := w.calculateChineseDensity(decoded)
// 同时满足有HTML标签和足够的中文密度
if hasHTMLTags && density > 0.3 {
rawContent = decoded
break
}
}
}
}
// 预处理rawContent(如果已找到)
if rawContent != "" {
// 首先进行多次URL解码,处理嵌套编码
for i := 0; i < 3; i++ { // 最多解码3次
if decoded, err := url.QueryUnescape(rawContent); err == nil && decoded != rawContent {
rawContent = decoded
} else {
break
}
}
// 替换HTML实体
rawContent = strings.ReplaceAll(rawContent, "<", "<")
rawContent = strings.ReplaceAll(rawContent, ">", ">")
rawContent = strings.ReplaceAll(rawContent, """, "\"")
rawContent = strings.ReplaceAll(rawContent, "&", "&")
rawContent = strings.ReplaceAll(rawContent, "\\n", "")
rawContent = strings.ReplaceAll(rawContent, "\\r", "")
rawContent = strings.ReplaceAll(rawContent, "\\t", "")
rawContent = strings.ReplaceAll(rawContent, "\\\"", "\"") // 处理转义的双引号
}
// 如果找到了内容,进行清理
if rawContent != "" {
// 移除HTML标签
tagRegex := regexp.MustCompile(`<[^>]*>`)
cleanText := tagRegex.ReplaceAllString(rawContent, "")
// 应用JavaScript大段过滤
cleanText = w.filterJavaScriptBlocks(cleanText)
// 移除多余的空白字符
spaceRegex := regexp.MustCompile(`\s+`)
cleanText = spaceRegex.ReplaceAllString(cleanText, " ")
cleanText = strings.TrimSpace(cleanText)
// 检查是否包含过多的JavaScript代码特征
jsCount := w.jsKeywordCount(cleanText)
chineseDensity := w.calculateChineseDensity(cleanText)
// 移除明显的JavaScript代码块 - 增强版,特别针对微信平台代码
// 1. 移除WX_BJ_REPORT相关代码
cleanText = regexp.MustCompile(`(?s)\s*WX_BJ_REPORT\s*\([^)]*\);`).ReplaceAllString(cleanText, "")
cleanText = regexp.MustCompile(`(?s)\s*var\s+WX_BJ_REPORT\s*=\s*function\s*\([^)]*\)\s*{[^}]*}\s*`).ReplaceAllString(cleanText, "")
cleanText = regexp.MustCompile(`(?s)\s*if\s*\(WX_BJ_REPORT\)[^;]*;`).ReplaceAllString(cleanText, "")
// 2. 移除BadJs相关代码
cleanText = regexp.MustCompile(`(?s)\s*BadJs\s*\([^)]*\);`).ReplaceAllString(cleanText, "")
cleanText = regexp.MustCompile(`(?s)\s*var\s+BadJs\s*=\s*function\s*\([^)]*\)\s*{[^}]*}\s*`).ReplaceAllString(cleanText, "")
cleanText = regexp.MustCompile(`(?s)\s*if\s*\(BadJs\)[^;]*;`).ReplaceAllString(cleanText, "")
// 3. 移除window.logs相关代码
cleanText = regexp.MustCompile(`(?s)\s*window\.logs\s*=\s*\[.*?\];`).ReplaceAllString(cleanText, "")
cleanText = regexp.MustCompile(`(?s)\s*window\.logs\s*\..*?;`).ReplaceAllString(cleanText, "")
// 4. 移除__moon_initcallback相关代码
cleanText = regexp.MustCompile(`(?s)\s*__moon_initcallback\s*=\s*function\s*\([^)]*\)\s*{[^}]*}\s*`).ReplaceAllString(cleanText, "")
cleanText = regexp.MustCompile(`(?s)\s*\.__moon_initcallback\s*\([^)]*\);`).ReplaceAllString(cleanText, "")
// 5. 移除try-catch块
cleanText = regexp.MustCompile(`(?s)\s*try\s*{[^}]*}\s*catch\s*\([^)]*\)\s*{[^}]*}\s*`).ReplaceAllString(cleanText, "")
// 6. 移除函数定义
cleanText = regexp.MustCompile(`(?s)\s*function\s+[^(]*\([^)]*\)\s*{[^}]*}\s*`).ReplaceAllString(cleanText, "")
cleanText = regexp.MustCompile(`(?s)\s*var\s+[^=]*=\s*function\s*\([^)]*\)\s*{[^}]*}\s*`).ReplaceAllString(cleanText, "")
cleanText = regexp.MustCompile(`(?s)\s*[a-zA-Z_$][a-zA-Z0-9_$]*\s*=\s*function\s*\([^)]*\)\s*{[^}]*}\s*`).ReplaceAllString(cleanText, "")
// 7. 移除IIFE函数
cleanText = regexp.MustCompile(`(?s)\s*\(function\s*\([^)]*\)\s*{[^}]*}\)\s*\(\);`).ReplaceAllString(cleanText, "")
// 8. 移除变量声明
cleanText = regexp.MustCompile(`(?s)\s*var\s+[a-zA-Z_$][a-zA-Z0-9_$]*\s*=\s*{[^}]*}\s*;?`).ReplaceAllString(cleanText, "")
cleanText = regexp.MustCompile(`(?s)\s*let\s+[a-zA-Z_$][a-zA-Z0-9_$]*\s*=\s*[^;]*;`).ReplaceAllString(cleanText, "")
cleanText = regexp.MustCompile(`(?s)\s*const\s+[a-zA-Z_$][a-zA-Z0-9_$]*\s*=\s*[^;]*;`).ReplaceAllString(cleanText, "")
cleanText = regexp.MustCompile(`(?s)\s*window\.[a-zA-Z_$][a-zA-Z0-9_$]*\s*=\s*[^;]*;`).ReplaceAllString(cleanText, "")
// 9. 移除控制流语句
cleanText = regexp.MustCompile(`(?s)\s*if\s*\([^)]*\)\s*{[^}]*}\s*`).ReplaceAllString(cleanText, "")
cleanText = regexp.MustCompile(`(?s)\s*for\s*\([^)]*\)\s*{[^}]*}\s*`).ReplaceAllString(cleanText, "")
cleanText = regexp.MustCompile(`(?s)\s*while\s*\([^)]*\)\s*{[^}]*}\s*`).ReplaceAllString(cleanText, "")
// 10. 移除JSON和数组
cleanText = regexp.MustCompile(`(?s)\s*\{\s*"[^"]*"\s*:\s*[^}]*\}\s*`).ReplaceAllString(cleanText, "")
cleanText = regexp.MustCompile(`(?s)\s*\[\s*[^\]]*\]\s*`).ReplaceAllString(cleanText, "")
// 11. 移除网络请求相关代码
cleanText = regexp.MustCompile(`(?s)\s*new\s+XMLHttpRequest\(\)[^;]*;`).ReplaceAllString(cleanText, "")
cleanText = regexp.MustCompile(`(?s)\s*xmlobj\s*\.[^;]*;`).ReplaceAllString(cleanText, "")
cleanText = regexp.MustCompile(`(?s)\s*fetch\s*\([^)]*\)[^;]*;`).ReplaceAllString(cleanText, "")
cleanText = regexp.MustCompile(`(?s)\s*axios\s*\.[^;]*;`).ReplaceAllString(cleanText, "")
// 12. 移除正则表达式和调试代码
cleanText = regexp.MustCompile(`(?s)\s*new\s+RegExp\([^)]*\)[^;]*;`).ReplaceAllString(cleanText, "")
cleanText = regexp.MustCompile(`(?s)\s*\/[^/]*\/[gimuy]*`).ReplaceAllString(cleanText, "")
cleanText = regexp.MustCompile(`(?s)\s*console\.[a-z]+\([^)]*\);`).ReplaceAllString(cleanText, "")
// 13. 移除事件处理相关代码
cleanText = regexp.MustCompile(`(?s)\s*document\.addEventListener\([^)]*\)[^;]*;`).ReplaceAllString(cleanText, "")
cleanText = regexp.MustCompile(`(?s)\s*window\.addEventListener\([^)]*\)[^;]*;`).ReplaceAllString(cleanText, "")
cleanText = regexp.MustCompile(`(?s)\s*on\$1\s*=\s*function\s*\([^)]*\)\s*{[^}]*}\s*`).ReplaceAllString(cleanText, "")
// 14. 移除定时器相关代码
cleanText = regexp.MustCompile(`(?s)\s*setTimeout\s*\([^)]*\)[^;]*;`).ReplaceAllString(cleanText, "")
cleanText = regexp.MustCompile(`(?s)\s*setInterval\s*\([^)]*\)[^;]*;`).ReplaceAllString(cleanText, "")
// 15. 移除微信特有的API调用
cleanText = regexp.MustCompile(`(?s)\s*WeixinJSBridge\s*\..*?;`).ReplaceAllString(cleanText, "")
cleanText = regexp.MustCompile(`(?s)\s*wx\.\w+\s*\([^)]*\)[^;]*;`).ReplaceAllString(cleanText, "")
// 16. 移除logsPagetime相关代码
cleanText = regexp.MustCompile(`(?s)\s*logsPagetime\s*\([^)]*\);`).ReplaceAllString(cleanText, "")
cleanText = regexp.MustCompile(`(?s)\s*var\s+logsPagetime\s*=\s*function\s*\([^)]*\)\s*{[^}]*}\s*`).ReplaceAllString(cleanText, "")
// 17. 移除特定的微信错误处理代码
cleanText = regexp.MustCompile(`(?s)\s*\.error\s*\([^)]*\);`).ReplaceAllString(cleanText, "")
cleanText = regexp.MustCompile(`(?s)\s*\.warn\s*\([^)]*\);`).ReplaceAllString(cleanText, "")
cleanText = regexp.MustCompile(`(?s)\s*\.info\s*\([^)]*\);`).ReplaceAllString(cleanText, "")
// 18. 移除微信平台特定的方法调用
cleanText = regexp.MustCompile(`(?s)\s*document\.write\s*\([^)]*\);`).ReplaceAllString(cleanText, "")
cleanText = regexp.MustCompile(`(?s)\s*document\.writeln\s*\([^)]*\);`).ReplaceAllString(cleanText, "")
// 如果JavaScript关键词较少且中文密度较高,可能是有效的文章内容
// 降低要求:只要中文密度 > 5% 或 长度 > 100 就认为有效
if (jsCount < 10 || chineseDensity > 0.05) && len(cleanText) > 50 {
// 按句子或段落分割,避免一行过长
if len(cleanText) > 0 {
// 首先尝试按段落分割
paragraphs := regexp.MustCompile(`[。!?.!?]\s*`).Split(cleanText, -1)
// 重组段落,保留标点符号
punctuations := regexp.MustCompile(`[。!?.!?]\s*`).FindAllString(cleanText, -1)
for i := 0; i < len(paragraphs); i++ {
if paragraphs[i] != "" {
if i < len(punctuations) {
paragraphs[i] += punctuations[i]
}
// 只添加非空且长度合理的段落(避免添加JavaScript片段)
paragraph := strings.TrimSpace(paragraphs[i])
// 降低过滤条件,增强中文密度考虑
paraDensity := w.calculateChineseDensity(paragraph)
paraJsCount := w.jsKeywordCount(paragraph)
if len(paragraph) > 10 && (paraJsCount < 3 || paraDensity > 0.1) {
textContent = append(textContent, paragraph)
}
}
}
// 如果没有成功分割成段落,直接添加整个文本
if len(textContent) == 0 && len(cleanText) > 50 && (w.jsKeywordCount(cleanText) < 5 || chineseDensity > 0.1) {
textContent = append(textContent, cleanText)
}
}
}
}
// 最后的备选方案:尝试从整个页面中提取非JavaScript的文本内容
// 【修改】参考Python版本,直接提取所有文本,然后过滤
if len(textContent) < 5 { // 如果提取的段落很少,说明前面的方法都失败了
fmt.Printf(" [调试] 前面提取方法只得到%d个段落,尝试简单提取方法\n", len(textContent))
// 方法1:优先尝试从 id="js_content" 容器中提取
contentRegex := regexp.MustCompile(`(?s)
]*id=["']js_content["'][^>]*>(.*?)
\s*`)
styleRegex := regexp.MustCompile(`(?s)`)
allText := scriptRegex.ReplaceAllString(content, "")
allText = styleRegex.ReplaceAllString(allText, "")
// 移除所有HTML标签
tagRegex := regexp.MustCompile(`<[^>]*>`)
allText = tagRegex.ReplaceAllString(allText, "\n")
// 移除HTML实体
allText = strings.ReplaceAll(allText, "<", "<")
allText = strings.ReplaceAll(allText, ">", ">")
allText = strings.ReplaceAll(allText, """, "\"")
allText = strings.ReplaceAll(allText, "&", "&")
allText = strings.ReplaceAll(allText, " ", " ")
// 按行分割,过滤空行和JS代码
textContent = []string{} // 重置
lines := strings.Split(allText, "\n")
for _, line := range lines {
line = strings.TrimSpace(line)
// 基础过滤:只保留有中文的行,且不是明显JS代码
if len(line) > 0 &&
!strings.HasPrefix(line, "var ") &&
!strings.HasPrefix(line, "function") &&
!strings.Contains(line, "window.") &&
w.calculateChineseDensity(line) > 0.1 {
textContent = append(textContent, line)
}
}
fmt.Printf(" [调试] 全局提取到 %d 个段落\n", len(textContent))
}
}
// 对提取的内容应用最终过滤,确保只保留真正的文章正文
filteredContent := w.finalContentFilter(textContent)
// 【调试】输出过滤前后的对比
fmt.Printf(" [调试] 过滤前段落数: %d, 过滤后段落数: %d\n", len(textContent), len(filteredContent))
if len(filteredContent) == 0 && len(textContent) > 0 {
fmt.Printf(" [调试] ⚠️ finalContentFilter 过滤掉了所有内容!\n")
fmt.Printf(" [调试] 过滤前第一段示例: %s\n", textContent[0][:min(len(textContent[0]), 200)])
}
return createTime, title, commentID, reqID, w.extractAuthor(content), filteredContent
}
// min 返回两个整数中的最小值(Go 1.21之前需要手动实现)
func min(a, b int) int {
if a < b {
return a
}
return b
}
// calculateChineseDensity 计算文本中中文字符的密度
func (w *WechatCrawler) calculateChineseDensity(text string) float64 {
if len(text) == 0 {
return 0
}
// 使用正确的Go语言Unicode范围表示法
chineseCount := 0
for _, char := range text {
if char >= 0x4e00 && char <= 0x9fa5 {
chineseCount++
}
}
return float64(chineseCount) / float64(len(text))
}
// 过滤大段JavaScript代码
func (w *WechatCrawler) filterJavaScriptBlocks(text string) string {
// 移除常见的JavaScript代码块模式
patterns := []string{
// 移除JavaScript函数声明
`(?s)function\s+[a-zA-Z_$][\w$]*\s*\([^)]*\)\s*{[^}]*}`,
// 移除匿名函数
`(?s)\(\s*function\s*\([^)]*\)\s*{[^}]*}\s*\)\s*\(\s*\)`,
// 移除对象字面量
`(?s)\{[^}]*\}`,
// 移除数组字面量
`(?s)\[[^\]]*\]`,
// 移除注释
`//[^\n]*`,
`/\*[^*]*\*/`,
// 移除微信特定错误报告代码
`(?s)WX_BJ_REPORT[^;]*;`,
`(?s)BadJs[^;]*;`,
`(?s)window\.[a-zA-Z_$][\w$]*[^;]*;`,
// 移除XMLHttpRequest相关代码
`(?s)xmlobj[^;]*;`,
`(?s)new\s+Image\([^)]*\)`,
`(?s)setRequestHeader[^;]*;`,
// 移除正则表达式
`/[^/]*\/[gimuy]*`,
}
result := text
for _, pattern := range patterns {
regex, err := regexp.Compile(pattern)
if err == nil {
result = regex.ReplaceAllString(result, "")
}
}
return result
}
// 提取纯中文文本
func (w *WechatCrawler) extractChineseText(text string) string {
var result []rune
for _, char := range text {
// 保留中文、标点符号、数字和英文字母,去除特殊字符
if (char >= 0x4e00 && char <= 0x9fa5) ||
unicode.IsPunct(char) ||
unicode.IsDigit(char) ||
unicode.IsLetter(char) ||
char == '\n' || char == ' ' {
result = append(result, char)
}
}
return string(result)
}
// finalContentFilter 最终内容过滤,确保只保留真正的文章正文
// 修改:大幅降低过滤门槛,参考Python版本的简单逻辑
func (w *WechatCrawler) finalContentFilter(textContent []string) []string {
var validParagraphs []string
// 【修改】如果提取的段落很少,说明可能是提取阶段的问题,直接返回
if len(textContent) <= 3 {
fmt.Printf(" [调试] 提取的段落太少(%d个),可能提取逻辑有问题,跳过过滤\n", len(textContent))
// 简单过滤:只去掉纯标题行和过短的内容
for _, text := range textContent {
text = strings.TrimSpace(text)
// 去掉明显的JavaScript关键词行
if len(text) > 5 &&
!strings.Contains(text, "function(") &&
!strings.Contains(text, "window.") &&
!strings.Contains(text, "var ") {
validParagraphs = append(validParagraphs, text)
}
}
return validParagraphs
}
// 【修改】降低过滤标准,参考Python版本
for _, text := range textContent {
// 基础清理
text = strings.TrimSpace(text)
// 计算中文字符数
chineseCount := 0
for _, char := range text {
if char >= 0x4e00 && char <= 0x9fa5 {
chineseCount++
}
}
// 计算中文密度
paraDensity := w.calculateChineseDensity(text)
paraJsCount := w.jsKeywordCount(text)
// 【大幅降低门槛】:
// - 长度 > 10(原来25)
// - 中文字符 > 3(原来15)
// - 中文密度 > 0.15(原来0.4)
// - JavaScript关键词 < 5(原来3)
if len(text) > 10 &&
!strings.Contains(text, "function(") &&
!strings.Contains(text, "window.") &&
!strings.Contains(text, "WX_BJ_REPORT") &&
!strings.Contains(text, "BadJs") &&
chineseCount > 3 &&
paraDensity > 0.15 &&
paraJsCount < 5 {
validParagraphs = append(validParagraphs, text)
}
}
// 【新增】如果过滤后还是空的,使用最宽松的规则
if len(validParagraphs) == 0 && len(textContent) > 0 {
fmt.Printf(" [调试] 标准过滤后仍为空,使用最宽松规则\n")
for _, text := range textContent {
text = strings.TrimSpace(text)
// 只要有中文字符且不是明显的JS代码就保留
overallDensity := w.calculateChineseDensity(text)
overallJsCount := w.jsKeywordCount(text)
if len(text) > 5 && overallDensity > 0.1 && overallJsCount < 10 {
validParagraphs = append(validParagraphs, text)
}
}
}
return validParagraphs
}
// jsKeywordCount 计算文本中JavaScript关键词的数量 - 增强版
func (w *WechatCrawler) jsKeywordCount(text string) int {
count := 0
// 新增加的高优先级过滤关键词
highPriorityKeywords := []string{
"logs = ", "window.", "LANG = ", "extInfo:", "pagetime[",
"BadJs;", "sec_open=", "xmlobj = ", "addEventListener",
"new Image()", "setRequestHeader", "onreadystatechange",
"var ", "let ", "const ", "function ", "return ",
}
// 基础JavaScript关键词
basicKeywords := []string{
"function", "var", "let", "const", "if(", "else", "for(", "while(",
"return", "setTimeout", "setInterval", "WeixinJSBridge", "JSON",
"console", "document", "window", "try{", "catch(", "throw",
}
// 微信平台特定关键词
wechatKeywords := []string{
"WX_BJ_REPORT", "BadJs", "__moon_initcallback", "logsPagetime",
"WeixinJSBridge", "wx.", "document.write", "document.writeln",
// 错误处理关键词
".error(", ".warn(", ".info(", ".debug(",
// 网络请求关键词
"XMLHttpRequest", "fetch(", "axios.", "xmlobj.",
}
lowerText := strings.ToLower(text)
// 计算高优先级关键词数量(权重更高)
for _, keyword := range highPriorityKeywords {
count += strings.Count(lowerText, strings.ToLower(keyword)) * 3
}
// 计算微信平台特定关键词数量
for _, keyword := range wechatKeywords {
count += strings.Count(lowerText, strings.ToLower(keyword)) * 2
}
// 计算基础JavaScript关键词数量
for _, keyword := range basicKeywords {
count += strings.Count(lowerText, strings.ToLower(keyword))
}
return count
}
// extractAuthor 提取文章作者信息
func (w *WechatCrawler) extractAuthor(content string) string {
authorPatterns := []string{
`var author\s*=\s*['"](.*?)['"]`,
`"author"\s*:\s*['"](.*?)['"]`,
`window\.author\s*=\s*['"](.*?)['"]`,
`
1 {
author := match[1]
// 尝试解码HTML实体和URL编码
author = strings.ReplaceAll(author, """, "\"")
author = strings.ReplaceAll(author, "&", "&")
author = strings.ReplaceAll(author, "<", "<")
author = strings.ReplaceAll(author, ">", ">")
if decoded, err := url.QueryUnescape(author); err == nil {
author = decoded
}
return author
}
}
return ""
}
// GetArticleStats 获取文章统计信息
func (w *WechatCrawler) GetArticleStats(link string, title string, commentID string, reqID string, createTime string) (map[string]string, error) {
// 解析链接参数
mid := ""
sn := ""
idx := ""
// 尝试从链接中提取参数
midRegex := regexp.MustCompile(`mid=(.*?)&`)
if match := midRegex.FindStringSubmatch(link); len(match) > 1 {
mid = match[1]
}
snRegex := regexp.MustCompile(`sn=(.*?)&`)
if match := snRegex.FindStringSubmatch(link); len(match) > 1 {
sn = match[1]
}
idxRegex := regexp.MustCompile(`idx=(.*?)&`)
if match := idxRegex.FindStringSubmatch(link); len(match) > 1 {
idx = match[1]
}
// 生成随机r值
r := fmt.Sprintf("0.%d", time.Now().UnixNano()%10000000000000000)
// 构建请求URL
detailURL := fmt.Sprintf("https://mp.weixin.qq.com/mp/getappmsgext?f=json&mock=&fasttmplajax=1&f=json&uin=%s&key=%s&pass_ticket=%s&__biz=%s",
w.uin, w.key, w.passTicket, w.biz)
// 构建请求数据
data := map[string]string{
"r": r,
"sn": sn,
"mid": mid,
"idx": idx,
"req_id": reqID,
"title": title,
"comment_id": commentID,
"appmsg_type": "9",
"__biz": w.biz,
"pass_ticket": w.passTicket,
"abtest_cookie": "",
"devicetype": "Windows 7 x64",
"version": "63090b13",
"is_need_ticket": "0",
"is_need_ad": "0",
"is_need_reward": "0",
"both_ad": "0",
"reward_uin_count": "0",
"send_time": "",
"msg_daily_idx": "1",
"is_original": "0",
"is_only_read": "1",
"scene": "38",
}
// 发送POST请求
resp, err := w.client.R().SetFormData(data).Post(detailURL)
if err != nil {
return nil, fmt.Errorf("请求统计信息失败: %v", err)
}
// 解析响应
var result map[string]interface{}
err = json.Unmarshal([]byte(resp.String()), &result)
if err != nil {
return nil, fmt.Errorf("解析统计信息失败: %v", err)
}
// 提取统计数据
stats := map[string]string{
"read_num": "0",
"old_like_num": "0",
"share_num": "0",
"show_read": "0",
}
// 从返回的JSON中提取所需数据
if appMsgExtInfo, ok := result["appmsgstat"].(map[string]interface{}); ok {
if readNum, ok := appMsgExtInfo["read_num"].(float64); ok {
stats["read_num"] = fmt.Sprintf("%.0f", readNum)
}
if likeNum, ok := appMsgExtInfo["old_like_num"].(float64); ok {
stats["old_like_num"] = fmt.Sprintf("%.0f", likeNum)
}
if shareNum, ok := appMsgExtInfo["share_num"].(float64); ok {
stats["share_num"] = fmt.Sprintf("%.0f", shareNum)
}
if showRead, ok := appMsgExtInfo["show_read"].(float64); ok {
stats["show_read"] = fmt.Sprintf("%.0f", showRead)
}
}
return stats, nil
}
// GetArticleComments 获取文章评论
func (w *WechatCrawler) GetArticleComments(commentID string) ([]string, []string, error) {
if commentID == "" {
return []string{}, []string{}, nil
}
// 构建评论请求URL
commentURL := fmt.Sprintf(
"https://mp.weixin.qq.com/mp/appmsg_comment?action=getcomment&__biz=%s&appmsgid=2247491372&idx=1&comment_id=%s&offset=0&limit=100&uin=%s&key=%s&pass_ticket=%s&wxtoken=&devicetype=Windows+10&clientversion=62060833&appmsg_token=",
w.biz, commentID, w.uin, w.key, w.passTicket)
// 发送请求
resp, err := w.client.R().Get(commentURL)
if err != nil {
return []string{}, []string{}, fmt.Errorf("获取评论失败: %v", err)
}
// 解析响应
var result map[string]interface{}
err = json.Unmarshal([]byte(resp.String()), &result)
if err != nil {
return []string{}, []string{}, fmt.Errorf("解析评论失败: %v", err)
}
// 提取评论和点赞数
var comments []string
var commentLikes []string
// 简化实现,在实际项目中需要根据返回的JSON结构正确提取数据
return comments, commentLikes, nil
}
// GetOfficialAccountLinkFromArticle 通过文章链接获取公众号主页链接
func (w *WechatCrawler) GetOfficialAccountLinkFromArticle(articleURL string) (string, error) {
// 首先尝试从URL中提取__biz参数(兼容旧格式)
bizRegex := regexp.MustCompile(`__biz=([^&]+)`)
match := bizRegex.FindStringSubmatch(articleURL)
if len(match) >= 2 {
biz := match[1]
// 更新当前实例的biz值
w.biz = biz
// 构建公众号主页链接
homePageURL := fmt.Sprintf("https://mp.weixin.qq.com/mp/profile_ext?action=home&__biz=%s&scene=124", biz)
return homePageURL, nil
}
// 如果从URL中提取失败,尝试从文章内容中提取
content, err := w.GetOneArticle(articleURL)
if err != nil {
return "", fmt.Errorf("获取文章内容失败: %v", err)
}
// 从文章内容中提取biz
contentBizRegex := regexp.MustCompile(`var biz = "(.*?);`)
contentMatch := contentBizRegex.FindStringSubmatch(content)
if len(contentMatch) < 2 {
// 尝试其他可能的biz格式
contentBizRegex2 := regexp.MustCompile(`__biz=(.*?)&`)
contentMatch = contentBizRegex2.FindStringSubmatch(content)
if len(contentMatch) < 2 {
return "", fmt.Errorf("无法从文章链接和内容中提取公众号信息")
}
}
// 清理biz值,移除可能的额外引号
biz := contentMatch[1]
biz = strings.ReplaceAll(biz, " || ", "")
biz = strings.ReplaceAll(biz, "\"", "")
// 更新当前实例的biz值
w.biz = biz
// 构建公众号主页链接
homePageURL := fmt.Sprintf("https://mp.weixin.qq.com/mp/profile_ext?action=home&__biz=%s&scene=124", biz)
return homePageURL, nil
}
// GetArticleList 获取公众号所有文章列表
func (w *WechatCrawler) GetArticleList() ([][]string, error) {
var allArticles [][]string
offset := 0
for {
fmt.Printf("正在获取第%d页文章...\n", offset/10+1)
result, err := w.GetNextList(offset)
if err != nil {
return allArticles, fmt.Errorf("获取文章列表失败: %v", err)
}
// 检查是否还有更多文章
mFlag, ok := result["m_flag"].(int)
if !ok {
// 尝试转换为float64(JSON反序列化可能将数字解析为float64)
if mFlagFloat, ok := result["m_flag"].(float64); ok {
mFlag = int(mFlagFloat)
} else {
mFlag = 0
}
}
if mFlag == 0 {
break
}
// 获取当前页的文章列表
passageList, ok := result["passage_list"].([][]string)
if !ok {
return allArticles, fmt.Errorf("文章列表格式错误")
}
// 添加到总列表
allArticles = append(allArticles, passageList...)
// 增加偏移量
offset += 10
// 随机延迟,避免被封禁
time.Sleep(time.Duration(2000+offset) * time.Millisecond)
}
// 转换链接
transformedArticles := w.TransformLinks(allArticles)
fmt.Printf("共获取到%d篇文章\n", len(transformedArticles))
return transformedArticles, nil
}
// SaveArticleListToExcel 保存文章列表到Excel
func (w *WechatCrawler) SaveArticleListToExcel(officialPath string, articleList [][]string, nickname string) error {
// 确保目录存在
if err := os.MkdirAll(officialPath, 0755); err != nil {
return fmt.Errorf("创建目录失败: %v", err)
}
// 保存转换后的链接文件
filePath := fmt.Sprintf("%s/文章列表(article_list)_直连链接.txt", officialPath)
var content strings.Builder
// 添加 UTF-8 BOM 头,确保 Excel 正确识别编码
content.WriteString("\xEF\xBB\xBF")
// 写入标题行
content.WriteString("序号,创建时间,标题,链接\n")
// 写入文章列表
for i, article := range articleList {
if len(article) < 4 {
continue // 跳过不完整的数据
}
// 转换时间戳为可读格式(如果是时间戳)
createTime := article[1]
// 调试输出:查看原始时间戳
if i == 0 { // 只打印第一篇文章,避免输出过多
fmt.Printf("调试信息 - 第1篇文章\n")
fmt.Printf(" article[0]: '%s'\n", article[0])
fmt.Printf(" article[1] (时间戳): '%s'\n", article[1])
fmt.Printf(" article[2] (标题): '%s'\n", article[2])
fmt.Printf(" 时间戳长度: %d\n", len(article[1]))
}
if createTime != "" && createTime != "0" {
// 尝试将字符串转换为时间戳
var ts int64
n, err := fmt.Sscanf(createTime, "%d", &ts)
if i == 0 {
fmt.Printf(" Sscanf 结果: n=%d, err=%v, ts=%d\n", n, err, ts)
}
if err == nil && n == 1 && ts > 0 {
// 转换为可读的日期时间格式
createTime = time.Unix(ts, 0).Format("2006-01-02 15:04:05")
if i == 0 {
fmt.Printf(" 转换后的时间: %s\n", createTime)
}
} else {
// 如果转换失败,保留原始值
if i == 0 {
fmt.Printf(" 转换失败,保留原始值: %s\n", createTime)
}
}
} else {
if i == 0 {
fmt.Printf(" 时间戳为空或为0,设置为'未知时间'\n")
}
createTime = "未知时间"
}
// 清理和转义标题(移除换行符、制表符等)
title := strings.TrimSpace(article[2])
title = strings.ReplaceAll(title, "\n", " ")
title = strings.ReplaceAll(title, "\r", " ")
title = strings.ReplaceAll(title, "\t", " ")
// 如果标题包含逗号或引号,需要用双引号包裹并转义内部引号
if strings.Contains(title, ",") || strings.Contains(title, "\"") || strings.Contains(title, "\n") {
title = "\"" + strings.ReplaceAll(title, "\"", "\"\"") + "\""
}
// 清理链接
link := strings.TrimSpace(article[3])
// 写入CSV行
content.WriteString(fmt.Sprintf("%d,%s,%s,%s\n", i+1, createTime, title, link))
}
// 写入文件
err := os.WriteFile(filePath, []byte(content.String()), 0644)
if err != nil {
return fmt.Errorf("保存文章列表失败: %v", err)
}
fmt.Printf("文章列表已保存到: %s\n", filePath)
fmt.Printf("共保存 %d 篇文章\n", len(articleList))
return nil
}
// TransformLinks 转换文章链接,将带有amp;的链接转换为直接可访问的链接
func (w *WechatCrawler) TransformLinks(articleList [][]string) [][]string {
transformedList := make([][]string, 0, len(articleList))
for _, article := range articleList {
if len(article) >= 4 {
// 转换链接,移除amp;
transformedLink := strings.Replace(article[3], "amp;", "", -1)
transformedArticle := []string{article[0], article[1], article[2], transformedLink}
transformedList = append(transformedList, transformedArticle)
}
}
return transformedList
}
// ReadArticleLinksFromExcel 从Excel读取文章链接
func (w *WechatCrawler) ReadArticleLinksFromExcel(filePath string) ([]string, error) {
// 简化实现,返回空列表
return []string{}, nil
}
// GetArticleDetail 获取单篇文章的详细信息
func (w *WechatCrawler) GetArticleDetail(link string) (*ArticleDetail, error) {
// 获取文章内容
content, err := w.GetOneArticle(link)
if err != nil {
return nil, err
}
// 【调试】保存原始HTML到文件,用于分析内容提取问题
debugPath := "./debug_article_raw.html"
if err := os.WriteFile(debugPath, []byte(content), 0644); err == nil {
fmt.Printf(" [调试] 原始HTML已保存: %s (长度: %d 字节)\n", debugPath, len(content))
}
// 提取文章信息
createTime, title, commentID, reqID, _, textContent := w.ExtractArticleInfo(content)
// 【调试】输出内容提取详情
fmt.Printf(" [调试] 提取结果 - 标题: %s, 段落数: %d\n", title, len(textContent))
if len(textContent) > 0 {
firstPara := textContent[0]
if len(firstPara) > 100 {
firstPara = firstPara[:100] + "..."
}
fmt.Printf(" [调试] 第一段: %s\n", firstPara)
} else {
fmt.Printf(" [调试] ⚠️ ExtractArticleInfo 未提取到任何内容!\n")
}
// 提取公众号名称
accountName := w.ExtractOfficialAccountName(content)
// 获取统计信息
stats, err := w.GetArticleStats(link, title, commentID, reqID, createTime)
if err != nil {
// 如果获取统计信息失败,使用默认值
stats = map[string]string{
"read_num": "0",
"old_like_num": "0",
"share_num": "0",
"show_read": "0",
}
}
// 获取评论信息
comments, commentLikes, _ := w.GetArticleComments(commentID)
// 构建文章详情
detail := &ArticleDetail{
LocalTime: time.Now().Format("2006-01-02 15:04:05"),
CreateTime: createTime,
Title: title,
OfficialName: accountName,
Link: link,
Content: textContent,
ReadCount: stats["read_num"],
LikeCount: stats["old_like_num"],
ShareCount: stats["share_num"],
Comments: comments,
CommentLikes: commentLikes,
CommentID: commentID,
}
return detail, nil
}
// GetDetailList 批量获取文章详情
func (w *WechatCrawler) GetDetailList(articleList [][]string, officialPath string) error {
// 确保目录存在
if err := os.MkdirAll(officialPath, 0755); err != nil {
return fmt.Errorf("创建目录失败: %v", err)
}
successCount := 0
errorCount := 0
errorLinks := [][]string{}
for i, article := range articleList {
if len(article) < 4 {
continue
}
link := article[3]
title := article[2]
fmt.Printf("正在处理第%d篇文章: %s\n", i+1, title)
// 获取文章详情
detail, err := w.GetArticleDetail(link)
if err != nil {
fmt.Printf("获取文章详情失败: %v\n", err)
errorCount++
errorLinks = append(errorLinks, article)
continue
}
// 保存文章详情 - 确保使用文章标题作为文件名,并清理非法字符
// 清理标题中的非法字符
cleanTitle := detail.Title
invalidChars := []string{"\\", "/", ":", "*", "?", "\"", "<", ">", "|"}
for _, char := range invalidChars {
cleanTitle = strings.ReplaceAll(cleanTitle, char, "_")
}
// 限制文件名长度,避免路径过长
if len(cleanTitle) > 100 {
cleanTitle = cleanTitle[:100]
}
filePath := fmt.Sprintf("%s/%s_文章详情.txt", officialPath, cleanTitle)
if err := w.SaveArticleDetailToExcel(detail, filePath); err != nil {
fmt.Printf("保存文章详情失败: %v\n", err)
errorCount++
errorLinks = append(errorLinks, article)
continue
}
successCount++
fmt.Printf("文章详情保存成功: %s\n", detail.Title)
// 随机延迟,避免被封禁
delayTime := 3000 + i*100 // 3秒基础延迟,递增
time.Sleep(time.Duration(delayTime) * time.Millisecond)
}
// 保存错误链接
if len(errorLinks) > 0 {
errorPath := fmt.Sprintf("%s/问题链接(error_links).txt", officialPath)
var content strings.Builder
content.WriteString("序号,创建时间,标题,链接\n")
for i, link := range errorLinks {
content.WriteString(fmt.Sprintf("%d,%s,%s,%s\n", i+1, link[1], link[2], link[3]))
}
err := os.WriteFile(errorPath, []byte(content.String()), 0644)
if err != nil {
fmt.Printf("保存错误链接失败: %v\n", err)
}
}
fmt.Printf("文章详情获取完成: 成功%d篇, 失败%d篇\n", successCount, errorCount)
return nil
}
// SaveArticleDetailToExcel 保存文章详情到Excel
func (c *WechatCrawler) SaveArticleDetailToExcel(article *ArticleDetail, filePath string) error {
// 【修复】不要清理整个路径!只需要确保目录存在即可
// filePath 已经在调用处清理过了文件名部分
// 这里直接使用即可
var content strings.Builder
// 添加 UTF-8 BOM 头,确保正确显示中文
content.WriteString("\xEF\xBB\xBF")
content.WriteString("=")
content.WriteString(strings.Repeat("=", 80))
content.WriteString("\n")
content.WriteString(fmt.Sprintf("本地创建时间: %s\n", article.LocalTime))
content.WriteString(fmt.Sprintf("文章发布时间: %s\n", article.CreateTime))
content.WriteString(fmt.Sprintf("公众号名称: %s\n", article.OfficialName))
content.WriteString(fmt.Sprintf("文章标题: %s\n", article.Title))
content.WriteString(fmt.Sprintf("文章链接: %s\n", article.Link))
content.WriteString(fmt.Sprintf("阅读量: %s\n", article.ReadCount))
content.WriteString(fmt.Sprintf("点赞数: %s\n", article.LikeCount))
content.WriteString(fmt.Sprintf("转发数: %s\n", article.ShareCount))
content.WriteString(strings.Repeat("=", 80))
content.WriteString("\n\n")
content.WriteString("文章内容:\n")
content.WriteString(strings.Repeat("-", 80))
content.WriteString("\n")
for i, line := range article.Content {
// 清理内容,移除多余的空白字符
cleanLine := strings.TrimSpace(line)
if cleanLine != "" {
content.WriteString(cleanLine)
content.WriteString("\n")
// 每个段落后添加空行,提高可读性
if i < len(article.Content)-1 {
content.WriteString("\n")
}
}
}
// 如果有评论,添加评论区
if len(article.Comments) > 0 {
content.WriteString("\n")
content.WriteString(strings.Repeat("=", 80))
content.WriteString("\n")
content.WriteString(fmt.Sprintf("评论区 (共 %d 条评论):\n", len(article.Comments)))
content.WriteString(strings.Repeat("-", 80))
content.WriteString("\n\n")
for i, comment := range article.Comments {
content.WriteString(fmt.Sprintf("%d. %s", i+1, comment))
if i < len(article.CommentLikes) && article.CommentLikes[i] != "" {
content.WriteString(fmt.Sprintf(" (点赞: %s)", article.CommentLikes[i]))
}
content.WriteString("\n\n")
}
}
content.WriteString("\n")
content.WriteString(strings.Repeat("=", 80))
content.WriteString("\n")
content.WriteString("文件结束\n")
// 写入文件
err := os.WriteFile(filePath, []byte(content.String()), 0644)
if err != nil {
return fmt.Errorf("保存文章详情失败: %v", err)
}
return nil
}
// GetListArticleFromFile 根据公众号名称或文章链接,从文件中读取文章列表并下载内容
func (w *WechatCrawler) GetListArticleFromFile(nameLink string, imgSaveFlag bool, contentSaveFlag bool) error {
// 1. 判断输入类型并获取公众号名称
nickname := ""
if strings.Contains(nameLink, "http") {
fmt.Println("检测到输入为链接,开始获取公众号名称")
// 从文章链接获取公众号信息
content, err := w.GetOneArticle(nameLink)
if err != nil {
return fmt.Errorf("获取文章内容失败: %v", err)
}
// 从内容中提取公众号名称
nickname = w.ExtractOfficialAccountName(content)
if nickname == "" {
return fmt.Errorf("无法从文章中提取公众号名称")
}
fmt.Printf("获取到公众号名称: %s\n", nickname)
} else {
fmt.Println("检测到输入为公众号名称")
nickname = nameLink
}
// 2. 构建文件路径
rootPath := "./data/"
officialPath := rootPath + nickname
// 【新增】创建"文章详细"子目录
articleDetailPath := officialPath + "/文章详细"
articleListPath := officialPath + "/文章列表(article_list)_直连链接.txt"
// 3. 检查文件是否存在
if _, err := os.Stat(articleListPath); os.IsNotExist(err) {
return fmt.Errorf("文件不存在,请检查目录文件: %s", articleListPath)
}
// 4. 读取文章链接列表
fileContent, err := os.ReadFile(articleListPath)
if err != nil {
return fmt.Errorf("读取文章列表文件失败: %v", err)
}
lines := strings.Split(string(fileContent), "\n")
var articleLinks []string
var articleTitles []string
var articleTimes []string
// 跳过BOM头和标题行,提取链接
for i, line := range lines {
if i == 0 || line == "" {
continue
}
// 移除可能的BOM头
line = strings.TrimPrefix(line, "\xEF\xBB\xBF")
line = strings.TrimSpace(line)
if line == "" {
continue
}
// 解析CSV行(处理带引号的字段)
var parts []string
inQuote := false
currentPart := ""
for _, char := range line {
if char == '"' {
inQuote = !inQuote
} else if char == ',' && !inQuote {
parts = append(parts, currentPart)
currentPart = ""
} else {
currentPart += string(char)
}
}
parts = append(parts, currentPart) // 添加最后一个字段
if len(parts) >= 4 {
// 序号,创建时间,标题,链接
time := strings.TrimSpace(parts[1])
title := strings.TrimSpace(parts[2])
link := strings.TrimSpace(parts[3])
// 清理引号
link = strings.Trim(link, "\"")
title = strings.Trim(title, "\"")
if link != "" && link != "链接" { // 跳过标题行
articleLinks = append(articleLinks, link)
articleTitles = append(articleTitles, title)
articleTimes = append(articleTimes, time)
}
}
}
fmt.Printf("成功读取到 %d 篇文章链接\n", len(articleLinks))
if len(articleLinks) == 0 {
return fmt.Errorf("未能从文件中提取到有效的文章链接")
}
// 5. 遍历下载每篇文章
successCount := 0
errorCount := 0
errorLinks := [][]string{} // 保存失败的文章信息
// 【新增】确保"文章详细"目录存在
if err := os.MkdirAll(articleDetailPath, 0755); err != nil {
return fmt.Errorf("创建文章详细目录失败: %v", err)
}
fmt.Printf("文章详细将保存到: %s\n", articleDetailPath)
for i, link := range articleLinks {
title := ""
if i < len(articleTitles) {
title = articleTitles[i]
}
creatTime := ""
if i < len(articleTimes) {
creatTime = articleTimes[i]
}
fmt.Printf("\n正在处理第 %d/%d 篇文章\n", i+1, len(articleLinks))
fmt.Printf("标题: %s\n", title)
fmt.Printf("链接: %s\n", link)
// 获取文章详情
detail, err := w.GetArticleDetail(link)
if err != nil {
fmt.Printf("❌ 获取文章详情失败: %v\n", err)
errorCount++
// 记录失败的文章
errorLinks = append(errorLinks, []string{
fmt.Sprintf("%d", i+1),
creatTime,
title,
link,
})
continue
}
// 保存文章内容
if contentSaveFlag {
// 清理标题中的非法字符
cleanTitle := detail.Title
invalidChars := []string{"\\", "/", ":", "*", "?", "\"", "<", ">", "|"}
for _, char := range invalidChars {
cleanTitle = strings.ReplaceAll(cleanTitle, char, "_")
}
// 限制文件名长度
if len(cleanTitle) > 100 {
cleanTitle = cleanTitle[:100]
}
// 【修改】生成文件路径,保存到"文章详细"子目录中
filePath := fmt.Sprintf("%s/%s_文章详情.txt", articleDetailPath, cleanTitle)
// 调试:打印文件保存路径和内容长度
fmt.Printf(" 保存路径: %s\n", filePath)
fmt.Printf(" 内容段落数: %d\n", len(detail.Content))
if len(detail.Content) > 0 {
previewLen := 50
if len(detail.Content[0]) < previewLen {
previewLen = len(detail.Content[0])
}
fmt.Printf(" 第一段内容预览: %s...\n", detail.Content[0][:previewLen])
} else {
fmt.Printf(" ⚠️ 警告:文章内容为空!\n")
}
if err := w.SaveArticleDetailToExcel(detail, filePath); err != nil {
fmt.Printf("❌ 保存文章详情失败: %v\n", err)
errorCount++
errorLinks = append(errorLinks, []string{
fmt.Sprintf("%d", i+1),
creatTime,
title,
link,
})
continue
}
fmt.Printf("✅ 文章保存成功: %s\n", detail.Title)
}
// TODO: 保存图片功能(如果需要)
if imgSaveFlag {
fmt.Println("图片保存功能暂未实现")
}
successCount++
// 添加延迟,避免被封
if i < len(articleLinks)-1 { // 不是最后一篇
delayTime := 3 + i/10 // 基础延迟3秒,每10篇增加1秒
fmt.Printf("为预防被封禁,延时 %d 秒...\n", delayTime)
time.Sleep(time.Duration(delayTime) * time.Second)
}
}
// 6. 保存失败的文章链接
if len(errorLinks) > 0 {
errorPath := officialPath + "/问题链接(error_links).txt"
var errorContent strings.Builder
// 添加 BOM 头
errorContent.WriteString("\xEF\xBB\xBF")
errorContent.WriteString("序号,创建时间,标题,链接\n")
for _, errorLink := range errorLinks {
// 处理标题中的逗号和引号
title := errorLink[2]
if strings.Contains(title, ",") || strings.Contains(title, "\"") {
title = "\"" + strings.ReplaceAll(title, "\"", "\"\"") + "\""
}
errorContent.WriteString(fmt.Sprintf("%s,%s,%s,%s\n",
errorLink[0], errorLink[1], title, errorLink[3]))
}
err := os.WriteFile(errorPath, []byte(errorContent.String()), 0644)
if err != nil {
fmt.Printf("⚠️ 保存错误链接失败: %v\n", err)
} else {
fmt.Printf("\n已保存失败的文章链接到: %s\n", errorPath)
}
}
fmt.Printf("\n" + strings.Repeat("=", 60) + "\n")
fmt.Printf("文章列表处理完成!\n")
fmt.Printf(" 成功: %d 篇\n", successCount)
fmt.Printf(" 失败: %d 篇\n", errorCount)
fmt.Printf(" 总计: %d 篇\n", len(articleLinks))
fmt.Printf(strings.Repeat("=", 60) + "\n")
return nil
}