xuwei/ai_keyword_baidu
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

init commit

Browse Source
This commit is contained in:
徐微
2025-12-08 15:44:38 +08:00
commit f2baf63ef6
2443 changed files with 272043 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

BIN
918543da9a71503048786acdad419b82.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 44 KiB

96
README.md Normal file
View File

@@ -0,0 +1,96 @@
# 项目简介
本项目用于自动化抓取百度和有来医生页面内容，包括：
- 百度搜索推荐词、相关搜索（静态页面，requests+BeautifulSoup）
- 有来医生文章标题和“大家还在搜”（动态页面，Playwright 浏览器自动化）
## 目录结构
```
├── baidu.py # 百度搜索推荐词及相关搜索抓取脚本
├── youlai.py # 有来医生页面内容抓取脚本（静态，仅标题）
├── youlai_ui.py # 有来医生页面内容抓取脚本（动态，标题+大家还在搜）
├── requirements.txt # 依赖包列表（完整环境）
├── baidu_result.txt # 百度抓取结果输出样例
├── youlai_result.txt # 有来医生抓取结果输出样例
├── 1.txt, test.py # 辅助文件
```
## 环境创建
建议使用 conda 创建虚拟环境：
```bash
conda create -n xhs python=3.10
conda activate xhs
pip install -r requirements.txt
```
## Playwright 浏览器驱动安装
首次使用 Playwright 需安装浏览器驱动（包括 Chromium/谷歌浏览器）：
```bash
playwright install
```
如需仅安装 Chromium（谷歌浏览器内核），可执行：
```bash
playwright install chromium
```
Playwright 会自动下载并配置所需的浏览器驱动，无需手动下载 ChromeDriver。
如需使用本地已安装的 Chrome 浏览器，可在脚本中指定 executable_path 参数。
## 脚本执行方法
### 1. 百度内容抓取
```bash
python baidu.py
```
根据提示输入关键词，结果保存到 baidu_result.txt。
**输出样例：**
```
大家都在搜:
糖尿病早期症状
怎么判断得了糖尿病
...
相关搜索:
糖尿病早期症状
怎么判断得了糖尿病
...
```
### 2. 有来医生静态页面抓取（仅标题）
```bash
python youlai.py
```
结果保存到 youlai_result.txt。
### 3. 有来医生动态页面抓取（推荐，支持 JS 渲染内容）
```bash
python youlai_ui.py
```
结果保存到 youlai_result.txt。
**输出样例：**
```
标题:
糖尿病的临床表现及治疗方法
大家还在搜:
糖尿治疗好方法
眼睛视力模糊是什么原因
...
```
## 依赖说明
所有依赖已在 requirements.txt 中列出，包含 requests、beautifulsoup4、playwright 及完整环境包。
## 常见问题
- Playwright 抓取不到内容时，请确认已正确安装驱动，并适当增加等待时间。
- requests 抓取不到 JS 渲染内容时，请优先使用 youlai_ui.py。
- 如需自定义页面或元素，请根据实际 class 名调整脚本选择器。
- 依赖包如有缺失，可根据报错补充到 requirements.txt。
---
如有问题可随时反馈。

119
baidu.py Normal file
View File

@@ -0,0 +1,119 @@
import requests
from bs4 import BeautifulSoup
def fetch_related_words(keyword):
cookies = {
'PSTM': '1764302604',
'BAIDUID': '17E56B6A4915D5B98222C8D7A7CFF059:FG=1',
'BD_HOME': '1',
'H_PS_PSSID': '63140_64007_65866_66117_66218_66194_66236_66243_66168_66362_66281_66264_66393_66395_66479_66510_66529_66553_66589_66590_66602_66614_66647_66679_66692_66695_66687',
'delPer': '0',
'BD_CK_SAM': '1',
'PSINO': '3',
'BAIDUID_BFESS': '17E56B6A4915D5B98222C8D7A7CFF059:FG=1',
'PAD_BROWSER': '1',
'BD_UPN': '12314753',
'BDORZ': 'B490B5EBF6F3CD402E515D22BCDA1598',
'BA_HECTOR': 'ala10k2421a0ag25a5a40524a10l8n1kii7of24',
'BIDUPSID': 'C047CB4D757AC8632D7B5792A4254C89',
'ZFY': 'hLpeh2:BHPDeKfEN3yuM7C:A7dmFl03pP:AkeekLlPw5J4:C',
'channel': 'baidusearch',
'H_WISE_SIDS': '63140_64007_65866_66117_66218_66194_66236_66243_66168_66362_66281_66264_66393_66395_66479_66510_66529_66553_66589_66590_66602_66614_66647_66679_66692_66695_66687',
'baikeVisitId': '7f510782-16ce-4371-ad1d-cc8c0ba5ccc8',
'COOKIE_SESSION': '0_0_1_0_0_0_1_0_1_1_7462_1_0_0_0_0_0_0_1764302605%7C1%230_0_1764302605%7C1',
'H_PS_645EC': 'c7554ktAJah5Z6fmLi0RDEpB3a2TvS0rgHEQ7JP12K2UeBuFhGHrlxODIbY',
'BDSVRTM': '16',
'WWW_ST': '1764310101036',
}
from urllib.parse import quote
headers = {
'Accept': '*/*',
'Accept-Language': 'zh-CN,zh;q=0.9',
'Connection': 'keep-alive',
'Referer': f'https://www.baidu.com/s?ie=utf-8&f=8&rsv_bp=1&tn=baidu&wd={quote(keyword)}',
'Sec-Fetch-Dest': 'empty',
'Sec-Fetch-Mode': 'cors',
'Sec-Fetch-Site': 'same-origin',
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/142.0.0.0 Safari/537.36',
'X-Requested-With': 'XMLHttpRequest',
# 'is_pbs': quote(keyword), # 中文字段已去除
# 'is_referer': f'https://www.baidu.com/s?ie=utf-8&f=8&rsv_bp=1&tn=baidu&wd={quote(keyword)}',
'is_xhr': '1',
'sec-ch-ua': '"Chromium";v="142", "Google Chrome";v="142", "Not_A Brand";v="99"',
'sec-ch-ua-mobile': '?0',
'sec-ch-ua-platform': '"Windows"',
}
params = [
('ie', 'utf-8'),
('mod', '1'),
('isbd', '1'),
('isid', 'b857f27e00205440'),
('ie', 'utf-8'),
('f', '8'),
('rsv_bp', '1'),
('tn', 'baidu'),
('wd', keyword),
('oq', keyword),
('rsv_pq', 'b857f27e00205440'),
('rsv_t', 'c7554ktAJah5Z6fmLi0RDEpB3a2TvS0rgHEQ7JP12K2UeBuFhGHrlxODIbY'),
('rqlang', 'cn'),
('rsv_enter', '1'),
('rsv_dl', 'tb'),
('rsv_btype', 't'),
('inputT', '23852'),
('rsv_sug2', '0'),
('rsv_sug3', '15'),
('rsv_sug1', '23'),
('rsv_sug7', '100'),
('rsv_sug4', '23852'),
('bs', keyword),
('rsv_sid', 'undefined'),
('_ss', '1'),
('clist', 'ddad409c4a1855aa'),
('hsug', ''),
('f4s', '1'),
('csor', '3'),
('_cr1', '35542'),
]
response = requests.get('https://www.baidu.com/s', params=params, cookies=cookies, headers=headers)
soup = BeautifulSoup(response.text, 'html.parser')
result = []
div = soup.find('div', class_='list_1V4Yg')
if div:
for a in div.find_all('a', class_='item_3WKCf'):
spans = a.find_all('span')
if len(spans) > 1:
result.append(spans[1].get_text(strip=True))
# 相关搜索内容
related_search = []
rs_label = soup.find('div', class_='c-color-t rs-label_ihUhK')
if rs_label:
rs_table = rs_label.find_next('table', class_='rs-table_3RiQc')
if rs_table:
for a in rs_table.find_all('a', class_='rs-link_2DE3Q'):
span = a.find('span', class_='rs-text_3K5mR')
if span:
related_search.append(span.get_text(strip=True))
# 保存所有内容到一个文件
with open('baidu_result.txt', 'w', encoding='utf-8') as f:
f.write('大家都在搜:\n')
for item in result:
f.write(item + '\n')
f.write('\n相关搜索:\n')
for item in related_search:
f.write(item + '\n')
return result, related_search
if __name__ == '__main__':
keyword = input('请输入关键词：')
words, related_search = fetch_related_words(keyword)
print("大家都在搜：", words)
print("相关搜索：", related_search)
print("所有内容已保存到 baidu_result.txt")

17
baidu.txt Normal file
View File

@@ -0,0 +1,17 @@
curl 'https://www.baidu.com/s?ie=utf-8&mod=1&isbd=1&isid=b857f27e00205440&ie=utf-8&f=8&rsv_bp=1&tn=baidu&wd=%E7%B3%96%E5%B0%BF%E7%97%85&oq=%25E7%25B3%2596%25E5%25B0%25BF%25E7%2597%2585&rsv_pq=b857f27e00205440&rsv_t=c7554ktAJah5Z6fmLi0RDEpB3a2TvS0rgHEQ7JP12K2UeBuFhGHrlxODIbY&rqlang=cn&rsv_enter=1&rsv_dl=tb&rsv_btype=t&inputT=23852&rsv_sug2=0&rsv_sug3=15&rsv_sug1=23&rsv_sug7=100&rsv_sug4=23852&bs=%E7%B3%96%E5%B0%BF%E7%97%85&rsv_sid=undefined&_ss=1&clist=ddad409c4a1855aa&hsug=&f4s=1&csor=3&_cr1=35542' \
-H 'Accept: */*' \
-H 'Accept-Language: zh-CN,zh;q=0.9' \
-H 'Connection: keep-alive' \
-b 'PSTM=1764302604; BAIDUID=17E56B6A4915D5B98222C8D7A7CFF059:FG=1; BD_HOME=1; H_PS_PSSID=63140_64007_65866_66117_66218_66194_66236_66243_66168_66362_66281_66264_66393_66395_66479_66510_66529_66553_66589_66590_66602_66614_66647_66679_66692_66695_66687; delPer=0; BD_CK_SAM=1; PSINO=3; BAIDUID_BFESS=17E56B6A4915D5B98222C8D7A7CFF059:FG=1; PAD_BROWSER=1; BD_UPN=12314753; BDORZ=B490B5EBF6F3CD402E515D22BCDA1598; BA_HECTOR=ala10k2421a0ag25a5a40524a10l8n1kii7of24; BIDUPSID=C047CB4D757AC8632D7B5792A4254C89; ZFY=hLpeh2:BHPDeKfEN3yuM7C:A7dmFl03pP:AkeekLlPw5J4:C; channel=baidusearch; H_WISE_SIDS=63140_64007_65866_66117_66218_66194_66236_66243_66168_66362_66281_66264_66393_66395_66479_66510_66529_66553_66589_66590_66602_66614_66647_66679_66692_66695_66687; baikeVisitId=7f510782-16ce-4371-ad1d-cc8c0ba5ccc8; COOKIE_SESSION=0_0_1_0_0_0_1_0_1_1_7462_1_0_0_0_0_0_0_1764302605%7C1%230_0_1764302605%7C1; H_PS_645EC=c7554ktAJah5Z6fmLi0RDEpB3a2TvS0rgHEQ7JP12K2UeBuFhGHrlxODIbY; BDSVRTM=16; WWW_ST=1764310101036' \
-H 'Referer: https://www.baidu.com/s?ie=utf-8&f=8&rsv_bp=1&tn=baidu&wd=%E7%B3%96%E5%B0%BF%E7%97%85&oq=%25E7%25B3%2596%25E5%25B0%25BF%25E7%2597%2585&rsv_pq=b857f27e00205440&rsv_t=c7554ktAJah5Z6fmLi0RDEpB3a2TvS0rgHEQ7JP12K2UeBuFhGHrlxODIbY&rqlang=cn&rsv_enter=1&rsv_dl=tb&rsv_btype=t&inputT=23852&rsv_sug2=0&rsv_sug3=15&rsv_sug1=23&rsv_sug7=100&rsv_sug4=23852' \
-H 'Sec-Fetch-Dest: empty' \
-H 'Sec-Fetch-Mode: cors' \
-H 'Sec-Fetch-Site: same-origin' \
-H 'User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/142.0.0.0 Safari/537.36' \
-H 'X-Requested-With: XMLHttpRequest' \
-H 'is_pbs: %E7%B3%96%E5%B0%BF%E7%97%85' \
-H 'is_referer: https://www.baidu.com/s?ie=utf-8&f=8&rsv_bp=1&tn=baidu&wd=%E7%B3%96%E5%B0%BF%E7%97%85&oq=%25E5%25B0%258F%25E7%25BA%25A2%25E4%25B9%25A6&rsv_pq=858fcec4000b85a5&rsv_t=7ba4cEvxcRvEH2NNzizIajpdj1FB0D6Dx3FSIu31uMWr%2FlYn6B9VRudQF0A&rqlang=cn&rsv_enter=1&rsv_dl=tb&rsv_sug3=14&rsv_sug1=21&rsv_sug7=101&rsv_btype=t&inputT=7688&rsv_sug4=7688&bs=%E5%B0%8F%E7%BA%A2%E4%B9%A6' \
-H 'is_xhr: 1' \
-H 'sec-ch-ua: "Chromium";v="142", "Google Chrome";v="142", "Not_A Brand";v="99"' \
-H 'sec-ch-ua-mobile: ?0' \
-H 'sec-ch-ua-platform: "Windows"'

3
baidu_result.txt Normal file
View File

@@ -0,0 +1,3 @@
大家都在搜:
相关搜索:

1072
log_baidu_keyword Normal file
View File

File diff suppressed because it is too large Load Diff

183
main.py Normal file
View File

@@ -0,0 +1,183 @@
import requests
from bs4 import BeautifulSoup
import pymysql
DB_CONFIG = {
'host': '8.149.233.36',
'user': 'ai_article_read',
'password': '7aK_H2yvokVumr84lLNDt8fDBp6P',
'database': 'ai_article',
'charset': 'utf8mb4'
}
def init_db():
# 只做连接关闭，去除建表语句
pass
def insert_keywords(words):
conn = pymysql.connect(**DB_CONFIG)
cursor = conn.cursor()
for word in words:
try:
cursor.execute(
"INSERT IGNORE INTO baidu_keyword (keyword) VALUES (%s)", (word,)
)
except Exception as e:
print(f'插入失败：{word}，原因：{e}')
conn.commit()
cursor.close()
conn.close()
def get_random_keyword():
conn = pymysql.connect(**DB_CONFIG)
cursor = conn.cursor()
cursor.execute("SELECT keyword FROM baidu_keyword WHERE crawled=0 ORDER BY RAND() LIMIT 1")
result = cursor.fetchone()
cursor.close()
conn.close()
return result[0] if result else None
# 不再需要 mark_crawled
def export_all_keywords(filename='seed_pool.txt'):
conn = pymysql.connect(**DB_CONFIG)
cursor = conn.cursor()
cursor.execute("SELECT keyword FROM baidu_keyword")
all_words = [row[0] for row in cursor.fetchall()]
cursor.close()
conn.close()
with open(filename, 'w', encoding='utf-8') as f:
for word in sorted(all_words):
f.write(word + '\n')
print(f'已导出 {len(all_words)} 个关键词到 {filename}')
def fetch_related_words(keyword):
cookies = {
# ...existing cookies...
'PSTM': '1764302604',
'BAIDUID': '17E56B6A4915D5B98222C8D7A7CFF059:FG=1',
'BD_HOME': '1',
'H_PS_PSSID': '63140_64007_65866_66117_66218_66194_66236_66243_66168_66362_66281_66264_66393_66395_66479_66510_66529_66553_66589_66590_66602_66614_66647_66679_66692_66695_66687',
'delPer': '0',
'BD_CK_SAM': '1',
'PSINO': '3',
'BAIDUID_BFESS': '17E56B6A4915D5B98222C8D7A7CFF059:FG=1',
'PAD_BROWSER': '1',
'BD_UPN': '12314753',
'BDORZ': 'B490B5EBF6F3CD402E515D22BCDA1598',
'BA_HECTOR': 'ala10k2421a0ag25a5a40524a10l8n1kii7of24',
'BIDUPSID': 'C047CB4D757AC8632D7B5792A4254C89',
'ZFY': 'hLpeh2:BHPDeKfEN3yuM7C:A7dmFl03pP:AkeekLlPw5J4:C',
'channel': 'baidusearch',
'H_WISE_SIDS': '63140_64007_65866_66117_66218_66194_66236_66243_66168_66362_66281_66264_66393_66395_66479_66510_66529_66553_66589_66590_66602_66614_66647_66679_66692_66695_66687',
'baikeVisitId': '7f510782-16ce-4371-ad1d-cc8c0ba5ccc8',
'COOKIE_SESSION': '0_0_1_0_0_0_1_0_1_1_7462_1_0_0_0_0_0_0_1764302605%7C1%230_0_1764302605%7C1',
'H_PS_645EC': 'c7554ktAJah5Z6fmLi0RDEpB3a2TvS0rgHEQ7JP12K2UeBuFhGHrlxODIbY',
'BDSVRTM': '16',
'WWW_ST': '1764310101036',
}
from urllib.parse import quote
headers = {
'Accept': '*/*',
'Accept-Language': 'zh-CN,zh;q=0.9',
'Connection': 'keep-alive',
'Referer': f'https://www.baidu.com/s?ie=utf-8&f=8&rsv_bp=1&tn=baidu&wd={quote(keyword)}',
'Sec-Fetch-Dest': 'empty',
'Sec-Fetch-Mode': 'cors',
'Sec-Fetch-Site': 'same-origin',
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/142.0.0.0 Safari/537.36',
'X-Requested-With': 'XMLHttpRequest',
'is_xhr': '1',
'sec-ch-ua': '"Chromium";v="142", "Google Chrome";v="142", "Not_A Brand";v="99"',
'sec-ch-ua-mobile': '?0',
'sec-ch-ua-platform': '"Windows"',
}
params = [
('ie', 'utf-8'),
('mod', '1'),
('isbd', '1'),
('isid', 'b857f27e00205440'),
('ie', 'utf-8'),
('f', '8'),
('rsv_bp', '1'),
('tn', 'baidu'),
('wd', keyword),
('oq', keyword),
('rsv_pq', 'b857f27e00205440'),
('rsv_t', 'c7554ktAJah5Z6fmLi0RDEpB3a2TvS0rgHEQ7JP12K2UeBuFhGHrlxODIbY'),
('rqlang', 'cn'),
('rsv_enter', '1'),
('rsv_dl', 'tb'),
('rsv_btype', 't'),
('inputT', '23852'),
('rsv_sug2', '0'),
('rsv_sug3', '15'),
('rsv_sug1', '23'),
('rsv_sug7', '100'),
('rsv_sug4', '23852'),
('bs', keyword),
('rsv_sid', 'undefined'),
('_ss', '1'),
('clist', 'ddad409c4a1855aa'),
('hsug', ''),
('f4s', '1'),
('csor', '3'),
('_cr1', '35542'),
]
response = requests.get('https://www.baidu.com/s', params=params, cookies=cookies, headers=headers)
soup = BeautifulSoup(response.text, 'html.parser')
result = []
div = soup.find('div', class_='list_1V4Yg')
if div:
for a in div.find_all('a', class_='item_3WKCf'):
spans = a.find_all('span')
if len(spans) > 1:
result.append(spans[1].get_text(strip=True))
related_search = []
rs_label = soup.find('div', class_='c-color-t rs-label_ihUhK')
if rs_label:
rs_table = rs_label.find_next('table', class_='rs-table_3RiQc')
if rs_table:
for a in rs_table.find_all('a', class_='rs-link_2DE3Q'):
span = a.find('span', class_='rs-text_3K5mR')
if span:
related_search.append(span.get_text(strip=True))
return result, related_search
def main():
# 首次插入种子词（如数据库为空时）
insert_keywords(['糖尿病'])
while True:
keyword = get_random_keyword()
if not keyword:
print('数据库无未抓取关键词，等待新词...')
import time
time.sleep(10)
continue
print(f'正在抓取：{keyword}')
try:
words, related_search = fetch_related_words(keyword)
except Exception as e:
print(f'抓取失败：{keyword}，原因：{e}')
continue
new_words = set(words + related_search)
insert_keywords(new_words)
# 抓取后标记 crawled=1
conn = pymysql.connect(**DB_CONFIG)
cursor = conn.cursor()
cursor.execute("UPDATE baidu_keyword SET crawled=1 WHERE keyword=%s", (keyword,))
conn.commit()
cursor.close()
conn.close()
export_all_keywords()
if __name__ == '__main__':
main()

87
requirements.txt Normal file
View File

@@ -0,0 +1,87 @@
aiofiles==24.1.0
aiohttp==3.9.5
aiohttp-cors==0.8.1
aiosignal==1.3.2
alembic==1.16.1
anyio==4.9.0
asgiref==3.8.1
async-lru==2.0.5
async-timeout==4.0.3
attrs==25.3.0
backports-datetime-fromisoformat==2.0.3
beautifulsoup4==4.14.2
biliup==0.4.98
blinker==1.9.0
Brotli==1.1.0
bs4==0.0.2
certifi==2025.4.26
cf_clearance==0.31.0
cffi==1.17.1
charset-normalizer==3.4.2
click==8.2.1
colorama==0.4.6
dnspython==2.7.0
eventlet==0.40.0
exceptiongroup==1.3.0
Flask==3.1.1
flask-cors==6.0.0
frozenlist==1.6.0
greenlet==3.2.2
h11==0.16.0
h2==4.2.0
hpack==4.1.0
httpcore==1.0.9
httpx==0.28.1
hyperframe==6.1.0
idna==3.10
isodate==0.7.2
itsdangerous==2.2.0
Jinja2==3.1.6
jsengine==1.0.7.post1
loguru==0.7.3
lxml==5.4.0
m3u8==6.0.0
Mako==1.3.10
MarkupSafe==3.0.2
multidict==6.4.4
outcome==1.3.0.post0
pillow==11.2.1
pip==25.3
playwright==1.52.0
propcache==0.3.1
protobuf==6.31.1
psutil==7.0.0
pyasn1==0.6.1
pycountry==24.6.1
pycparser==2.22
pycryptodome==3.23.0
pyee==13.0.0
PySocks==1.7.1
PyYAML==6.0.2
qrcode==8.2
requests==2.32.3
rsa==4.9.1
schedule==1.2.2
setuptools==80.9.0
sniffio==1.3.1
sortedcontainers==2.4.0
soupsieve==2.8
SQLAlchemy==2.0.41
stream-gears==0.2.2
streamlink==7.3.0
tomli==2.2.1
tomli_w==1.2.0
trio==0.30.0
trio-websocket==0.12.2
truststore==0.10.1
typing_extensions==4.13.2
urllib3==2.4.0
websocket-client==1.8.0
Werkzeug==3.1.3
wheel==0.45.1
win32_setctime==1.2.0
wsproto==1.2.0
xhs==0.2.13
yarl==1.20.0
ykdl==1.8.2
yt-dlp==2025.5.22

0
seed_pool.txt Normal file
View File

247
venv/bin/Activate.ps1 Normal file
View File

@@ -0,0 +1,247 @@
<#
.Synopsis
Activate a Python virtual environment for the current PowerShell session.
.Description
Pushes the python executable for a virtual environment to the front of the
$Env:PATH environment variable and sets the prompt to signify that you are
in a Python virtual environment. Makes use of the command line switches as
well as the `pyvenv.cfg` file values present in the virtual environment.
.Parameter VenvDir
Path to the directory that contains the virtual environment to activate. The
default value for this is the parent of the directory that the Activate.ps1
script is located within.
.Parameter Prompt
The prompt prefix to display when this virtual environment is activated. By
default, this prompt is the name of the virtual environment folder (VenvDir)
surrounded by parentheses and followed by a single space (ie. '(.venv) ').
.Example
Activate.ps1
Activates the Python virtual environment that contains the Activate.ps1 script.
.Example
Activate.ps1 -Verbose
Activates the Python virtual environment that contains the Activate.ps1 script,
and shows extra information about the activation as it executes.
.Example
Activate.ps1 -VenvDir C:\Users\MyUser\Common\.venv
Activates the Python virtual environment located in the specified location.
.Example
Activate.ps1 -Prompt "MyPython"
Activates the Python virtual environment that contains the Activate.ps1 script,
and prefixes the current prompt with the specified string (surrounded in
parentheses) while the virtual environment is active.
.Notes
On Windows, it may be required to enable this Activate.ps1 script by setting the
execution policy for the user. You can do this by issuing the following PowerShell
command:
PS C:\> Set-ExecutionPolicy -ExecutionPolicy RemoteSigned -Scope CurrentUser
For more information on Execution Policies:
https://go.microsoft.com/fwlink/?LinkID=135170
#>
Param(
[Parameter(Mandatory = $false)]
[String]
$VenvDir,
[Parameter(Mandatory = $false)]
[String]
$Prompt
)
<# Function declarations --------------------------------------------------- #>
<#
.Synopsis
Remove all shell session elements added by the Activate script, including the
addition of the virtual environment's Python executable from the beginning of
the PATH variable.
.Parameter NonDestructive
If present, do not remove this function from the global namespace for the
session.
#>
function global:deactivate ([switch]$NonDestructive) {
# Revert to original values
# The prior prompt:
if (Test-Path -Path Function:_OLD_VIRTUAL_PROMPT) {
Copy-Item -Path Function:_OLD_VIRTUAL_PROMPT -Destination Function:prompt
Remove-Item -Path Function:_OLD_VIRTUAL_PROMPT
}
# The prior PYTHONHOME:
if (Test-Path -Path Env:_OLD_VIRTUAL_PYTHONHOME) {
Copy-Item -Path Env:_OLD_VIRTUAL_PYTHONHOME -Destination Env:PYTHONHOME
Remove-Item -Path Env:_OLD_VIRTUAL_PYTHONHOME
}
# The prior PATH:
if (Test-Path -Path Env:_OLD_VIRTUAL_PATH) {
Copy-Item -Path Env:_OLD_VIRTUAL_PATH -Destination Env:PATH
Remove-Item -Path Env:_OLD_VIRTUAL_PATH
}
# Just remove the VIRTUAL_ENV altogether:
if (Test-Path -Path Env:VIRTUAL_ENV) {
Remove-Item -Path env:VIRTUAL_ENV
}
# Just remove VIRTUAL_ENV_PROMPT altogether.
if (Test-Path -Path Env:VIRTUAL_ENV_PROMPT) {
Remove-Item -Path env:VIRTUAL_ENV_PROMPT
}
# Just remove the _PYTHON_VENV_PROMPT_PREFIX altogether:
if (Get-Variable -Name "_PYTHON_VENV_PROMPT_PREFIX" -ErrorAction SilentlyContinue) {
Remove-Variable -Name _PYTHON_VENV_PROMPT_PREFIX -Scope Global -Force
}
# Leave deactivate function in the global namespace if requested:
if (-not $NonDestructive) {
Remove-Item -Path function:deactivate
}
}
<#
.Description
Get-PyVenvConfig parses the values from the pyvenv.cfg file located in the
given folder, and returns them in a map.
For each line in the pyvenv.cfg file, if that line can be parsed into exactly
two strings separated by `=` (with any amount of whitespace surrounding the =)
then it is considered a `key = value` line. The left hand string is the key,
the right hand is the value.
If the value starts with a `'` or a `"` then the first and last character is
stripped from the value before being captured.
.Parameter ConfigDir
Path to the directory that contains the `pyvenv.cfg` file.
#>
function Get-PyVenvConfig(
[String]
$ConfigDir
) {
Write-Verbose "Given ConfigDir=$ConfigDir, obtain values in pyvenv.cfg"
# Ensure the file exists, and issue a warning if it doesn't (but still allow the function to continue).
$pyvenvConfigPath = Join-Path -Resolve -Path $ConfigDir -ChildPath 'pyvenv.cfg' -ErrorAction Continue
# An empty map will be returned if no config file is found.
$pyvenvConfig = @{ }
if ($pyvenvConfigPath) {
Write-Verbose "File exists, parse `key = value` lines"
$pyvenvConfigContent = Get-Content -Path $pyvenvConfigPath
$pyvenvConfigContent | ForEach-Object {
$keyval = $PSItem -split "\s*=\s*", 2
if ($keyval[0] -and $keyval[1]) {
$val = $keyval[1]
# Remove extraneous quotations around a string value.
if ("'""".Contains($val.Substring(0, 1))) {
$val = $val.Substring(1, $val.Length - 2)
}
$pyvenvConfig[$keyval[0]] = $val
Write-Verbose "Adding Key: '$($keyval[0])'='$val'"
}
}
}
return $pyvenvConfig
}
<# Begin Activate script --------------------------------------------------- #>
# Determine the containing directory of this script
$VenvExecPath = Split-Path -Parent $MyInvocation.MyCommand.Definition
$VenvExecDir = Get-Item -Path $VenvExecPath
Write-Verbose "Activation script is located in path: '$VenvExecPath'"
Write-Verbose "VenvExecDir Fullname: '$($VenvExecDir.FullName)"
Write-Verbose "VenvExecDir Name: '$($VenvExecDir.Name)"
# Set values required in priority: CmdLine, ConfigFile, Default
# First, get the location of the virtual environment, it might not be
# VenvExecDir if specified on the command line.
if ($VenvDir) {
Write-Verbose "VenvDir given as parameter, using '$VenvDir' to determine values"
}
else {
Write-Verbose "VenvDir not given as a parameter, using parent directory name as VenvDir."
$VenvDir = $VenvExecDir.Parent.FullName.TrimEnd("\\/")
Write-Verbose "VenvDir=$VenvDir"
}
# Next, read the `pyvenv.cfg` file to determine any required value such
# as `prompt`.
$pyvenvCfg = Get-PyVenvConfig -ConfigDir $VenvDir
# Next, set the prompt from the command line, or the config file, or
# just use the name of the virtual environment folder.
if ($Prompt) {
Write-Verbose "Prompt specified as argument, using '$Prompt'"
}
else {
Write-Verbose "Prompt not specified as argument to script, checking pyvenv.cfg value"
if ($pyvenvCfg -and $pyvenvCfg['prompt']) {
Write-Verbose " Setting based on value in pyvenv.cfg='$($pyvenvCfg['prompt'])'"
$Prompt = $pyvenvCfg['prompt'];
}
else {
Write-Verbose " Setting prompt based on parent's directory's name. (Is the directory name passed to venv module when creating the virtual environment)"
Write-Verbose " Got leaf-name of $VenvDir='$(Split-Path -Path $venvDir -Leaf)'"
$Prompt = Split-Path -Path $venvDir -Leaf
}
}
Write-Verbose "Prompt = '$Prompt'"
Write-Verbose "VenvDir='$VenvDir'"
# Deactivate any currently active virtual environment, but leave the
# deactivate function in place.
deactivate -nondestructive
# Now set the environment variable VIRTUAL_ENV, used by many tools to determine
# that there is an activated venv.
$env:VIRTUAL_ENV = $VenvDir
if (-not $Env:VIRTUAL_ENV_DISABLE_PROMPT) {
Write-Verbose "Setting prompt to '$Prompt'"
# Set the prompt to include the env name
# Make sure _OLD_VIRTUAL_PROMPT is global
function global:_OLD_VIRTUAL_PROMPT { "" }
Copy-Item -Path function:prompt -Destination function:_OLD_VIRTUAL_PROMPT
New-Variable -Name _PYTHON_VENV_PROMPT_PREFIX -Description "Python virtual environment prompt prefix" -Scope Global -Option ReadOnly -Visibility Public -Value $Prompt
function global:prompt {
Write-Host -NoNewline -ForegroundColor Green "($_PYTHON_VENV_PROMPT_PREFIX) "
_OLD_VIRTUAL_PROMPT
}
$env:VIRTUAL_ENV_PROMPT = $Prompt
}
# Clear PYTHONHOME
if (Test-Path -Path Env:PYTHONHOME) {
Copy-Item -Path Env:PYTHONHOME -Destination Env:_OLD_VIRTUAL_PYTHONHOME
Remove-Item -Path Env:PYTHONHOME
}
# Add the venv to the PATH
Copy-Item -Path Env:PATH -Destination Env:_OLD_VIRTUAL_PATH
$Env:PATH = "$VenvExecDir$([System.IO.Path]::PathSeparator)$Env:PATH"

70
venv/bin/activate Normal file
View File

@@ -0,0 +1,70 @@
# This file must be used with "source bin/activate" *from bash*
# You cannot run it directly
deactivate () {
# reset old environment variables
if [ -n "${_OLD_VIRTUAL_PATH:-}" ] ; then
PATH="${_OLD_VIRTUAL_PATH:-}"
export PATH
unset _OLD_VIRTUAL_PATH
fi
if [ -n "${_OLD_VIRTUAL_PYTHONHOME:-}" ] ; then
PYTHONHOME="${_OLD_VIRTUAL_PYTHONHOME:-}"
export PYTHONHOME
unset _OLD_VIRTUAL_PYTHONHOME
fi
# Call hash to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
hash -r 2> /dev/null
if [ -n "${_OLD_VIRTUAL_PS1:-}" ] ; then
PS1="${_OLD_VIRTUAL_PS1:-}"
export PS1
unset _OLD_VIRTUAL_PS1
fi
unset VIRTUAL_ENV
unset VIRTUAL_ENV_PROMPT
if [ ! "${1:-}" = "nondestructive" ] ; then
# Self destruct!
unset -f deactivate
fi
}
# unset irrelevant variables
deactivate nondestructive
# on Windows, a path can contain colons and backslashes and has to be converted:
if [ "${OSTYPE:-}" = "cygwin" ] || [ "${OSTYPE:-}" = "msys" ] ; then
# transform D:\path\to\venv to /d/path/to/venv on MSYS
# and to /cygdrive/d/path/to/venv on Cygwin
export VIRTUAL_ENV=$(cygpath /home/work/xw/keyword_crawl/venv)
else
# use the path as-is
export VIRTUAL_ENV=/home/work/xw/keyword_crawl/venv
fi
_OLD_VIRTUAL_PATH="$PATH"
PATH="$VIRTUAL_ENV/"bin":$PATH"
export PATH
# unset PYTHONHOME if set
# this will fail if PYTHONHOME is set to the empty string (which is bad anyway)
# could use `if (set -u; : $PYTHONHOME) ;` in bash
if [ -n "${PYTHONHOME:-}" ] ; then
_OLD_VIRTUAL_PYTHONHOME="${PYTHONHOME:-}"
unset PYTHONHOME
fi
if [ -z "${VIRTUAL_ENV_DISABLE_PROMPT:-}" ] ; then
_OLD_VIRTUAL_PS1="${PS1:-}"
PS1='(venv) '"${PS1:-}"
export PS1
VIRTUAL_ENV_PROMPT='(venv) '
export VIRTUAL_ENV_PROMPT
fi
# Call hash to forget past commands. Without forgetting
# past commands the $PATH changes we made may not be respected
hash -r 2> /dev/null

27
venv/bin/activate.csh Normal file
View File

@@ -0,0 +1,27 @@
# This file must be used with "source bin/activate.csh" *from csh*.
# You cannot run it directly.
# Created by Davide Di Blasi <davidedb@gmail.com>.
# Ported to Python 3.3 venv by Andrew Svetlov <andrew.svetlov@gmail.com>
alias deactivate 'test $?_OLD_VIRTUAL_PATH != 0 && setenv PATH "$_OLD_VIRTUAL_PATH" && unset _OLD_VIRTUAL_PATH; rehash; test $?_OLD_VIRTUAL_PROMPT != 0 && set prompt="$_OLD_VIRTUAL_PROMPT" && unset _OLD_VIRTUAL_PROMPT; unsetenv VIRTUAL_ENV; unsetenv VIRTUAL_ENV_PROMPT; test "\!:*" != "nondestructive" && unalias deactivate'
# Unset irrelevant variables.
deactivate nondestructive
setenv VIRTUAL_ENV /home/work/xw/keyword_crawl/venv
set _OLD_VIRTUAL_PATH="$PATH"
setenv PATH "$VIRTUAL_ENV/"bin":$PATH"
set _OLD_VIRTUAL_PROMPT="$prompt"
if (! "$?VIRTUAL_ENV_DISABLE_PROMPT") then
set prompt = '(venv) '"$prompt"
setenv VIRTUAL_ENV_PROMPT '(venv) '
endif
alias pydoc python -m pydoc
rehash

69
venv/bin/activate.fish Normal file
View File

@@ -0,0 +1,69 @@
# This file must be used with "source <venv>/bin/activate.fish" *from fish*
# (https://fishshell.com/). You cannot run it directly.
function deactivate -d "Exit virtual environment and return to normal shell environment"
# reset old environment variables
if test -n "$_OLD_VIRTUAL_PATH"
set -gx PATH $_OLD_VIRTUAL_PATH
set -e _OLD_VIRTUAL_PATH
end
if test -n "$_OLD_VIRTUAL_PYTHONHOME"
set -gx PYTHONHOME $_OLD_VIRTUAL_PYTHONHOME
set -e _OLD_VIRTUAL_PYTHONHOME
end
if test -n "$_OLD_FISH_PROMPT_OVERRIDE"
set -e _OLD_FISH_PROMPT_OVERRIDE
# prevents error when using nested fish instances (Issue #93858)
if functions -q _old_fish_prompt
functions -e fish_prompt
functions -c _old_fish_prompt fish_prompt
functions -e _old_fish_prompt
end
end
set -e VIRTUAL_ENV
set -e VIRTUAL_ENV_PROMPT
if test "$argv[1]" != "nondestructive"
# Self-destruct!
functions -e deactivate
end
end
# Unset irrelevant variables.
deactivate nondestructive
set -gx VIRTUAL_ENV /home/work/xw/keyword_crawl/venv
set -gx _OLD_VIRTUAL_PATH $PATH
set -gx PATH "$VIRTUAL_ENV/"bin $PATH
# Unset PYTHONHOME if set.
if set -q PYTHONHOME
set -gx _OLD_VIRTUAL_PYTHONHOME $PYTHONHOME
set -e PYTHONHOME
end
if test -z "$VIRTUAL_ENV_DISABLE_PROMPT"
# fish uses a function instead of an env var to generate the prompt.
# Save the current fish_prompt function as the function _old_fish_prompt.
functions -c fish_prompt _old_fish_prompt
# With the original prompt function renamed, we can override with our own.
function fish_prompt
# Save the return status of the last command.
set -l old_status $status
# Output the venv prompt; color taken from the blue of the Python logo.
printf "%s%s%s" (set_color 4B8BBE) '(venv) ' (set_color normal)
# Restore the return status of the previous command.
echo "exit $old_status" | .
# Output the original/"old" prompt.
_old_fish_prompt
end
set -gx _OLD_FISH_PROMPT_OVERRIDE "$VIRTUAL_ENV"
set -gx VIRTUAL_ENV_PROMPT '(venv) '
end

8
venv/bin/normalizer Normal file
View File

@@ -0,0 +1,8 @@
#!/home/work/xw/keyword_crawl/venv/bin/python3
# -*- coding: utf-8 -*-
import re
import sys
from charset_normalizer.cli import cli_detect
if __name__ == '__main__':
sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0])
sys.exit(cli_detect())

8
venv/bin/pip Normal file
View File

@@ -0,0 +1,8 @@
#!/home/work/xw/keyword_crawl/venv/bin/python3
# -*- coding: utf-8 -*-
import re
import sys
from pip._internal.cli.main import main
if __name__ == '__main__':
sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0])
sys.exit(main())

8
venv/bin/pip3 Normal file
View File

@@ -0,0 +1,8 @@
#!/home/work/xw/keyword_crawl/venv/bin/python3
# -*- coding: utf-8 -*-
import re
import sys
from pip._internal.cli.main import main
if __name__ == '__main__':
sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0])
sys.exit(main())

8
venv/bin/pip3.12 Normal file
View File

@@ -0,0 +1,8 @@
#!/home/work/xw/keyword_crawl/venv/bin/python3
# -*- coding: utf-8 -*-
import re
import sys
from pip._internal.cli.main import main
if __name__ == '__main__':
sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0])
sys.exit(main())

0
venv/bin/python Normal file
View File

0
venv/bin/python3 Normal file
View File

0
venv/bin/python3.12 Normal file
View File

1
venv/lib/python3.12/site-packages/README.rst Normal file
View File

@@ -0,0 +1 @@
This is a dummy package designed to prevent namesquatting on PyPI. You should install `beautifulsoup4 <https://pypi.python.org/pypi/beautifulsoup4>`_ instead.

BIN
venv/lib/python3.12/site-packages/__pycache__/typing_extensions.cpython-312.pyc Normal file
View File

Binary file not shown.

1
venv/lib/python3.12/site-packages/beautifulsoup4-4.14.3.dist-info/INSTALLER Normal file
View File

@@ -0,0 +1 @@
pip

123
venv/lib/python3.12/site-packages/beautifulsoup4-4.14.3.dist-info/METADATA Normal file
View File

@@ -0,0 +1,123 @@
Metadata-Version: 2.4
Name: beautifulsoup4
Version: 4.14.3
Summary: Screen-scraping library
Project-URL: Download, https://www.crummy.com/software/BeautifulSoup/bs4/download/
Project-URL: Homepage, https://www.crummy.com/software/BeautifulSoup/bs4/
Author-email: Leonard Richardson <leonardr@segfault.org>
License: MIT License
License-File: AUTHORS
License-File: LICENSE
Keywords: HTML,XML,parse,soup
Classifier: Development Status :: 5 - Production/Stable
Classifier: Intended Audience :: Developers
Classifier: License :: OSI Approved :: MIT License
Classifier: Programming Language :: Python
Classifier: Programming Language :: Python :: 3
Classifier: Topic :: Software Development :: Libraries :: Python Modules
Classifier: Topic :: Text Processing :: Markup :: HTML
Classifier: Topic :: Text Processing :: Markup :: SGML
Classifier: Topic :: Text Processing :: Markup :: XML
Requires-Python: >=3.7.0
Requires-Dist: soupsieve>=1.6.1
Requires-Dist: typing-extensions>=4.0.0
Provides-Extra: cchardet
Requires-Dist: cchardet; extra == 'cchardet'
Provides-Extra: chardet
Requires-Dist: chardet; extra == 'chardet'
Provides-Extra: charset-normalizer
Requires-Dist: charset-normalizer; extra == 'charset-normalizer'
Provides-Extra: html5lib
Requires-Dist: html5lib; extra == 'html5lib'
Provides-Extra: lxml
Requires-Dist: lxml; extra == 'lxml'
Description-Content-Type: text/markdown
Beautiful Soup is a library that makes it easy to scrape information
from web pages. It sits atop an HTML or XML parser, providing Pythonic
idioms for iterating, searching, and modifying the parse tree.
# Quick start
```
>>> from bs4 import BeautifulSoup
>>> soup = BeautifulSoup("<p>Some<b>bad<i>HTML")
>>> print(soup.prettify())
<html>
<body>
<p>
Some
<b>
bad
<i>
HTML
</i>
</b>
</p>
</body>
</html>
>>> soup.find(string="bad")
'bad'
>>> soup.i
<i>HTML</i>
#
>>> soup = BeautifulSoup("<tag1>Some<tag2/>bad<tag3>XML", "xml")
#
>>> print(soup.prettify())
<?xml version="1.0" encoding="utf-8"?>
<tag1>
Some
<tag2/>
bad
<tag3>
XML
</tag3>
</tag1>
```
To go beyond the basics, [comprehensive documentation is available](https://www.crummy.com/software/BeautifulSoup/bs4/doc/).
# Links
* [Homepage](https://www.crummy.com/software/BeautifulSoup/bs4/)
* [Documentation](https://www.crummy.com/software/BeautifulSoup/bs4/doc/)
* [Discussion group](https://groups.google.com/group/beautifulsoup/)
* [Development](https://code.launchpad.net/beautifulsoup/)
* [Bug tracker](https://bugs.launchpad.net/beautifulsoup/)
* [Complete changelog](https://git.launchpad.net/beautifulsoup/tree/CHANGELOG)
# Note on Python 2 sunsetting
Beautiful Soup's support for Python 2 was discontinued on December 31,
2020: one year after the sunset date for Python 2 itself. From this
point onward, new Beautiful Soup development will exclusively target
Python 3. The final release of Beautiful Soup 4 to support Python 2
was 4.9.3.
# Supporting the project
If you use Beautiful Soup as part of your professional work, please consider a
[Tidelift subscription](https://tidelift.com/subscription/pkg/pypi-beautifulsoup4?utm_source=pypi-beautifulsoup4&utm_medium=referral&utm_campaign=readme).
This will support many of the free software projects your organization
depends on, not just Beautiful Soup.
If you use Beautiful Soup for personal projects, the best way to say
thank you is to read
[Tool Safety](https://www.crummy.com/software/BeautifulSoup/zine/), a zine I
wrote about what Beautiful Soup has taught me about software
development.
# Building the documentation
The bs4/doc/ directory contains full documentation in Sphinx
format. Run `make html` in that directory to create HTML
documentation.
# Running the unit tests
Beautiful Soup supports unit test discovery using Pytest:
```
$ pytest
```

37
venv/lib/python3.12/site-packages/beautifulsoup4-4.14.3.dist-info/RECORD Normal file
View File

@@ -0,0 +1,37 @@
beautifulsoup4-4.14.3.dist-info/INSTALLER,sha256=zuuue4knoyJ-UwPPXg8fezS7VCrXJQrAP7zeNuwvFQg,4
beautifulsoup4-4.14.3.dist-info/METADATA,sha256=Ac93vA8Xp9FtgOcKXFM8ESfVdztimUfJ3WUpVlhKtsY,3812
beautifulsoup4-4.14.3.dist-info/RECORD,,
beautifulsoup4-4.14.3.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
beautifulsoup4-4.14.3.dist-info/licenses/AUTHORS,sha256=uYkjiRjh_aweRnF8tAW2PpJJeickE68NmJwd9siry28,2201
beautifulsoup4-4.14.3.dist-info/licenses/LICENSE,sha256=VbTY1LHlvIbRDvrJG3TIe8t3UmsPW57a-LnNKtxzl7I,1441
bs4/__init__.py,sha256=E7wiVp7oQK0JhdAYxpehZa8drv3W_sJv5oeTFiBfR5o,44386
bs4/__pycache__/__init__.cpython-312.pyc,,
bs4/__pycache__/_deprecation.cpython-312.pyc,,
bs4/__pycache__/_typing.cpython-312.pyc,,
bs4/__pycache__/_warnings.cpython-312.pyc,,
bs4/__pycache__/css.cpython-312.pyc,,
bs4/__pycache__/dammit.cpython-312.pyc,,
bs4/__pycache__/diagnose.cpython-312.pyc,,
bs4/__pycache__/element.cpython-312.pyc,,
bs4/__pycache__/exceptions.cpython-312.pyc,,
bs4/__pycache__/filter.cpython-312.pyc,,
bs4/__pycache__/formatter.cpython-312.pyc,,
bs4/_deprecation.py,sha256=niHJCk37APg8KEuFOa57ZXaxLdBmc_2V6uuaJqu7r30,2408
bs4/_typing.py,sha256=zNcx7R1yCTK8WwtumP28hc7CJ3pMyZXj_VAeYaNXMZA,7549
bs4/_warnings.py,sha256=ZuOETgcnEbZgw2N0nnNXn6wvtrn2ut7AF0d98bvkMFc,4711
bs4/builder/__init__.py,sha256=Rl4qjOXvdyyyjayOFqbkgoUoo81IgoyKD-RwWeVK59g,31194
bs4/builder/__pycache__/__init__.cpython-312.pyc,,
bs4/builder/__pycache__/_html5lib.cpython-312.pyc,,
bs4/builder/__pycache__/_htmlparser.cpython-312.pyc,,
bs4/builder/__pycache__/_lxml.cpython-312.pyc,,
bs4/builder/_html5lib.py,sha256=hL6xUk4_I2i5CMguFoYFlrI26cY4Dut7fOEQrUctHIM,23607
bs4/builder/_htmlparser.py,sha256=CnULPQV2rm4vLojJABpQ7Xm9diddnEZx2Wcz_VTC1Mg,17445
bs4/builder/_lxml.py,sha256=ks1e8boA_nOA2oomAhxeudccR6ThbEE-EllFqHRoPLA,18969
bs4/css.py,sha256=_m_l_4SGWHnY620VJ21j_qQH1RX3p91sYVemgKxaLsM,12713
bs4/dammit.py,sha256=ZJWa9K32X6N2imFHleqUq0ekf592weU1lvULN_WYWYk,57024
bs4/diagnose.py,sha256=at98iuxyOrqec4V8iwkTIbNUqBCsq9Lr3fDAQx2129Y,7846
bs4/element.py,sha256=oXmj7LG_2NpsDK90mq73q0PMK0FjFBIGSeTTJLVwwTc,120237
bs4/exceptions.py,sha256=Q9FOadNe8QRvzDMaKSXe2Wtl8JK_oAZW7mbFZBVP_GE,951
bs4/filter.py,sha256=rw8ZNhTDLEJVCEiSifou5tZR_3zBLeuvAyouY82qU_E,29201
bs4/formatter.py,sha256=uBT0k6W8O5kJ9PCuJYjra97yoUqC-dlM9D_v-oRM0r8,10478
bs4/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0

4
venv/lib/python3.12/site-packages/beautifulsoup4-4.14.3.dist-info/WHEEL Normal file
View File

@@ -0,0 +1,4 @@
Wheel-Version: 1.0
Generator: hatchling 1.27.0
Root-Is-Purelib: true
Tag: py3-none-any

49
venv/lib/python3.12/site-packages/beautifulsoup4-4.14.3.dist-info/licenses/AUTHORS Normal file
View File

@@ -0,0 +1,49 @@
Behold, mortal, the origins of Beautiful Soup...
================================================
Leonard Richardson is the primary maintainer.
Aaron DeVore, Isaac Muse and Chris Papademetrious have made
significant contributions to the code base.
Mark Pilgrim provided the encoding detection code that forms the base
of UnicodeDammit.
Thomas Kluyver and Ezio Melotti finished the work of getting Beautiful
Soup 4 working under Python 3.
Simon Willison wrote soupselect, which was used to make Beautiful Soup
support CSS selectors. Isaac Muse wrote SoupSieve, which made it
possible to _remove_ the CSS selector code from Beautiful Soup.
Sam Ruby helped with a lot of edge cases.
Jonathan Ellis was awarded the prestigious Beau Potage D'Or for his
work in solving the nestable tags conundrum.
An incomplete list of people have contributed patches to Beautiful
Soup:
Istvan Albert, Andrew Lin, Anthony Baxter, Oliver Beattie, Andrew
Boyko, Tony Chang, Francisco Canas, "Delong", Zephyr Fang, Fuzzy,
Roman Gaufman, Yoni Gilad, Richie Hindle, Toshihiro Kamiya, Peteris
Krumins, Kent Johnson, Marek Kapolka, Andreas Kostyrka, Roel Kramer,
Ben Last, Robert Leftwich, Stefaan Lippens, "liquider", Staffan
Malmgren, Ksenia Marasanova, JP Moins, Adam Monsen, John Nagle, "Jon",
Ed Oskiewicz, Martijn Peters, Greg Phillips, Giles Radford, Stefano
Revera, Arthur Rudolph, Marko Samastur, James Salter, Jouni Seppänen,
Alexander Schmolck, Tim Shirley, Geoffrey Sneddon, Ville Skyttä,
"Vikas", Jens Svalgaard, Andy Theyers, Eric Weiser, Glyn Webster, John
Wiseman, Paul Wright, Danny Yoo
An incomplete list of people who made suggestions or found bugs or
found ways to break Beautiful Soup:
Hanno Böck, Matteo Bertini, Chris Curvey, Simon Cusack, Bruce Eckel,
Matt Ernst, Michael Foord, Tom Harris, Bill de hOra, Donald Howes,
Matt Patterson, Scott Roberts, Steve Strassmann, Mike Williams,
warchild at redho dot com, Sami Kuisma, Carlos Rocha, Bob Hutchison,
Joren Mc, Michal Migurski, John Kleven, Tim Heaney, Tripp Lilley, Ed
Summers, Dennis Sutch, Chris Smith, Aaron Swartz, Stuart
Turner, Greg Edwards, Kevin J Kalupson, Nikos Kouremenos, Artur de
Sousa Rocha, Yichun Wei, Per Vognsen

31
venv/lib/python3.12/site-packages/beautifulsoup4-4.14.3.dist-info/licenses/LICENSE Normal file
View File

@@ -0,0 +1,31 @@
Beautiful Soup is made available under the MIT license:
Copyright (c) Leonard Richardson
Permission is hereby granted, free of charge, to any person obtaining
a copy of this software and associated documentation files (the
"Software"), to deal in the Software without restriction, including
without limitation the rights to use, copy, modify, merge, publish,
distribute, sublicense, and/or sell copies of the Software, and to
permit persons to whom the Software is furnished to do so, subject to
the following conditions:
The above copyright notice and this permission notice shall be
included in all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
Beautiful Soup incorporates code from the html5lib library, which is
also made available under the MIT license. Copyright (c) James Graham
and other contributors
Beautiful Soup has an optional dependency on the soupsieve library,
which is also made available under the MIT license. Copyright (c)
Isaac Muse

1
venv/lib/python3.12/site-packages/bs4-0.0.2.dist-info/INSTALLER Normal file
View File

@@ -0,0 +1 @@
pip

10
venv/lib/python3.12/site-packages/bs4-0.0.2.dist-info/METADATA Normal file
View File

@@ -0,0 +1,10 @@
Metadata-Version: 2.1
Name: bs4
Version: 0.0.2
Summary: Dummy package for Beautiful Soup (beautifulsoup4)
Author-email: Leonard Richardson <leonardr@segfault.org>
License: MIT License
Requires-Dist: beautifulsoup4
Description-Content-Type: text/x-rst
This is a dummy package designed to prevent namesquatting on PyPI. You should install `beautifulsoup4 <https://pypi.python.org/pypi/beautifulsoup4>`_ instead.

6
venv/lib/python3.12/site-packages/bs4-0.0.2.dist-info/RECORD Normal file
View File

@@ -0,0 +1,6 @@
README.rst,sha256=KMs4D-t40JC-oge8vGS3O5gueksurGqAIFxPtHZAMXQ,159
bs4-0.0.2.dist-info/INSTALLER,sha256=zuuue4knoyJ-UwPPXg8fezS7VCrXJQrAP7zeNuwvFQg,4
bs4-0.0.2.dist-info/METADATA,sha256=GEwOSFCOYLu11XQR3O2dMO7ZTpKFZpGoIUG0gkFVgA8,411
bs4-0.0.2.dist-info/RECORD,,
bs4-0.0.2.dist-info/REQUESTED,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
bs4-0.0.2.dist-info/WHEEL,sha256=VYAwk8D_V6zmIA2XKK-k7Fem_KAtVk3hugaRru3yjGc,105

0
venv/lib/python3.12/site-packages/bs4-0.0.2.dist-info/REQUESTED Normal file
View File

5
venv/lib/python3.12/site-packages/bs4-0.0.2.dist-info/WHEEL Normal file
View File

@@ -0,0 +1,5 @@
Wheel-Version: 1.0
Generator: hatchling 1.21.0
Root-Is-Purelib: true
Tag: py2-none-any
Tag: py3-none-any

1174
venv/lib/python3.12/site-packages/bs4/__init__.py Normal file
View File

File diff suppressed because it is too large Load Diff

BIN
venv/lib/python3.12/site-packages/bs4/__pycache__/__init__.cpython-312.pyc Normal file
View File

Binary file not shown.

BIN
venv/lib/python3.12/site-packages/bs4/__pycache__/_deprecation.cpython-312.pyc Normal file
View File

Binary file not shown.

BIN
venv/lib/python3.12/site-packages/bs4/__pycache__/_typing.cpython-312.pyc Normal file
View File

Binary file not shown.

BIN
venv/lib/python3.12/site-packages/bs4/__pycache__/_warnings.cpython-312.pyc Normal file
View File

Binary file not shown.

BIN
venv/lib/python3.12/site-packages/bs4/__pycache__/css.cpython-312.pyc Normal file
View File

Binary file not shown.

BIN
venv/lib/python3.12/site-packages/bs4/__pycache__/dammit.cpython-312.pyc Normal file
View File

Binary file not shown.

BIN
venv/lib/python3.12/site-packages/bs4/__pycache__/diagnose.cpython-312.pyc Normal file
View File

Binary file not shown.

BIN
venv/lib/python3.12/site-packages/bs4/__pycache__/element.cpython-312.pyc Normal file
View File

Binary file not shown.

BIN
venv/lib/python3.12/site-packages/bs4/__pycache__/exceptions.cpython-312.pyc Normal file
View File

Binary file not shown.

BIN
venv/lib/python3.12/site-packages/bs4/__pycache__/filter.cpython-312.pyc Normal file
View File

Binary file not shown.

BIN
venv/lib/python3.12/site-packages/bs4/__pycache__/formatter.cpython-312.pyc Normal file
View File

Binary file not shown.

80
venv/lib/python3.12/site-packages/bs4/_deprecation.py Normal file
View File

@@ -0,0 +1,80 @@
"""Helper functions for deprecation.
This interface is itself unstable and may change without warning. Do
not use these functions yourself, even as a joke. The underscores are
there for a reason. No support will be given.
In particular, most of this will go away without warning once
Beautiful Soup drops support for Python 3.11, since Python 3.12
defines a `@typing.deprecated()
decorator. <https://peps.python.org/pep-0702/>`_
"""
import functools
import warnings
from typing import (
Any,
Callable,
)
def _deprecated_alias(old_name: str, new_name: str, version: str):
"""Alias one attribute name to another for backward compatibility
:meta private:
"""
@property # type:ignore
def alias(self) -> Any:
":meta private:"
warnings.warn(
f"Access to deprecated property {old_name}. (Replaced by {new_name}) -- Deprecated since version {version}.",
DeprecationWarning,
stacklevel=2,
)
return getattr(self, new_name)
@alias.setter
def alias(self, value: str) -> None:
":meta private:"
warnings.warn(
f"Write to deprecated property {old_name}. (Replaced by {new_name}) -- Deprecated since version {version}.",
DeprecationWarning,
stacklevel=2,
)
return setattr(self, new_name, value)
return alias
def _deprecated_function_alias(
old_name: str, new_name: str, version: str
) -> Callable[[Any], Any]:
def alias(self, *args: Any, **kwargs: Any) -> Any:
":meta private:"
warnings.warn(
f"Call to deprecated method {old_name}. (Replaced by {new_name}) -- Deprecated since version {version}.",
DeprecationWarning,
stacklevel=2,
)
return getattr(self, new_name)(*args, **kwargs)
return alias
def _deprecated(replaced_by: str, version: str) -> Callable:
def deprecate(func: Callable) -> Callable:
@functools.wraps(func)
def with_warning(*args: Any, **kwargs: Any) -> Any:
":meta private:"
warnings.warn(
f"Call to deprecated method {func.__name__}. (Replaced by {replaced_by}) -- Deprecated since version {version}.",
DeprecationWarning,
stacklevel=2,
)
return func(*args, **kwargs)
return with_warning
return deprecate

205
venv/lib/python3.12/site-packages/bs4/_typing.py Normal file
View File

@@ -0,0 +1,205 @@
# Custom type aliases used throughout Beautiful Soup to improve readability.
# Notes on improvements to the type system in newer versions of Python
# that can be used once Beautiful Soup drops support for older
# versions:
#
# * ClassVar can be put on class variables now.
# * In 3.10, x|y is an accepted shorthand for Union[x,y].
# * In 3.10, TypeAlias gains capabilities that can be used to
# improve the tree matching types (I don't remember what, exactly).
# * In 3.9 it's possible to specialize the re.Match type,
# e.g. re.Match[str]. In 3.8 there's a typing.re namespace for this,
# but it's removed in 3.12, so to support the widest possible set of
# versions I'm not using it.
from typing_extensions import (
runtime_checkable,
Protocol,
TypeAlias,
)
from typing import (
Any,
Callable,
Dict,
IO,
Iterable,
Mapping,
Optional,
Pattern,
TYPE_CHECKING,
Union,
)
if TYPE_CHECKING:
from bs4.element import (
AttributeValueList,
NamespacedAttribute,
NavigableString,
PageElement,
ResultSet,
Tag,
)
@runtime_checkable
class _RegularExpressionProtocol(Protocol):
"""A protocol object which can accept either Python's built-in
`re.Pattern` objects, or the similar ``Regex`` objects defined by the
third-party ``regex`` package.
"""
def search(
self, string: str, pos: int = ..., endpos: int = ...
) -> Optional[Any]: ...
@property
def pattern(self) -> str: ...
# Aliases for markup in various stages of processing.
#
#: The rawest form of markup: either a string, bytestring, or an open filehandle.
_IncomingMarkup: TypeAlias = Union[str, bytes, IO[str], IO[bytes]]
#: Markup that is in memory but has (potentially) yet to be converted
#: to Unicode.
_RawMarkup: TypeAlias = Union[str, bytes]
# Aliases for character encodings
#
#: A data encoding.
_Encoding: TypeAlias = str
#: One or more data encodings.
_Encodings: TypeAlias = Iterable[_Encoding]
# Aliases for XML namespaces
#
#: The prefix for an XML namespace.
_NamespacePrefix: TypeAlias = str
#: The URL of an XML namespace
_NamespaceURL: TypeAlias = str
#: A mapping of prefixes to namespace URLs.
_NamespaceMapping: TypeAlias = Dict[_NamespacePrefix, _NamespaceURL]
#: A mapping of namespace URLs to prefixes
_InvertedNamespaceMapping: TypeAlias = Dict[_NamespaceURL, _NamespacePrefix]
# Aliases for the attribute values associated with HTML/XML tags.
#
#: The value associated with an HTML or XML attribute. This is the
#: relatively unprocessed value Beautiful Soup expects to come from a
#: `TreeBuilder`.
_RawAttributeValue: TypeAlias = str
#: A dictionary of names to `_RawAttributeValue` objects. This is how
#: Beautiful Soup expects a `TreeBuilder` to represent a tag's
#: attribute values.
_RawAttributeValues: TypeAlias = (
"Mapping[Union[str, NamespacedAttribute], _RawAttributeValue]"
)
#: An attribute value in its final form, as stored in the
# `Tag` class, after it has been processed and (in some cases)
# split into a list of strings.
_AttributeValue: TypeAlias = Union[str, "AttributeValueList"]
#: A dictionary of names to :py:data:`_AttributeValue` objects. This is what
#: a tag's attributes look like after processing.
_AttributeValues: TypeAlias = Dict[str, _AttributeValue]
#: The methods that deal with turning :py:data:`_RawAttributeValue` into
#: :py:data:`_AttributeValue` may be called several times, even after the values
#: are already processed (e.g. when cloning a tag), so they need to
#: be able to acommodate both possibilities.
_RawOrProcessedAttributeValues: TypeAlias = Union[_RawAttributeValues, _AttributeValues]
#: A number of tree manipulation methods can take either a `PageElement` or a
#: normal Python string (which will be converted to a `NavigableString`).
_InsertableElement: TypeAlias = Union["PageElement", str]
# Aliases to represent the many possibilities for matching bits of a
# parse tree.
#
# This is very complicated because we're applying a formal type system
# to some very DWIM code. The types we end up with will be the types
# of the arguments to the SoupStrainer constructor and (more
# familiarly to Beautiful Soup users) the find* methods.
#: A function that takes a PageElement and returns a yes-or-no answer.
_PageElementMatchFunction: TypeAlias = Callable[["PageElement"], bool]
#: A function that takes the raw parsed ingredients of a markup tag
#: and returns a yes-or-no answer.
# Not necessary at the moment.
# _AllowTagCreationFunction:TypeAlias = Callable[[Optional[str], str, Optional[_RawAttributeValues]], bool]
#: A function that takes the raw parsed ingredients of a markup string node
#: and returns a yes-or-no answer.
# Not necessary at the moment.
# _AllowStringCreationFunction:TypeAlias = Callable[[Optional[str]], bool]
#: A function that takes a `Tag` and returns a yes-or-no answer.
#: A `TagNameMatchRule` expects this kind of function, if you're
#: going to pass it a function.
_TagMatchFunction: TypeAlias = Callable[["Tag"], bool]
#: A function that takes a string (or None) and returns a yes-or-no
#: answer. An `AttributeValueMatchRule` expects this kind of function, if
#: you're going to pass it a function.
_NullableStringMatchFunction: TypeAlias = Callable[[Optional[str]], bool]
#: A function that takes a string and returns a yes-or-no answer. A
# `StringMatchRule` expects this kind of function, if you're going to
# pass it a function.
_StringMatchFunction: TypeAlias = Callable[[str], bool]
#: Either a tag name, an attribute value or a string can be matched
#: against a string, bytestring, regular expression, or a boolean.
_BaseStrainable: TypeAlias = Union[str, bytes, Pattern[str], bool]
#: A tag can be matched either with the `_BaseStrainable` options, or
#: using a function that takes the `Tag` as its sole argument.
_BaseStrainableElement: TypeAlias = Union[_BaseStrainable, _TagMatchFunction]
#: A tag's attribute value can be matched either with the
#: `_BaseStrainable` options, or using a function that takes that
#: value as its sole argument.
_BaseStrainableAttribute: TypeAlias = Union[_BaseStrainable, _NullableStringMatchFunction]
#: A tag can be matched using either a single criterion or a list of
#: criteria.
_StrainableElement: TypeAlias = Union[
_BaseStrainableElement, Iterable[_BaseStrainableElement]
]
#: An attribute value can be matched using either a single criterion
#: or a list of criteria.
_StrainableAttribute: TypeAlias = Union[
_BaseStrainableAttribute, Iterable[_BaseStrainableAttribute]
]
#: An string can be matched using the same techniques as
#: an attribute value.
_StrainableString: TypeAlias = _StrainableAttribute
#: A dictionary may be used to match against multiple attribute vlaues at once.
_StrainableAttributes: TypeAlias = Dict[str, _StrainableAttribute]
#: Many Beautiful soup methods return a PageElement or an ResultSet of
#: PageElements. A PageElement is either a Tag or a NavigableString.
#: These convenience aliases make it easier for IDE users to see which methods
#: are available on the objects they're dealing with.
_OneElement: TypeAlias = Union["PageElement", "Tag", "NavigableString"]
_AtMostOneElement: TypeAlias = Optional[_OneElement]
_AtMostOneTag: TypeAlias = Optional["Tag"]
_AtMostOneNavigableString: TypeAlias = Optional["NavigableString"]
_QueryResults: TypeAlias = "ResultSet[_OneElement]"
_SomeTags: TypeAlias = "ResultSet[Tag]"
_SomeNavigableStrings: TypeAlias = "ResultSet[NavigableString]"

98
venv/lib/python3.12/site-packages/bs4/_warnings.py Normal file
View File

@@ -0,0 +1,98 @@
"""Define some custom warnings."""
class GuessedAtParserWarning(UserWarning):
"""The warning issued when BeautifulSoup has to guess what parser to
use -- probably because no parser was specified in the constructor.
"""
MESSAGE: str = """No parser was explicitly specified, so I'm using the best available %(markup_type)s parser for this system ("%(parser)s"). This usually isn't a problem, but if you run this code on another system, or in a different virtual environment, it may use a different parser and behave differently.
The code that caused this warning is on line %(line_number)s of the file %(filename)s. To get rid of this warning, pass the additional argument 'features="%(parser)s"' to the BeautifulSoup constructor.
"""
class UnusualUsageWarning(UserWarning):
"""A superclass for warnings issued when Beautiful Soup sees
something that is typically the result of a mistake in the calling
code, but might be intentional on the part of the user. If it is
in fact intentional, you can filter the individual warning class
to get rid of the warning. If you don't like Beautiful Soup
second-guessing what you are doing, you can filter the
UnusualUsageWarningclass itself and get rid of these entirely.
"""
class MarkupResemblesLocatorWarning(UnusualUsageWarning):
"""The warning issued when BeautifulSoup is given 'markup' that
actually looks like a resource locator -- a URL or a path to a file
on disk.
"""
#: :meta private:
GENERIC_MESSAGE: str = """
However, if you want to parse some data that happens to look like a %(what)s, then nothing has gone wrong: you are using Beautiful Soup correctly, and this warning is spurious and can be filtered. To make this warning go away, run this code before calling the BeautifulSoup constructor:
from bs4 import MarkupResemblesLocatorWarning
import warnings
warnings.filterwarnings("ignore", category=MarkupResemblesLocatorWarning)
"""
URL_MESSAGE: str = (
"""The input passed in on this line looks more like a URL than HTML or XML.
If you meant to use Beautiful Soup to parse the web page found at a certain URL, then something has gone wrong. You should use an Python package like 'requests' to fetch the content behind the URL. Once you have the content as a string, you can feed that string into Beautiful Soup."""
+ GENERIC_MESSAGE
)
FILENAME_MESSAGE: str = (
"""The input passed in on this line looks more like a filename than HTML or XML.
If you meant to use Beautiful Soup to parse the contents of a file on disk, then something has gone wrong. You should open the file first, using code like this:
filehandle = open(your filename)
You can then feed the open filehandle into Beautiful Soup instead of using the filename."""
+ GENERIC_MESSAGE
)
class AttributeResemblesVariableWarning(UnusualUsageWarning, SyntaxWarning):
"""The warning issued when Beautiful Soup suspects a provided
attribute name may actually be the misspelled name of a Beautiful
Soup variable. Generally speaking, this is only used in cases like
"_class" where it's very unlikely the user would be referencing an
XML attribute with that name.
"""
MESSAGE: str = """%(original)r is an unusual attribute name and is a common misspelling for %(autocorrect)r.
If you meant %(autocorrect)r, change your code to use it, and this warning will go away.
If you really did mean to check the %(original)r attribute, this warning is spurious and can be filtered. To make it go away, run this code before creating your BeautifulSoup object:
from bs4 import AttributeResemblesVariableWarning
import warnings
warnings.filterwarnings("ignore", category=AttributeResemblesVariableWarning)
"""
class XMLParsedAsHTMLWarning(UnusualUsageWarning):
"""The warning issued when an HTML parser is used to parse
XML that is not (as far as we can tell) XHTML.
"""
MESSAGE: str = """It looks like you're using an HTML parser to parse an XML document.
Assuming this really is an XML document, what you're doing might work, but you should know that using an XML parser will be more reliable. To parse this document as XML, make sure you have the Python package 'lxml' installed, and pass the keyword argument `features="xml"` into the BeautifulSoup constructor.
If you want or need to use an HTML parser on this document, you can make this warning go away by filtering it. To do that, run this code before calling the BeautifulSoup constructor:
from bs4 import XMLParsedAsHTMLWarning
import warnings
warnings.filterwarnings("ignore", category=XMLParsedAsHTMLWarning)
"""

848
venv/lib/python3.12/site-packages/bs4/builder/__init__.py Normal file
View File

@@ -0,0 +1,848 @@
from __future__ import annotations
# Use of this source code is governed by the MIT license.
__license__ = "MIT"
from collections import defaultdict
import re
from types import ModuleType
from typing import (
Any,
cast,
Dict,
Iterable,
List,
Optional,
Pattern,
Set,
Tuple,
Type,
TYPE_CHECKING,
)
import warnings
import sys
from bs4.element import (
AttributeDict,
AttributeValueList,
CharsetMetaAttributeValue,
ContentMetaAttributeValue,
RubyParenthesisString,
RubyTextString,
Stylesheet,
Script,
TemplateString,
nonwhitespace_re,
)
# Exceptions were moved to their own module in 4.13. Import here for
# backwards compatibility.
from bs4.exceptions import ParserRejectedMarkup
from bs4._typing import (
_AttributeValues,
_RawAttributeValue,
)
from bs4._warnings import XMLParsedAsHTMLWarning
if TYPE_CHECKING:
from bs4 import BeautifulSoup
from bs4.element import (
NavigableString,
Tag,
)
from bs4._typing import (
_AttributeValue,
_Encoding,
_Encodings,
_RawOrProcessedAttributeValues,
_RawMarkup,
)
__all__ = [
"HTMLTreeBuilder",
"SAXTreeBuilder",
"TreeBuilder",
"TreeBuilderRegistry",
]
# Some useful features for a TreeBuilder to have.
FAST = "fast"
PERMISSIVE = "permissive"
STRICT = "strict"
XML = "xml"
HTML = "html"
HTML_5 = "html5"
__all__ = [
"TreeBuilderRegistry",
"TreeBuilder",
"HTMLTreeBuilder",
"DetectsXMLParsedAsHTML",
"ParserRejectedMarkup", # backwards compatibility only as of 4.13.0
]
class TreeBuilderRegistry(object):
"""A way of looking up TreeBuilder subclasses by their name or by desired
features.
"""
builders_for_feature: Dict[str, List[Type[TreeBuilder]]]
builders: List[Type[TreeBuilder]]
def __init__(self) -> None:
self.builders_for_feature = defaultdict(list)
self.builders = []
def register(self, treebuilder_class: type[TreeBuilder]) -> None:
"""Register a treebuilder based on its advertised features.
:param treebuilder_class: A subclass of `TreeBuilder`. its
`TreeBuilder.features` attribute should list its features.
"""
for feature in treebuilder_class.features:
self.builders_for_feature[feature].insert(0, treebuilder_class)
self.builders.insert(0, treebuilder_class)
def lookup(self, *features: str) -> Optional[Type[TreeBuilder]]:
"""Look up a TreeBuilder subclass with the desired features.
:param features: A list of features to look for. If none are
provided, the most recently registered TreeBuilder subclass
will be used.
:return: A TreeBuilder subclass, or None if there's no
registered subclass with all the requested features.
"""
if len(self.builders) == 0:
# There are no builders at all.
return None
if len(features) == 0:
# They didn't ask for any features. Give them the most
# recently registered builder.
return self.builders[0]
# Go down the list of features in order, and eliminate any builders
# that don't match every feature.
feature_list = list(features)
feature_list.reverse()
candidates = None
candidate_set = None
while len(feature_list) > 0:
feature = feature_list.pop()
we_have_the_feature = self.builders_for_feature.get(feature, [])
if len(we_have_the_feature) > 0:
if candidates is None:
candidates = we_have_the_feature
candidate_set = set(candidates)
elif candidate_set is not None:
# Eliminate any candidates that don't have this feature.
candidate_set = candidate_set.intersection(set(we_have_the_feature))
# The only valid candidates are the ones in candidate_set.
# Go through the original list of candidates and pick the first one
# that's in candidate_set.
if candidate_set is None or candidates is None:
return None
for candidate in candidates:
if candidate in candidate_set:
return candidate
return None
#: The `BeautifulSoup` constructor will take a list of features
#: and use it to look up `TreeBuilder` classes in this registry.
builder_registry: TreeBuilderRegistry = TreeBuilderRegistry()
class TreeBuilder(object):
"""Turn a textual document into a Beautiful Soup object tree.
This is an abstract superclass which smooths out the behavior of
different parser libraries into a single, unified interface.
:param multi_valued_attributes: If this is set to None, the
TreeBuilder will not turn any values for attributes like
'class' into lists. Setting this to a dictionary will
customize this behavior; look at :py:attr:`bs4.builder.HTMLTreeBuilder.DEFAULT_CDATA_LIST_ATTRIBUTES`
for an example.
Internally, these are called "CDATA list attributes", but that
probably doesn't make sense to an end-user, so the argument name
is ``multi_valued_attributes``.
:param preserve_whitespace_tags: A set of tags to treat
the way <pre> tags are treated in HTML. Tags in this set
are immune from pretty-printing; their contents will always be
output as-is.
:param string_containers: A dictionary mapping tag names to
the classes that should be instantiated to contain the textual
contents of those tags. The default is to use NavigableString
for every tag, no matter what the name. You can override the
default by changing :py:attr:`DEFAULT_STRING_CONTAINERS`.
:param store_line_numbers: If the parser keeps track of the line
numbers and positions of the original markup, that information
will, by default, be stored in each corresponding
:py:class:`bs4.element.Tag` object. You can turn this off by
passing store_line_numbers=False; then Tag.sourcepos and
Tag.sourceline will always be None. If the parser you're using
doesn't keep track of this information, then store_line_numbers
is irrelevant.
:param attribute_dict_class: The value of a multi-valued attribute
(such as HTML's 'class') willl be stored in an instance of this
class. The default is Beautiful Soup's built-in
`AttributeValueList`, which is a normal Python list, and you
will probably never need to change it.
"""
USE_DEFAULT: Any = object() #: :meta private:
def __init__(
self,
multi_valued_attributes: Dict[str, Set[str]] = USE_DEFAULT,
preserve_whitespace_tags: Set[str] = USE_DEFAULT,
store_line_numbers: bool = USE_DEFAULT,
string_containers: Dict[str, Type[NavigableString]] = USE_DEFAULT,
empty_element_tags: Set[str] = USE_DEFAULT,
attribute_dict_class: Type[AttributeDict] = AttributeDict,
attribute_value_list_class: Type[AttributeValueList] = AttributeValueList,
):
self.soup = None
if multi_valued_attributes is self.USE_DEFAULT:
multi_valued_attributes = self.DEFAULT_CDATA_LIST_ATTRIBUTES
self.cdata_list_attributes = multi_valued_attributes
if preserve_whitespace_tags is self.USE_DEFAULT:
preserve_whitespace_tags = self.DEFAULT_PRESERVE_WHITESPACE_TAGS
self.preserve_whitespace_tags = preserve_whitespace_tags
if empty_element_tags is self.USE_DEFAULT:
self.empty_element_tags = self.DEFAULT_EMPTY_ELEMENT_TAGS
else:
self.empty_element_tags = empty_element_tags
# TODO: store_line_numbers is probably irrelevant now that
# the behavior of sourceline and sourcepos has been made consistent
# everywhere.
if store_line_numbers == self.USE_DEFAULT:
store_line_numbers = self.TRACKS_LINE_NUMBERS
self.store_line_numbers = store_line_numbers
if string_containers == self.USE_DEFAULT:
string_containers = self.DEFAULT_STRING_CONTAINERS
self.string_containers = string_containers
self.attribute_dict_class = attribute_dict_class
self.attribute_value_list_class = attribute_value_list_class
NAME: str = "[Unknown tree builder]"
ALTERNATE_NAMES: Iterable[str] = []
features: Iterable[str] = []
is_xml: bool = False
picklable: bool = False
soup: Optional[BeautifulSoup] #: :meta private:
#: A tag will be considered an empty-element
#: tag when and only when it has no contents.
empty_element_tags: Optional[Set[str]] = None #: :meta private:
cdata_list_attributes: Dict[str, Set[str]] #: :meta private:
preserve_whitespace_tags: Set[str] #: :meta private:
string_containers: Dict[str, Type[NavigableString]] #: :meta private:
tracks_line_numbers: bool #: :meta private:
#: A value for these tag/attribute combinations is a space- or
#: comma-separated list of CDATA, rather than a single CDATA.
DEFAULT_CDATA_LIST_ATTRIBUTES: Dict[str, Set[str]] = defaultdict(set)
#: Whitespace should be preserved inside these tags.
DEFAULT_PRESERVE_WHITESPACE_TAGS: Set[str] = set()
#: The textual contents of tags with these names should be
#: instantiated with some class other than `bs4.element.NavigableString`.
DEFAULT_STRING_CONTAINERS: Dict[str, Type[bs4.element.NavigableString]] = {} # type:ignore
#: By default, tags are treated as empty-element tags if they have
#: no contents--that is, using XML rules. HTMLTreeBuilder
#: defines a different set of DEFAULT_EMPTY_ELEMENT_TAGS based on the
#: HTML 4 and HTML5 standards.
DEFAULT_EMPTY_ELEMENT_TAGS: Optional[Set[str]] = None
#: Most parsers don't keep track of line numbers.
TRACKS_LINE_NUMBERS: bool = False
def initialize_soup(self, soup: BeautifulSoup) -> None:
"""The BeautifulSoup object has been initialized and is now
being associated with the TreeBuilder.
:param soup: A BeautifulSoup object.
"""
self.soup = soup
def reset(self) -> None:
"""Do any work necessary to reset the underlying parser
for a new document.
By default, this does nothing.
"""
pass
def can_be_empty_element(self, tag_name: str) -> bool:
"""Might a tag with this name be an empty-element tag?
The final markup may or may not actually present this tag as
self-closing.
For instance: an HTMLBuilder does not consider a <p> tag to be
an empty-element tag (it's not in
HTMLBuilder.empty_element_tags). This means an empty <p> tag
will be presented as "<p></p>", not "<p/>" or "<p>".
The default implementation has no opinion about which tags are
empty-element tags, so a tag will be presented as an
empty-element tag if and only if it has no children.
"<foo></foo>" will become "<foo/>", and "<foo>bar</foo>" will
be left alone.
:param tag_name: The name of a markup tag.
"""
if self.empty_element_tags is None:
return True
return tag_name in self.empty_element_tags
def feed(self, markup: _RawMarkup) -> None:
"""Run incoming markup through some parsing process."""
raise NotImplementedError()
def prepare_markup(
self,
markup: _RawMarkup,
user_specified_encoding: Optional[_Encoding] = None,
document_declared_encoding: Optional[_Encoding] = None,
exclude_encodings: Optional[_Encodings] = None,
) -> Iterable[Tuple[_RawMarkup, Optional[_Encoding], Optional[_Encoding], bool]]:
"""Run any preliminary steps necessary to make incoming markup
acceptable to the parser.
:param markup: The markup that's about to be parsed.
:param user_specified_encoding: The user asked to try this encoding
to convert the markup into a Unicode string.
:param document_declared_encoding: The markup itself claims to be
in this encoding. NOTE: This argument is not used by the
calling code and can probably be removed.
:param exclude_encodings: The user asked *not* to try any of
these encodings.
:yield: A series of 4-tuples: (markup, encoding, declared encoding,
has undergone character replacement)
Each 4-tuple represents a strategy that the parser can try
to convert the document to Unicode and parse it. Each
strategy will be tried in turn.
By default, the only strategy is to parse the markup
as-is. See `LXMLTreeBuilderForXML` and
`HTMLParserTreeBuilder` for implementations that take into
account the quirks of particular parsers.
:meta private:
"""
yield markup, None, None, False
def test_fragment_to_document(self, fragment: str) -> str:
"""Wrap an HTML fragment to make it look like a document.
Different parsers do this differently. For instance, lxml
introduces an empty <head> tag, and html5lib
doesn't. Abstracting this away lets us write simple tests
which run HTML fragments through the parser and compare the
results against other HTML fragments.
This method should not be used outside of unit tests.
:param fragment: A fragment of HTML.
:return: A full HTML document.
:meta private:
"""
return fragment
def set_up_substitutions(self, tag: Tag) -> bool:
"""Set up any substitutions that will need to be performed on
a `Tag` when it's output as a string.
By default, this does nothing. See `HTMLTreeBuilder` for a
case where this is used.
:return: Whether or not a substitution was performed.
:meta private:
"""
return False
def _replace_cdata_list_attribute_values(
self, tag_name: str, attrs: _RawOrProcessedAttributeValues
) -> _AttributeValues:
"""When an attribute value is associated with a tag that can
have multiple values for that attribute, convert the string
value to a list of strings.
Basically, replaces class="foo bar" with class=["foo", "bar"]
NOTE: This method modifies its input in place.
:param tag_name: The name of a tag.
:param attrs: A dictionary containing the tag's attributes.
Any appropriate attribute values will be modified in place.
:return: The modified dictionary that was originally passed in.
"""
# First, cast the attrs dict to _AttributeValues. This might
# not be accurate yet, but it will be by the time this method
# returns.
modified_attrs = cast(_AttributeValues, attrs)
if not modified_attrs or not self.cdata_list_attributes:
# Nothing to do.
return modified_attrs
# There is at least a possibility that we need to modify one of
# the attribute values.
universal: Set[str] = self.cdata_list_attributes.get("*", set())
tag_specific = self.cdata_list_attributes.get(tag_name.lower(), None)
for attr in list(modified_attrs.keys()):
modified_value: _AttributeValue
if attr in universal or (tag_specific and attr in tag_specific):
# We have a "class"-type attribute whose string
# value is a whitespace-separated list of
# values. Split it into a list.
original_value: _AttributeValue = modified_attrs[attr]
if isinstance(original_value, _RawAttributeValue):
# This is a _RawAttributeValue (a string) that
# needs to be split and converted to a
# AttributeValueList so it can be an
# _AttributeValue.
modified_value = self.attribute_value_list_class(
nonwhitespace_re.findall(original_value)
)
else:
# html5lib calls setAttributes twice for the
# same tag when rearranging the parse tree. On
# the second call the attribute value here is
# already a list. This can also happen when a
# Tag object is cloned. If this happens, leave
# the value alone rather than trying to split
# it again.
modified_value = original_value
modified_attrs[attr] = modified_value
return modified_attrs
class SAXTreeBuilder(TreeBuilder):
"""A Beautiful Soup treebuilder that listens for SAX events.
This is not currently used for anything, and it will be removed
soon. It was a good idea, but it wasn't properly integrated into the
rest of Beautiful Soup, so there have been long stretches where it
hasn't worked properly.
"""
def __init__(self, *args: Any, **kwargs: Any) -> None:
warnings.warn(
"The SAXTreeBuilder class was deprecated in 4.13.0 and will be removed soon thereafter. It is completely untested and probably doesn't work; do not use it.",
DeprecationWarning,
stacklevel=2,
)
super(SAXTreeBuilder, self).__init__(*args, **kwargs)
def feed(self, markup: _RawMarkup) -> None:
raise NotImplementedError()
def close(self) -> None:
pass
def startElement(self, name: str, attrs: Dict[str, str]) -> None:
attrs = AttributeDict((key[1], value) for key, value in list(attrs.items()))
# print("Start %s, %r" % (name, attrs))
assert self.soup is not None
self.soup.handle_starttag(name, None, None, attrs)
def endElement(self, name: str) -> None:
# print("End %s" % name)
assert self.soup is not None
self.soup.handle_endtag(name)
def startElementNS(
self, nsTuple: Tuple[str, str], nodeName: str, attrs: Dict[str, str]
) -> None:
# Throw away (ns, nodeName) for now.
self.startElement(nodeName, attrs)
def endElementNS(self, nsTuple: Tuple[str, str], nodeName: str) -> None:
# Throw away (ns, nodeName) for now.
self.endElement(nodeName)
# handler.endElementNS((ns, node.nodeName), node.nodeName)
def startPrefixMapping(self, prefix: str, nodeValue: str) -> None:
# Ignore the prefix for now.
pass
def endPrefixMapping(self, prefix: str) -> None:
# Ignore the prefix for now.
# handler.endPrefixMapping(prefix)
pass
def characters(self, content: str) -> None:
assert self.soup is not None
self.soup.handle_data(content)
def startDocument(self) -> None:
pass
def endDocument(self) -> None:
pass
class HTMLTreeBuilder(TreeBuilder):
"""This TreeBuilder knows facts about HTML, such as which tags are treated
specially by the HTML standard.
"""
#: Some HTML tags are defined as having no contents. Beautiful Soup
#: treats these specially.
DEFAULT_EMPTY_ELEMENT_TAGS: Optional[Set[str]] = set(
[
# These are from HTML5.
"area",
"base",
"br",
"col",
"embed",
"hr",
"img",
"input",
"keygen",
"link",
"menuitem",
"meta",
"param",
"source",
"track",
"wbr",
# These are from earlier versions of HTML and are removed in HTML5.
"basefont",
"bgsound",
"command",
"frame",
"image",
"isindex",
"nextid",
"spacer",
]
)
#: The HTML standard defines these tags as block-level elements. Beautiful
#: Soup does not treat these elements differently from other elements,
#: but it may do so eventually, and this information is available if
#: you need to use it.
DEFAULT_BLOCK_ELEMENTS: Set[str] = set(
[
"address",
"article",
"aside",
"blockquote",
"canvas",
"dd",
"div",
"dl",
"dt",
"fieldset",
"figcaption",
"figure",
"footer",
"form",
"h1",
"h2",
"h3",
"h4",
"h5",
"h6",
"header",
"hr",
"li",
"main",
"nav",
"noscript",
"ol",
"output",
"p",
"pre",
"section",
"table",
"tfoot",
"ul",
"video",
]
)
#: These HTML tags need special treatment so they can be
#: represented by a string class other than `bs4.element.NavigableString`.
#:
#: For some of these tags, it's because the HTML standard defines
#: an unusual content model for them. I made this list by going
#: through the HTML spec
#: (https://html.spec.whatwg.org/#metadata-content) and looking for
#: "metadata content" elements that can contain strings.
#:
#: The Ruby tags (<rt> and <rp>) are here despite being normal
#: "phrasing content" tags, because the content they contain is
#: qualitatively different from other text in the document, and it
#: can be useful to be able to distinguish it.
#:
#: TODO: Arguably <noscript> could go here but it seems
#: qualitatively different from the other tags.
DEFAULT_STRING_CONTAINERS: Dict[str, Type[bs4.element.NavigableString]] = { # type:ignore
"rt": RubyTextString,
"rp": RubyParenthesisString,
"style": Stylesheet,
"script": Script,
"template": TemplateString,
}
#: The HTML standard defines these attributes as containing a
#: space-separated list of values, not a single value. That is,
#: class="foo bar" means that the 'class' attribute has two values,
#: 'foo' and 'bar', not the single value 'foo bar'. When we
#: encounter one of these attributes, we will parse its value into
#: a list of values if possible. Upon output, the list will be
#: converted back into a string.
DEFAULT_CDATA_LIST_ATTRIBUTES: Dict[str, Set[str]] = {
"*": {"class", "accesskey", "dropzone"},
"a": {"rel", "rev"},
"link": {"rel", "rev"},
"td": {"headers"},
"th": {"headers"},
"form": {"accept-charset"},
"object": {"archive"},
# These are HTML5 specific, as are *.accesskey and *.dropzone above.
"area": {"rel"},
"icon": {"sizes"},
"iframe": {"sandbox"},
"output": {"for"},
}
#: By default, whitespace inside these HTML tags will be
#: preserved rather than being collapsed.
DEFAULT_PRESERVE_WHITESPACE_TAGS: set[str] = set(["pre", "textarea"])
def set_up_substitutions(self, tag: Tag) -> bool:
"""Replace the declared encoding in a <meta> tag with a placeholder,
to be substituted when the tag is output to a string.
An HTML document may come in to Beautiful Soup as one
encoding, but exit in a different encoding, and the <meta> tag
needs to be changed to reflect this.
:return: Whether or not a substitution was performed.
:meta private:
"""
# We are only interested in <meta> tags
if tag.name != "meta":
return False
# TODO: This cast will fail in the (very unlikely) scenario
# that the programmer who instantiates the TreeBuilder
# specifies meta['content'] or meta['charset'] as
# cdata_list_attributes.
content: Optional[str] = cast(Optional[str], tag.get("content"))
charset: Optional[str] = cast(Optional[str], tag.get("charset"))
# But we can accommodate meta['http-equiv'] being made a
# cdata_list_attribute (again, very unlikely) without much
# trouble.
http_equiv: List[str] = tag.get_attribute_list("http-equiv")
# We are interested in <meta> tags that say what encoding the
# document was originally in. This means HTML 5-style <meta>
# tags that provide the "charset" attribute. It also means
# HTML 4-style <meta> tags that provide the "content"
# attribute and have "http-equiv" set to "content-type".
#
# In both cases we will replace the value of the appropriate
# attribute with a standin object that can take on any
# encoding.
substituted = False
if charset is not None:
# HTML 5 style:
# <meta charset="utf8">
tag["charset"] = CharsetMetaAttributeValue(charset)
substituted = True
elif content is not None and any(
x.lower() == "content-type" for x in http_equiv
):
# HTML 4 style:
# <meta http-equiv="content-type" content="text/html; charset=utf8">
tag["content"] = ContentMetaAttributeValue(content)
substituted = True
return substituted
class DetectsXMLParsedAsHTML(object):
"""A mixin class for any class (a TreeBuilder, or some class used by a
TreeBuilder) that's in a position to detect whether an XML
document is being incorrectly parsed as HTML, and issue an
appropriate warning.
This requires being able to observe an incoming processing
instruction that might be an XML declaration, and also able to
observe tags as they're opened. If you can't do that for a given
`TreeBuilder`, there's a less reliable implementation based on
examining the raw markup.
"""
#: Regular expression for seeing if string markup has an <html> tag.
LOOKS_LIKE_HTML: Pattern[str] = re.compile("<[^ +]html", re.I)
#: Regular expression for seeing if byte markup has an <html> tag.
LOOKS_LIKE_HTML_B: Pattern[bytes] = re.compile(b"<[^ +]html", re.I)
#: The start of an XML document string.
XML_PREFIX: str = "<?xml"
#: The start of an XML document bytestring.
XML_PREFIX_B: bytes = b"<?xml"
# This is typed as str, not `ProcessingInstruction`, because this
# check may be run before any Beautiful Soup objects are created.
_first_processing_instruction: Optional[str] #: :meta private:
_root_tag_name: Optional[str] #: :meta private:
@classmethod
def warn_if_markup_looks_like_xml(
cls, markup: Optional[_RawMarkup], stacklevel: int = 3
) -> bool:
"""Perform a check on some markup to see if it looks like XML
that's not XHTML. If so, issue a warning.
This is much less reliable than doing the check while parsing,
but some of the tree builders can't do that.
:param stacklevel: The stacklevel of the code calling this\
function.
:return: True if the markup looks like non-XHTML XML, False
otherwise.
"""
if markup is None:
return False
markup = markup[:500]
if isinstance(markup, bytes):
markup_b: bytes = markup
looks_like_xml = markup_b.startswith(
cls.XML_PREFIX_B
) and not cls.LOOKS_LIKE_HTML_B.search(markup)
else:
markup_s: str = markup
looks_like_xml = markup_s.startswith(
cls.XML_PREFIX
) and not cls.LOOKS_LIKE_HTML.search(markup)
if looks_like_xml:
cls._warn(stacklevel=stacklevel + 2)
return True
return False
@classmethod
def _warn(cls, stacklevel: int = 5) -> None:
"""Issue a warning about XML being parsed as HTML."""
warnings.warn(
XMLParsedAsHTMLWarning.MESSAGE,
XMLParsedAsHTMLWarning,
stacklevel=stacklevel,
)
def _initialize_xml_detector(self) -> None:
"""Call this method before parsing a document."""
self._first_processing_instruction = None
self._root_tag_name = None
def _document_might_be_xml(self, processing_instruction: str) -> None:
"""Call this method when encountering an XML declaration, or a
"processing instruction" that might be an XML declaration.
This helps Beautiful Soup detect potential issues later, if
the XML document turns out to be a non-XHTML document that's
being parsed as XML.
"""
if (
self._first_processing_instruction is not None
or self._root_tag_name is not None
):
# The document has already started. Don't bother checking
# anymore.
return
self._first_processing_instruction = processing_instruction
# We won't know until we encounter the first tag whether or
# not this is actually a problem.
def _root_tag_encountered(self, name: str) -> None:
"""Call this when you encounter the document's root tag.
This is where we actually check whether an XML document is
being incorrectly parsed as HTML, and issue the warning.
"""
if self._root_tag_name is not None:
# This method was incorrectly called multiple times. Do
# nothing.
return
self._root_tag_name = name
if (
name != "html"
and self._first_processing_instruction is not None
and self._first_processing_instruction.lower().startswith("xml ")
):
# We encountered an XML declaration and then a tag other
# than 'html'. This is a reliable indicator that a
# non-XHTML document is being parsed as XML.
self._warn(stacklevel=10)
def register_treebuilders_from(module: ModuleType) -> None:
"""Copy TreeBuilders from the given module into this module."""
this_module = sys.modules[__name__]
for name in module.__all__:
obj = getattr(module, name)
if issubclass(obj, TreeBuilder):
setattr(this_module, name, obj)
this_module.__all__.append(name)
# Register the builder while we're at it.
this_module.builder_registry.register(obj)
# Builders are registered in reverse order of priority, so that custom
# builder registrations will take precedence. In general, we want lxml
# to take precedence over html5lib, because it's faster. And we only
# want to use HTMLParser as a last resort.
from . import _htmlparser # noqa: E402
register_treebuilders_from(_htmlparser)
try:
from . import _html5lib
register_treebuilders_from(_html5lib)
except ImportError:
# They don't have html5lib installed.
pass
try:
from . import _lxml
register_treebuilders_from(_lxml)
except ImportError:
# They don't have lxml installed.
pass

BIN
venv/lib/python3.12/site-packages/bs4/builder/__pycache__/__init__.cpython-312.pyc Normal file
View File

Binary file not shown.

BIN
venv/lib/python3.12/site-packages/bs4/builder/__pycache__/_html5lib.cpython-312.pyc Normal file
View File

Binary file not shown.

BIN
venv/lib/python3.12/site-packages/bs4/builder/__pycache__/_htmlparser.cpython-312.pyc Normal file
View File

Binary file not shown.

BIN
venv/lib/python3.12/site-packages/bs4/builder/__pycache__/_lxml.cpython-312.pyc Normal file
View File

Binary file not shown.

611
venv/lib/python3.12/site-packages/bs4/builder/_html5lib.py Normal file
View File

@@ -0,0 +1,611 @@
# Use of this source code is governed by the MIT license.
__license__ = "MIT"
__all__ = [
"HTML5TreeBuilder",
]
from typing import (
Any,
cast,
Dict,
Iterable,
Optional,
Sequence,
TYPE_CHECKING,
Tuple,
Union,
)
from typing_extensions import TypeAlias
from bs4._typing import (
_AttributeValue,
_AttributeValues,
_Encoding,
_Encodings,
_NamespaceURL,
_RawMarkup,
)
import warnings
from bs4.builder import (
DetectsXMLParsedAsHTML,
PERMISSIVE,
HTML,
HTML_5,
HTMLTreeBuilder,
)
from bs4.element import (
NamespacedAttribute,
PageElement,
nonwhitespace_re,
)
import html5lib
from html5lib.constants import (
namespaces,
)
from bs4.element import (
Comment,
Doctype,
NavigableString,
Tag,
)
if TYPE_CHECKING:
from bs4 import BeautifulSoup
from html5lib.treebuilders import base as treebuilder_base
class HTML5TreeBuilder(HTMLTreeBuilder):
"""Use `html5lib <https://github.com/html5lib/html5lib-python>`_ to
build a tree.
Note that `HTML5TreeBuilder` does not support some common HTML
`TreeBuilder` features. Some of these features could theoretically
be implemented, but at the very least it's quite difficult,
because html5lib moves the parse tree around as it's being built.
Specifically:
* This `TreeBuilder` doesn't use different subclasses of
`NavigableString` (e.g. `Script`) based on the name of the tag
in which the string was found.
* You can't use a `SoupStrainer` to parse only part of a document.
"""
NAME: str = "html5lib"
features: Iterable[str] = [NAME, PERMISSIVE, HTML_5, HTML]
#: html5lib can tell us which line number and position in the
#: original file is the source of an element.
TRACKS_LINE_NUMBERS: bool = True
underlying_builder: "TreeBuilderForHtml5lib" #: :meta private:
user_specified_encoding: Optional[_Encoding]
def prepare_markup(
self,
markup: _RawMarkup,
user_specified_encoding: Optional[_Encoding] = None,
document_declared_encoding: Optional[_Encoding] = None,
exclude_encodings: Optional[_Encodings] = None,
) -> Iterable[Tuple[_RawMarkup, Optional[_Encoding], Optional[_Encoding], bool]]:
# Store the user-specified encoding for use later on.
self.user_specified_encoding = user_specified_encoding
# document_declared_encoding and exclude_encodings aren't used
# ATM because the html5lib TreeBuilder doesn't use
# UnicodeDammit.
for variable, name in (
(document_declared_encoding, "document_declared_encoding"),
(exclude_encodings, "exclude_encodings"),
):
if variable:
warnings.warn(
f"You provided a value for {name}, but the html5lib tree builder doesn't support {name}.",
stacklevel=3,
)
# html5lib only parses HTML, so if it's given XML that's worth
# noting.
DetectsXMLParsedAsHTML.warn_if_markup_looks_like_xml(markup, stacklevel=3)
yield (markup, None, None, False)
# These methods are defined by Beautiful Soup.
def feed(self, markup: _RawMarkup) -> None:
"""Run some incoming markup through some parsing process,
populating the `BeautifulSoup` object in `HTML5TreeBuilder.soup`.
"""
if self.soup is not None and self.soup.parse_only is not None:
warnings.warn(
"You provided a value for parse_only, but the html5lib tree builder doesn't support parse_only. The entire document will be parsed.",
stacklevel=4,
)
# self.underlying_builder is probably None now, but it'll be set
# when html5lib calls self.create_treebuilder().
parser = html5lib.HTMLParser(tree=self.create_treebuilder)
assert self.underlying_builder is not None
self.underlying_builder.parser = parser
extra_kwargs = dict()
if not isinstance(markup, str):
# kwargs, specifically override_encoding, will eventually
# be passed in to html5lib's
# HTMLBinaryInputStream.__init__.
extra_kwargs["override_encoding"] = self.user_specified_encoding
doc = parser.parse(markup, **extra_kwargs) # type:ignore
# Set the character encoding detected by the tokenizer.
if isinstance(markup, str):
# We need to special-case this because html5lib sets
# charEncoding to UTF-8 if it gets Unicode input.
doc.original_encoding = None
else:
original_encoding = parser.tokenizer.stream.charEncoding[0] # type:ignore
# The encoding is an html5lib Encoding object. We want to
# use a string for compatibility with other tree builders.
original_encoding = original_encoding.name
doc.original_encoding = original_encoding
self.underlying_builder.parser = None
def create_treebuilder(
self, namespaceHTMLElements: bool
) -> "TreeBuilderForHtml5lib":
"""Called by html5lib to instantiate the kind of class it
calls a 'TreeBuilder'.
:param namespaceHTMLElements: Whether or not to namespace HTML elements.
:meta private:
"""
self.underlying_builder = TreeBuilderForHtml5lib(
namespaceHTMLElements, self.soup, store_line_numbers=self.store_line_numbers
)
return self.underlying_builder
def test_fragment_to_document(self, fragment: str) -> str:
"""See `TreeBuilder`."""
return "<html><head></head><body>%s</body></html>" % fragment
class TreeBuilderForHtml5lib(treebuilder_base.TreeBuilder):
soup: "BeautifulSoup" #: :meta private:
parser: Optional[html5lib.HTMLParser] #: :meta private:
def __init__(
self,
namespaceHTMLElements: bool,
soup: Optional["BeautifulSoup"] = None,
store_line_numbers: bool = True,
**kwargs: Any,
):
if soup:
self.soup = soup
else:
warnings.warn(
"The optionality of the 'soup' argument to the TreeBuilderForHtml5lib constructor is deprecated as of Beautiful Soup 4.13.0: 'soup' is now required. If you can't pass in a BeautifulSoup object here, or you get this warning and it seems mysterious to you, please contact the Beautiful Soup developer team for possible un-deprecation.",
DeprecationWarning,
stacklevel=2,
)
from bs4 import BeautifulSoup
# TODO: Why is the parser 'html.parser' here? Using
# html5lib doesn't cause an infinite loop and is more
# accurate. Best to get rid of this entire section, I think.
self.soup = BeautifulSoup(
"", "html.parser", store_line_numbers=store_line_numbers, **kwargs
)
# TODO: What are **kwargs exactly? Should they be passed in
# here in addition to/instead of being passed to the BeautifulSoup
# constructor?
super(TreeBuilderForHtml5lib, self).__init__(namespaceHTMLElements)
# This will be set later to a real html5lib HTMLParser object,
# which we can use to track the current line number.
self.parser = None
self.store_line_numbers = store_line_numbers
def documentClass(self) -> "Element":
self.soup.reset()
return Element(self.soup, self.soup, None)
def insertDoctype(self, token: Dict[str, Any]) -> None:
name: str = cast(str, token["name"])
publicId: Optional[str] = cast(Optional[str], token["publicId"])
systemId: Optional[str] = cast(Optional[str], token["systemId"])
doctype = Doctype.for_name_and_ids(name, publicId, systemId)
self.soup.object_was_parsed(doctype)
def elementClass(self, name: str, namespace: str) -> "Element":
sourceline: Optional[int] = None
sourcepos: Optional[int] = None
if self.parser is not None and self.store_line_numbers:
# This represents the point immediately after the end of the
# tag. We don't know when the tag started, but we do know
# where it ended -- the character just before this one.
sourceline, sourcepos = self.parser.tokenizer.stream.position() # type:ignore
assert sourcepos is not None
sourcepos = sourcepos - 1
tag = self.soup.new_tag(
name, namespace, sourceline=sourceline, sourcepos=sourcepos
)
return Element(tag, self.soup, namespace)
def commentClass(self, data: str) -> "TextNode":
return TextNode(Comment(data), self.soup)
def fragmentClass(self) -> "Element":
"""This is only used by html5lib HTMLParser.parseFragment(),
which is never used by Beautiful Soup, only by the html5lib
unit tests. Since we don't currently hook into those tests,
the implementation is left blank.
"""
raise NotImplementedError()
def getFragment(self) -> "Element":
"""This is only used by the html5lib unit tests. Since we
don't currently hook into those tests, the implementation is
left blank.
"""
raise NotImplementedError()
def appendChild(self, node: "Element") -> None:
# TODO: This code is not covered by the BS4 tests, and
# apparently not triggered by the html5lib test suite either.
# But it doesn't seem test-specific and there are calls to it
# (or a method with the same name) all over html5lib, so I'm
# leaving the implementation in place rather than replacing it
# with NotImplementedError()
self.soup.append(node.element)
def getDocument(self) -> "BeautifulSoup":
return self.soup
def testSerializer(self, node: "Element") -> None:
"""This is only used by the html5lib unit tests. Since we
don't currently hook into those tests, the implementation is
left blank.
"""
raise NotImplementedError()
class AttrList(object):
"""Represents a Tag's attributes in a way compatible with html5lib."""
element: Tag
attrs: _AttributeValues
def __init__(self, element: Tag):
self.element = element
self.attrs = dict(self.element.attrs)
def __iter__(self) -> Iterable[Tuple[str, _AttributeValue]]:
return list(self.attrs.items()).__iter__()
def __setitem__(self, name: str, value: _AttributeValue) -> None:
# If this attribute is a multi-valued attribute for this element,
# turn its value into a list.
list_attr = self.element.cdata_list_attributes or {}
if name in list_attr.get("*", []) or (
self.element.name in list_attr
and name in list_attr.get(self.element.name, [])
):
# A node that is being cloned may have already undergone
# this procedure. Check for this and skip it.
if not isinstance(value, list):
assert isinstance(value, str)
value = self.element.attribute_value_list_class(
nonwhitespace_re.findall(value)
)
self.element[name] = value
def items(self) -> Iterable[Tuple[str, _AttributeValue]]:
return list(self.attrs.items())
def keys(self) -> Iterable[str]:
return list(self.attrs.keys())
def __len__(self) -> int:
return len(self.attrs)
def __getitem__(self, name: str) -> _AttributeValue:
return self.attrs[name]
def __contains__(self, name: str) -> bool:
return name in list(self.attrs.keys())
class BeautifulSoupNode(treebuilder_base.Node):
# A node can correspond to _either_ a Tag _or_ a NavigableString.
tag: Optional[Tag]
string: Optional[NavigableString]
soup: "BeautifulSoup"
namespace: Optional[_NamespaceURL]
@property
def element(self) -> PageElement:
assert self.tag is not None or self.string is not None
if self.tag is not None:
return self.tag
else:
assert self.string is not None
return self.string
@property
def nodeType(self) -> int:
"""Return the html5lib constant corresponding to the type of
the underlying DOM object.
NOTE: This property is only accessed by the html5lib test
suite, not by Beautiful Soup proper.
"""
raise NotImplementedError()
# TODO-TYPING: typeshed stubs are incorrect about this;
# cloneNode returns a new Node, not None.
def cloneNode(self) -> treebuilder_base.Node: # type:ignore
raise NotImplementedError()
class Element(BeautifulSoupNode):
namespace: Optional[_NamespaceURL]
def __init__(
self, element: Tag, soup: "BeautifulSoup", namespace: Optional[_NamespaceURL]
):
self.tag = element
self.string = None
self.soup = soup
self.namespace = namespace
treebuilder_base.Node.__init__(self, element.name)
def appendChild(self, node: "BeautifulSoupNode") -> None:
string_child: Optional[NavigableString] = None
child: PageElement
if type(node.string) is NavigableString:
# We check for NavigableString *only* because we want to avoid
# joining PreformattedStrings, such as Comments, with nearby strings.
string_child = child = node.string
else:
child = node.element
node.parent = self
if (
child is not None
and child.parent is not None
and not isinstance(child, str)
):
node.element.extract()
if (
string_child is not None
and self.tag is not None and self.tag.contents
and type(self.tag.contents[-1]) is NavigableString
):
# We are appending a string onto another string.
# TODO This has O(n^2) performance, for input like
# "a</a>a</a>a</a>..."
old_element = self.tag.contents[-1]
new_element = self.soup.new_string(old_element + string_child)
old_element.replace_with(new_element)
self.soup._most_recent_element = new_element
else:
if isinstance(node, str):
# Create a brand new NavigableString from this string.
child = self.soup.new_string(node)
# Tell Beautiful Soup to act as if it parsed this element
# immediately after the parent's last descendant. (Or
# immediately after the parent, if it has no children.)
if self.tag is not None and self.tag.contents:
most_recent_element = self.tag._last_descendant(False)
elif self.element.next_element is not None:
# Something from further ahead in the parse tree is
# being inserted into this earlier element. This is
# very annoying because it means an expensive search
# for the last element in the tree.
most_recent_element = self.soup._last_descendant()
else:
most_recent_element = self.element
self.soup.object_was_parsed(
child, parent=self.tag, most_recent_element=most_recent_element
)
def getAttributes(self) -> AttrList:
assert self.tag is not None
return AttrList(self.tag)
# An HTML5lib attribute name may either be a single string,
# or a tuple (namespace, name).
_Html5libAttributeName: TypeAlias = Union[str, Tuple[str, str]]
# Now we can define the type this method accepts as a dictionary
# mapping those attribute names to single string values.
_Html5libAttributes: TypeAlias = Dict[_Html5libAttributeName, str]
def setAttributes(self, attributes: Optional[_Html5libAttributes]) -> None:
assert self.tag is not None
if attributes is not None and len(attributes) > 0:
# Replace any namespaced attributes with
# NamespacedAttribute objects.
for name, value in list(attributes.items()):
if isinstance(name, tuple):
new_name = NamespacedAttribute(*name)
del attributes[name]
attributes[new_name] = value
# We can now cast attributes to the type of Dict
# used by Beautiful Soup.
normalized_attributes = cast(_AttributeValues, attributes)
# Values for tags like 'class' came in as single strings;
# replace them with lists of strings as appropriate.
self.soup.builder._replace_cdata_list_attribute_values(
self.name, normalized_attributes
)
# Then set the attributes on the Tag associated with this
# BeautifulSoupNode.
for name, value_or_values in list(normalized_attributes.items()):
self.tag[name] = value_or_values
# The attributes may contain variables that need substitution.
# Call set_up_substitutions manually.
#
# The Tag constructor called this method when the Tag was created,
# but we just set/changed the attributes, so call it again.
self.soup.builder.set_up_substitutions(self.tag)
attributes = property(getAttributes, setAttributes)
def insertText(
self, data: str, insertBefore: Optional["BeautifulSoupNode"] = None
) -> None:
text = TextNode(self.soup.new_string(data), self.soup)
if insertBefore:
self.insertBefore(text, insertBefore)
else:
self.appendChild(text)
def insertBefore(
self, node: "BeautifulSoupNode", refNode: "BeautifulSoupNode"
) -> None:
assert self.tag is not None
index = self.tag.index(refNode.element)
if (
type(node.element) is NavigableString
and self.tag.contents
and type(self.tag.contents[index - 1]) is NavigableString
):
# (See comments in appendChild)
old_node = self.tag.contents[index - 1]
assert type(old_node) is NavigableString
new_str = self.soup.new_string(old_node + node.element)
old_node.replace_with(new_str)
else:
self.tag.insert(index, node.element)
node.parent = self
def removeChild(self, node: "Element") -> None:
node.element.extract()
def reparentChildren(self, newParent: "Element") -> None:
"""Move all of this tag's children into another tag."""
# print("MOVE", self.element.contents)
# print("FROM", self.element)
# print("TO", new_parent.element)
element = self.tag
assert element is not None
new_parent_element = newParent.tag
assert new_parent_element is not None
# Determine what this tag's next_element will be once all the children
# are removed.
final_next_element = element.next_sibling
new_parents_last_descendant = new_parent_element._last_descendant(False, False)
if len(new_parent_element.contents) > 0:
# The new parent already contains children. We will be
# appending this tag's children to the end.
# We can make this assertion since we know new_parent has
# children.
assert new_parents_last_descendant is not None
new_parents_last_child = new_parent_element.contents[-1]
new_parents_last_descendant_next_element = (
new_parents_last_descendant.next_element
)
else:
# The new parent contains no children.
new_parents_last_child = None
new_parents_last_descendant_next_element = new_parent_element.next_element
to_append = element.contents
if len(to_append) > 0:
# Set the first child's previous_element and previous_sibling
# to elements within the new parent
first_child = to_append[0]
if new_parents_last_descendant is not None:
first_child.previous_element = new_parents_last_descendant
else:
first_child.previous_element = new_parent_element
first_child.previous_sibling = new_parents_last_child
if new_parents_last_descendant is not None:
new_parents_last_descendant.next_element = first_child
else:
new_parent_element.next_element = first_child
if new_parents_last_child is not None:
new_parents_last_child.next_sibling = first_child
# Find the very last element being moved. It is now the
# parent's last descendant. It has no .next_sibling and
# its .next_element is whatever the previous last
# descendant had.
last_childs_last_descendant = to_append[-1]._last_descendant(
is_initialized=False, accept_self=True
)
# Since we passed accept_self=True into _last_descendant,
# there's no possibility that the result is None.
assert last_childs_last_descendant is not None
last_childs_last_descendant.next_element = (
new_parents_last_descendant_next_element
)
if new_parents_last_descendant_next_element is not None:
# TODO-COVERAGE: This code has no test coverage and
# I'm not sure how to get html5lib to go through this
# path, but it's just the other side of the previous
# line.
new_parents_last_descendant_next_element.previous_element = (
last_childs_last_descendant
)
last_childs_last_descendant.next_sibling = None
for child in to_append:
child.parent = new_parent_element
new_parent_element.contents.append(child)
# Now that this element has no children, change its .next_element.
element.contents = []
element.next_element = final_next_element
# print("DONE WITH MOVE")
# print("FROM", self.element)
# print("TO", new_parent_element)
# TODO-TYPING: typeshed stubs are incorrect about this;
# hasContent returns a boolean, not None.
def hasContent(self) -> bool: # type:ignore
return self.tag is None or len(self.tag.contents) > 0
# TODO-TYPING: typeshed stubs are incorrect about this;
# cloneNode returns a new Node, not None.
def cloneNode(self) -> treebuilder_base.Node: # type:ignore
assert self.tag is not None
tag = self.soup.new_tag(self.tag.name, self.namespace)
node = Element(tag, self.soup, self.namespace)
for key, value in self.attributes:
node.attributes[key] = value
return node
def getNameTuple(self) -> Tuple[Optional[_NamespaceURL], str]:
if self.namespace is None:
return namespaces["html"], self.name
else:
return self.namespace, self.name
nameTuple = property(getNameTuple)
class TextNode(BeautifulSoupNode):
def __init__(self, element: NavigableString, soup: "BeautifulSoup"):
treebuilder_base.Node.__init__(self, None)
self.tag = None
self.string = element
self.soup = soup

462
venv/lib/python3.12/site-packages/bs4/builder/_htmlparser.py Normal file
View File

@@ -0,0 +1,462 @@
# encoding: utf-8
"""Use the HTMLParser library to parse HTML files that aren't too bad."""
from __future__ import annotations
# Use of this source code is governed by the MIT license.
__license__ = "MIT"
__all__ = [
"HTMLParserTreeBuilder",
]
from html.parser import HTMLParser
from typing import (
Any,
Callable,
cast,
Dict,
Iterable,
List,
Optional,
TYPE_CHECKING,
Tuple,
Type,
Union,
)
from bs4.element import (
AttributeDict,
CData,
Comment,
Declaration,
Doctype,
ProcessingInstruction,
)
from bs4.dammit import EntitySubstitution, UnicodeDammit
from bs4.builder import (
DetectsXMLParsedAsHTML,
HTML,
HTMLTreeBuilder,
STRICT,
)
from bs4.exceptions import ParserRejectedMarkup
if TYPE_CHECKING:
from bs4 import BeautifulSoup
from bs4.element import NavigableString
from bs4._typing import (
_Encoding,
_Encodings,
_RawMarkup,
)
HTMLPARSER = "html.parser"
_DuplicateAttributeHandler = Callable[[Dict[str, str], str, str], None]
class BeautifulSoupHTMLParser(HTMLParser, DetectsXMLParsedAsHTML):
#: Constant to handle duplicate attributes by ignoring later values
#: and keeping the earlier ones.
REPLACE: str = "replace"
#: Constant to handle duplicate attributes by replacing earlier values
#: with later ones.
IGNORE: str = "ignore"
"""A subclass of the Python standard library's HTMLParser class, which
listens for HTMLParser events and translates them into calls
to Beautiful Soup's tree construction API.
:param on_duplicate_attribute: A strategy for what to do if a
tag includes the same attribute more than once. Accepted
values are: REPLACE (replace earlier values with later
ones, the default), IGNORE (keep the earliest value
encountered), or a callable. A callable must take three
arguments: the dictionary of attributes already processed,
the name of the duplicate attribute, and the most recent value
encountered.
"""
def __init__(
self,
soup: BeautifulSoup,
*args: Any,
on_duplicate_attribute: Union[str, _DuplicateAttributeHandler] = REPLACE,
**kwargs: Any,
):
self.soup = soup
self.on_duplicate_attribute = on_duplicate_attribute
self.attribute_dict_class = soup.builder.attribute_dict_class
HTMLParser.__init__(self, *args, **kwargs)
# Keep a list of empty-element tags that were encountered
# without an explicit closing tag. If we encounter a closing tag
# of this type, we'll associate it with one of those entries.
#
# This isn't a stack because we don't care about the
# order. It's a list of closing tags we've already handled and
# will ignore, assuming they ever show up.
self.already_closed_empty_element = []
self._initialize_xml_detector()
on_duplicate_attribute: Union[str, _DuplicateAttributeHandler]
already_closed_empty_element: List[str]
soup: BeautifulSoup
def error(self, message: str) -> None:
# NOTE: This method is required so long as Python 3.9 is
# supported. The corresponding code is removed from HTMLParser
# in 3.5, but not removed from ParserBase until 3.10.
# https://github.com/python/cpython/issues/76025
#
# The original implementation turned the error into a warning,
# but in every case I discovered, this made HTMLParser
# immediately crash with an error message that was less
# helpful than the warning. The new implementation makes it
# more clear that html.parser just can't parse this
# markup. The 3.10 implementation does the same, though it
# raises AssertionError rather than calling a method. (We
# catch this error and wrap it in a ParserRejectedMarkup.)
raise ParserRejectedMarkup(message)
def handle_startendtag(
self, tag: str, attrs: List[Tuple[str, Optional[str]]]
) -> None:
"""Handle an incoming empty-element tag.
html.parser only calls this method when the markup looks like
<tag/>.
"""
# `handle_empty_element` tells handle_starttag not to close the tag
# just because its name matches a known empty-element tag. We
# know that this is an empty-element tag, and we want to call
# handle_endtag ourselves.
self.handle_starttag(tag, attrs, handle_empty_element=False)
self.handle_endtag(tag)
def handle_starttag(
self,
tag: str,
attrs: List[Tuple[str, Optional[str]]],
handle_empty_element: bool = True,
) -> None:
"""Handle an opening tag, e.g. '<tag>'
:param handle_empty_element: True if this tag is known to be
an empty-element tag (i.e. there is not expected to be any
closing tag).
"""
# TODO: handle namespaces here?
attr_dict: AttributeDict = self.attribute_dict_class()
for key, value in attrs:
# Change None attribute values to the empty string
# for consistency with the other tree builders.
if value is None:
value = ""
if key in attr_dict:
# A single attribute shows up multiple times in this
# tag. How to handle it depends on the
# on_duplicate_attribute setting.
on_dupe = self.on_duplicate_attribute
if on_dupe == self.IGNORE:
pass
elif on_dupe in (None, self.REPLACE):
attr_dict[key] = value
else:
on_dupe = cast(_DuplicateAttributeHandler, on_dupe)
on_dupe(attr_dict, key, value)
else:
attr_dict[key] = value
# print("START", tag)
sourceline: Optional[int]
sourcepos: Optional[int]
if self.soup.builder.store_line_numbers:
sourceline, sourcepos = self.getpos()
else:
sourceline = sourcepos = None
tagObj = self.soup.handle_starttag(
tag, None, None, attr_dict, sourceline=sourceline, sourcepos=sourcepos
)
if tagObj is not None and tagObj.is_empty_element and handle_empty_element:
# Unlike other parsers, html.parser doesn't send separate end tag
# events for empty-element tags. (It's handled in
# handle_startendtag, but only if the original markup looked like
# <tag/>.)
#
# So we need to call handle_endtag() ourselves. Since we
# know the start event is identical to the end event, we
# don't want handle_endtag() to cross off any previous end
# events for tags of this name.
self.handle_endtag(tag, check_already_closed=False)
# But we might encounter an explicit closing tag for this tag
# later on. If so, we want to ignore it.
self.already_closed_empty_element.append(tag)
if self._root_tag_name is None:
self._root_tag_encountered(tag)
def handle_endtag(self, tag: str, check_already_closed: bool = True) -> None:
"""Handle a closing tag, e.g. '</tag>'
:param tag: A tag name.
:param check_already_closed: True if this tag is expected to
be the closing portion of an empty-element tag,
e.g. '<tag></tag>'.
"""
# print("END", tag)
if check_already_closed and tag in self.already_closed_empty_element:
# This is a redundant end tag for an empty-element tag.
# We've already called handle_endtag() for it, so just
# check it off the list.
# print("ALREADY CLOSED", tag)
self.already_closed_empty_element.remove(tag)
else:
self.soup.handle_endtag(tag)
def handle_data(self, data: str) -> None:
"""Handle some textual data that shows up between tags."""
self.soup.handle_data(data)
def handle_charref(self, name: str) -> None:
"""Handle a numeric character reference by converting it to the
corresponding Unicode character and treating it as textual
data.
:param name: Character number, possibly in hexadecimal.
"""
# TODO: This was originally a workaround for a bug in
# HTMLParser. (http://bugs.python.org/issue13633) The bug has
# been fixed, but removing this code still makes some
# Beautiful Soup tests fail. This needs investigation.
real_name:int
if name.startswith("x"):
real_name = int(name.lstrip("x"), 16)
elif name.startswith("X"):
real_name = int(name.lstrip("X"), 16)
else:
real_name = int(name)
data, replacement_added = UnicodeDammit.numeric_character_reference(real_name)
if replacement_added:
self.soup.contains_replacement_characters = True
self.handle_data(data)
def handle_entityref(self, name: str) -> None:
"""Handle a named entity reference by converting it to the
corresponding Unicode character(s) and treating it as textual
data.
:param name: Name of the entity reference.
"""
character = EntitySubstitution.HTML_ENTITY_TO_CHARACTER.get(name)
if character is not None:
data = character
else:
# If this were XML, it would be ambiguous whether "&foo"
# was an character entity reference with a missing
# semicolon or the literal string "&foo". Since this is
# HTML, we have a complete list of all character entity references,
# and this one wasn't found, so assume it's the literal string "&foo".
data = "&%s" % name
self.handle_data(data)
def handle_comment(self, data: str) -> None:
"""Handle an HTML comment.
:param data: The text of the comment.
"""
self.soup.endData()
self.soup.handle_data(data)
self.soup.endData(Comment)
def handle_decl(self, decl: str) -> None:
"""Handle a DOCTYPE declaration.
:param data: The text of the declaration.
"""
self.soup.endData()
decl = decl[len("DOCTYPE ") :]
self.soup.handle_data(decl)
self.soup.endData(Doctype)
def unknown_decl(self, data: str) -> None:
"""Handle a declaration of unknown type -- probably a CDATA block.
:param data: The text of the declaration.
"""
cls: Type[NavigableString]
if data.upper().startswith("CDATA["):
cls = CData
data = data[len("CDATA[") :]
else:
cls = Declaration
self.soup.endData()
self.soup.handle_data(data)
self.soup.endData(cls)
def handle_pi(self, data: str) -> None:
"""Handle a processing instruction.
:param data: The text of the instruction.
"""
self.soup.endData()
self.soup.handle_data(data)
self._document_might_be_xml(data)
self.soup.endData(ProcessingInstruction)
class HTMLParserTreeBuilder(HTMLTreeBuilder):
"""A Beautiful soup `bs4.builder.TreeBuilder` that uses the
:py:class:`html.parser.HTMLParser` parser, found in the Python
standard library.
"""
is_xml: bool = False
picklable: bool = True
NAME: str = HTMLPARSER
features: Iterable[str] = [NAME, HTML, STRICT]
parser_args: Tuple[Iterable[Any], Dict[str, Any]]
#: The html.parser knows which line number and position in the
#: original file is the source of an element.
TRACKS_LINE_NUMBERS: bool = True
def __init__(
self,
parser_args: Optional[Iterable[Any]] = None,
parser_kwargs: Optional[Dict[str, Any]] = None,
**kwargs: Any,
):
"""Constructor.
:param parser_args: Positional arguments to pass into
the BeautifulSoupHTMLParser constructor, once it's
invoked.
:param parser_kwargs: Keyword arguments to pass into
the BeautifulSoupHTMLParser constructor, once it's
invoked.
:param kwargs: Keyword arguments for the superclass constructor.
"""
# Some keyword arguments will be pulled out of kwargs and placed
# into parser_kwargs.
extra_parser_kwargs = dict()
for arg in ("on_duplicate_attribute",):
if arg in kwargs:
value = kwargs.pop(arg)
extra_parser_kwargs[arg] = value
super(HTMLParserTreeBuilder, self).__init__(**kwargs)
parser_args = parser_args or []
parser_kwargs = parser_kwargs or {}
parser_kwargs.update(extra_parser_kwargs)
parser_kwargs["convert_charrefs"] = False
self.parser_args = (parser_args, parser_kwargs)
def prepare_markup(
self,
markup: _RawMarkup,
user_specified_encoding: Optional[_Encoding] = None,
document_declared_encoding: Optional[_Encoding] = None,
exclude_encodings: Optional[_Encodings] = None,
) -> Iterable[Tuple[str, Optional[_Encoding], Optional[_Encoding], bool]]:
"""Run any preliminary steps necessary to make incoming markup
acceptable to the parser.
:param markup: Some markup -- probably a bytestring.
:param user_specified_encoding: The user asked to try this encoding.
:param document_declared_encoding: The markup itself claims to be
in this encoding.
:param exclude_encodings: The user asked _not_ to try any of
these encodings.
:yield: A series of 4-tuples: (markup, encoding, declared encoding,
has undergone character replacement)
Each 4-tuple represents a strategy for parsing the document.
This TreeBuilder uses Unicode, Dammit to convert the markup
into Unicode, so the ``markup`` element of the tuple will
always be a string.
"""
if isinstance(markup, str):
# Parse Unicode as-is.
yield (markup, None, None, False)
return
# Ask UnicodeDammit to sniff the most likely encoding.
known_definite_encodings: List[_Encoding] = []
if user_specified_encoding:
# This was provided by the end-user; treat it as a known
# definite encoding per the algorithm laid out in the
# HTML5 spec. (See the EncodingDetector class for
# details.)
known_definite_encodings.append(user_specified_encoding)
user_encodings: List[_Encoding] = []
if document_declared_encoding:
# This was found in the document; treat it as a slightly
# lower-priority user encoding.
user_encodings.append(document_declared_encoding)
dammit = UnicodeDammit(
markup,
known_definite_encodings=known_definite_encodings,
user_encodings=user_encodings,
is_html=True,
exclude_encodings=exclude_encodings,
)
if dammit.unicode_markup is None:
# In every case I've seen, Unicode, Dammit is able to
# convert the markup into Unicode, even if it needs to use
# REPLACEMENT CHARACTER. But there is a code path that
# could result in unicode_markup being None, and
# HTMLParser can only parse Unicode, so here we handle
# that code path.
raise ParserRejectedMarkup(
"Could not convert input to Unicode, and html.parser will not accept bytestrings."
)
else:
yield (
dammit.unicode_markup,
dammit.original_encoding,
dammit.declared_html_encoding,
dammit.contains_replacement_characters,
)
def feed(self, markup: _RawMarkup, _parser_class:type[BeautifulSoupHTMLParser] =BeautifulSoupHTMLParser) -> None:
"""
:param markup: The markup to feed into the parser.
:param _parser_class: An HTMLParser subclass to use. This is only intended for use in unit tests.
"""
args, kwargs = self.parser_args
# HTMLParser.feed will only handle str, but
# BeautifulSoup.markup is allowed to be _RawMarkup, because
# it's set by the yield value of
# TreeBuilder.prepare_markup. Fortunately,
# HTMLParserTreeBuilder.prepare_markup always yields a str
# (UnicodeDammit.unicode_markup).
assert isinstance(markup, str)
# We know BeautifulSoup calls TreeBuilder.initialize_soup
# before calling feed(), so we can assume self.soup
# is set.
assert self.soup is not None
parser = _parser_class(self.soup, *args, **kwargs)
try:
parser.feed(markup)
parser.close()
except AssertionError as e:
# html.parser raises AssertionError in rare cases to
# indicate a fatal problem with the markup, especially
# when there's an error in the doctype declaration.
raise ParserRejectedMarkup(e)
parser.already_closed_empty_element = []

501
venv/lib/python3.12/site-packages/bs4/builder/_lxml.py Normal file
View File

@@ -0,0 +1,501 @@
# encoding: utf-8
from __future__ import annotations
# Use of this source code is governed by the MIT license.
__license__ = "MIT"
__all__ = [
"LXMLTreeBuilderForXML",
"LXMLTreeBuilder",
]
from typing import (
Any,
Dict,
Iterable,
List,
Optional,
Set,
Tuple,
Type,
TYPE_CHECKING,
Union,
)
from io import BytesIO
from io import StringIO
from typing_extensions import TypeAlias
from lxml import etree # type:ignore
from bs4.element import (
AttributeDict,
XMLAttributeDict,
Comment,
Doctype,
NamespacedAttribute,
ProcessingInstruction,
XMLProcessingInstruction,
)
from bs4.builder import (
DetectsXMLParsedAsHTML,
FAST,
HTML,
HTMLTreeBuilder,
PERMISSIVE,
TreeBuilder,
XML,
)
from bs4.dammit import EncodingDetector
from bs4.exceptions import ParserRejectedMarkup
if TYPE_CHECKING:
from bs4._typing import (
_Encoding,
_Encodings,
_NamespacePrefix,
_NamespaceURL,
_NamespaceMapping,
_InvertedNamespaceMapping,
_RawMarkup,
)
from bs4 import BeautifulSoup
LXML: str = "lxml"
def _invert(d: dict[Any, Any]) -> dict[Any, Any]:
"Invert a dictionary."
return dict((v, k) for k, v in list(d.items()))
_LXMLParser: TypeAlias = Union[etree.XMLParser, etree.HTMLParser]
_ParserOrParserClass: TypeAlias = Union[
_LXMLParser, Type[etree.XMLParser], Type[etree.HTMLParser]
]
class LXMLTreeBuilderForXML(TreeBuilder):
DEFAULT_PARSER_CLASS: Type[etree.XMLParser] = etree.XMLParser
is_xml: bool = True
#: Set this to true (probably by passing huge_tree=True into the :
#: BeautifulSoup constructor) to enable the lxml feature "disable security
#: restrictions and support very deep trees and very long text
#: content".
huge_tree: bool
processing_instruction_class: Type[ProcessingInstruction]
NAME: str = "lxml-xml"
ALTERNATE_NAMES: Iterable[str] = ["xml"]
# Well, it's permissive by XML parser standards.
features: Iterable[str] = [NAME, LXML, XML, FAST, PERMISSIVE]
CHUNK_SIZE: int = 512
# This namespace mapping is specified in the XML Namespace
# standard.
DEFAULT_NSMAPS: _NamespaceMapping = dict(xml="http://www.w3.org/XML/1998/namespace")
DEFAULT_NSMAPS_INVERTED: _InvertedNamespaceMapping = _invert(DEFAULT_NSMAPS)
nsmaps: List[Optional[_InvertedNamespaceMapping]]
empty_element_tags: Optional[Set[str]]
parser: Any
_default_parser: Optional[etree.XMLParser]
# NOTE: If we parsed Element objects and looked at .sourceline,
# we'd be able to see the line numbers from the original document.
# But instead we build an XMLParser or HTMLParser object to serve
# as the target of parse messages, and those messages don't include
# line numbers.
# See: https://bugs.launchpad.net/lxml/+bug/1846906
def initialize_soup(self, soup: BeautifulSoup) -> None:
"""Let the BeautifulSoup object know about the standard namespace
mapping.
:param soup: A `BeautifulSoup`.
"""
# Beyond this point, self.soup is set, so we can assume (and
# assert) it's not None whenever necessary.
super(LXMLTreeBuilderForXML, self).initialize_soup(soup)
self._register_namespaces(self.DEFAULT_NSMAPS)
def _register_namespaces(self, mapping: Dict[str, str]) -> None:
"""Let the BeautifulSoup object know about namespaces encountered
while parsing the document.
This might be useful later on when creating CSS selectors.
This will track (almost) all namespaces, even ones that were
only in scope for part of the document. If two namespaces have
the same prefix, only the first one encountered will be
tracked. Un-prefixed namespaces are not tracked.
:param mapping: A dictionary mapping namespace prefixes to URIs.
"""
assert self.soup is not None
for key, value in list(mapping.items()):
# This is 'if key' and not 'if key is not None' because we
# don't track un-prefixed namespaces. Soupselect will
# treat an un-prefixed namespace as the default, which
# causes confusion in some cases.
if key and key not in self.soup._namespaces:
# Let the BeautifulSoup object know about a new namespace.
# If there are multiple namespaces defined with the same
# prefix, the first one in the document takes precedence.
self.soup._namespaces[key] = value
def default_parser(self, encoding: Optional[_Encoding]) -> _ParserOrParserClass:
"""Find the default parser for the given encoding.
:return: Either a parser object or a class, which
will be instantiated with default arguments.
"""
if self._default_parser is not None:
return self._default_parser
return self.DEFAULT_PARSER_CLASS(target=self, recover=True, huge_tree=self.huge_tree, encoding=encoding)
def parser_for(self, encoding: Optional[_Encoding]) -> _LXMLParser:
"""Instantiate an appropriate parser for the given encoding.
:param encoding: A string.
:return: A parser object such as an `etree.XMLParser`.
"""
# Use the default parser.
parser = self.default_parser(encoding)
if callable(parser):
# Instantiate the parser with default arguments
parser = parser(target=self, recover=True, huge_tree=self.huge_tree, encoding=encoding)
return parser
def __init__(
self,
parser: Optional[etree.XMLParser] = None,
empty_element_tags: Optional[Set[str]] = None,
huge_tree: bool = False,
**kwargs: Any,
):
# TODO: Issue a warning if parser is present but not a
# callable, since that means there's no way to create new
# parsers for different encodings.
self._default_parser = parser
self.soup = None
self.nsmaps = [self.DEFAULT_NSMAPS_INVERTED]
self.active_namespace_prefixes = [dict(self.DEFAULT_NSMAPS)]
if self.is_xml:
self.processing_instruction_class = XMLProcessingInstruction
else:
self.processing_instruction_class = ProcessingInstruction
if "attribute_dict_class" not in kwargs:
kwargs["attribute_dict_class"] = XMLAttributeDict
self.huge_tree = huge_tree
super(LXMLTreeBuilderForXML, self).__init__(**kwargs)
def _getNsTag(self, tag: str) -> Tuple[Optional[str], str]:
# Split the namespace URL out of a fully-qualified lxml tag
# name. Copied from lxml's src/lxml/sax.py.
if tag[0] == "{" and "}" in tag:
namespace, name = tag[1:].split("}", 1)
return (namespace, name)
return (None, tag)
def prepare_markup(
self,
markup: _RawMarkup,
user_specified_encoding: Optional[_Encoding] = None,
document_declared_encoding: Optional[_Encoding] = None,
exclude_encodings: Optional[_Encodings] = None,
) -> Iterable[
Tuple[Union[str, bytes], Optional[_Encoding], Optional[_Encoding], bool]
]:
"""Run any preliminary steps necessary to make incoming markup
acceptable to the parser.
lxml really wants to get a bytestring and convert it to
Unicode itself. So instead of using UnicodeDammit to convert
the bytestring to Unicode using different encodings, this
implementation uses EncodingDetector to iterate over the
encodings, and tell lxml to try to parse the document as each
one in turn.
:param markup: Some markup -- hopefully a bytestring.
:param user_specified_encoding: The user asked to try this encoding.
:param document_declared_encoding: The markup itself claims to be
in this encoding.
:param exclude_encodings: The user asked _not_ to try any of
these encodings.
:yield: A series of 4-tuples: (markup, encoding, declared encoding,
has undergone character replacement)
Each 4-tuple represents a strategy for converting the
document to Unicode and parsing it. Each strategy will be tried
in turn.
"""
if not self.is_xml:
# We're in HTML mode, so if we're given XML, that's worth
# noting.
DetectsXMLParsedAsHTML.warn_if_markup_looks_like_xml(markup, stacklevel=3)
if isinstance(markup, str):
# We were given Unicode. Maybe lxml can parse Unicode on
# this system?
# TODO: This is a workaround for
# https://bugs.launchpad.net/lxml/+bug/1948551.
# We can remove it once the upstream issue is fixed.
if len(markup) > 0 and markup[0] == "\N{BYTE ORDER MARK}":
markup = markup[1:]
yield markup, None, document_declared_encoding, False
if isinstance(markup, str):
# No, apparently not. Convert the Unicode to UTF-8 and
# tell lxml to parse it as UTF-8.
yield (markup.encode("utf8"), "utf8", document_declared_encoding, False)
# Since the document was Unicode in the first place, there
# is no need to try any more strategies; we know this will
# work.
return
known_definite_encodings: List[_Encoding] = []
if user_specified_encoding:
# This was provided by the end-user; treat it as a known
# definite encoding per the algorithm laid out in the
# HTML5 spec. (See the EncodingDetector class for
# details.)
known_definite_encodings.append(user_specified_encoding)
user_encodings: List[_Encoding] = []
if document_declared_encoding:
# This was found in the document; treat it as a slightly
# lower-priority user encoding.
user_encodings.append(document_declared_encoding)
detector = EncodingDetector(
markup,
known_definite_encodings=known_definite_encodings,
user_encodings=user_encodings,
is_html=not self.is_xml,
exclude_encodings=exclude_encodings,
)
for encoding in detector.encodings:
yield (detector.markup, encoding, document_declared_encoding, False)
def feed(self, markup: _RawMarkup) -> None:
io: Union[BytesIO, StringIO]
if isinstance(markup, bytes):
io = BytesIO(markup)
elif isinstance(markup, str):
io = StringIO(markup)
# initialize_soup is called before feed, so we know this
# is not None.
assert self.soup is not None
# Call feed() at least once, even if the markup is empty,
# or the parser won't be initialized.
data = io.read(self.CHUNK_SIZE)
try:
self.parser = self.parser_for(self.soup.original_encoding)
self.parser.feed(data)
while len(data) != 0:
# Now call feed() on the rest of the data, chunk by chunk.
data = io.read(self.CHUNK_SIZE)
if len(data) != 0:
self.parser.feed(data)
self.parser.close()
except (UnicodeDecodeError, LookupError, etree.ParserError) as e:
raise ParserRejectedMarkup(e)
def close(self) -> None:
self.nsmaps = [self.DEFAULT_NSMAPS_INVERTED]
def start(
self,
tag: str | bytes,
attrib: Dict[str | bytes, str | bytes],
nsmap: _NamespaceMapping = {},
) -> None:
# This is called by lxml code as a result of calling
# BeautifulSoup.feed(), and we know self.soup is set by the time feed()
# is called.
assert self.soup is not None
assert isinstance(tag, str)
# We need to recreate the attribute dict for three
# reasons. First, for type checking, so we can assert there
# are no bytestrings in the keys or values. Second, because we
# need a mutable dict--lxml might send us an immutable
# dictproxy. Third, so we can handle namespaced attribute
# names by converting the keys to NamespacedAttributes.
new_attrib: Dict[Union[str, NamespacedAttribute], str] = (
self.attribute_dict_class()
)
for k, v in attrib.items():
assert isinstance(k, str)
assert isinstance(v, str)
new_attrib[k] = v
nsprefix: Optional[_NamespacePrefix] = None
namespace: Optional[_NamespaceURL] = None
# Invert each namespace map as it comes in.
if len(nsmap) == 0 and len(self.nsmaps) > 1:
# There are no new namespaces for this tag, but
# non-default namespaces are in play, so we need a
# separate tag stack to know when they end.
self.nsmaps.append(None)
elif len(nsmap) > 0:
# A new namespace mapping has come into play.
# First, Let the BeautifulSoup object know about it.
self._register_namespaces(nsmap)
# Then, add it to our running list of inverted namespace
# mappings.
self.nsmaps.append(_invert(nsmap))
# The currently active namespace prefixes have
# changed. Calculate the new mapping so it can be stored
# with all Tag objects created while these prefixes are in
# scope.
current_mapping = dict(self.active_namespace_prefixes[-1])
current_mapping.update(nsmap)
# We should not track un-prefixed namespaces as we can only hold one
# and it will be recognized as the default namespace by soupsieve,
# which may be confusing in some situations.
if "" in current_mapping:
del current_mapping[""]
self.active_namespace_prefixes.append(current_mapping)
# Also treat the namespace mapping as a set of attributes on the
# tag, so we can recreate it later.
for prefix, namespace in list(nsmap.items()):
attribute = NamespacedAttribute(
"xmlns", prefix, "http://www.w3.org/2000/xmlns/"
)
new_attrib[attribute] = namespace
# Namespaces are in play. Find any attributes that came in
# from lxml with namespaces attached to their names, and
# turn then into NamespacedAttribute objects.
final_attrib: AttributeDict = self.attribute_dict_class()
for attr, value in list(new_attrib.items()):
namespace, attr = self._getNsTag(attr)
if namespace is None:
final_attrib[attr] = value
else:
nsprefix = self._prefix_for_namespace(namespace)
attr = NamespacedAttribute(nsprefix, attr, namespace)
final_attrib[attr] = value
namespace, tag = self._getNsTag(tag)
nsprefix = self._prefix_for_namespace(namespace)
self.soup.handle_starttag(
tag,
namespace,
nsprefix,
final_attrib,
namespaces=self.active_namespace_prefixes[-1],
)
def _prefix_for_namespace(
self, namespace: Optional[_NamespaceURL]
) -> Optional[_NamespacePrefix]:
"""Find the currently active prefix for the given namespace."""
if namespace is None:
return None
for inverted_nsmap in reversed(self.nsmaps):
if inverted_nsmap is not None and namespace in inverted_nsmap:
return inverted_nsmap[namespace]
return None
def end(self, tag: str | bytes) -> None:
assert self.soup is not None
assert isinstance(tag, str)
self.soup.endData()
namespace, tag = self._getNsTag(tag)
nsprefix = None
if namespace is not None:
for inverted_nsmap in reversed(self.nsmaps):
if inverted_nsmap is not None and namespace in inverted_nsmap:
nsprefix = inverted_nsmap[namespace]
break
self.soup.handle_endtag(tag, nsprefix)
if len(self.nsmaps) > 1:
# This tag, or one of its parents, introduced a namespace
# mapping, so pop it off the stack.
out_of_scope_nsmap = self.nsmaps.pop()
if out_of_scope_nsmap is not None:
# This tag introduced a namespace mapping which is no
# longer in scope. Recalculate the currently active
# namespace prefixes.
self.active_namespace_prefixes.pop()
def pi(self, target: str, data: str) -> None:
assert self.soup is not None
self.soup.endData()
data = target + " " + data
self.soup.handle_data(data)
self.soup.endData(self.processing_instruction_class)
def data(self, data: str | bytes) -> None:
assert self.soup is not None
assert isinstance(data, str)
self.soup.handle_data(data)
def doctype(self, name: str, pubid: str, system: str) -> None:
assert self.soup is not None
self.soup.endData()
doctype_string = Doctype._string_for_name_and_ids(name, pubid, system)
self.soup.handle_data(doctype_string)
self.soup.endData(containerClass=Doctype)
def comment(self, text: str | bytes) -> None:
"Handle comments as Comment objects."
assert self.soup is not None
assert isinstance(text, str)
self.soup.endData()
self.soup.handle_data(text)
self.soup.endData(Comment)
def test_fragment_to_document(self, fragment: str) -> str:
"""See `TreeBuilder`."""
return '<?xml version="1.0" encoding="utf-8"?>\n%s' % fragment
class LXMLTreeBuilder(HTMLTreeBuilder, LXMLTreeBuilderForXML):
NAME: str = LXML
ALTERNATE_NAMES: Iterable[str] = ["lxml-html"]
features: Iterable[str] = list(ALTERNATE_NAMES) + [NAME, HTML, FAST, PERMISSIVE]
is_xml: bool = False
def default_parser(self, encoding: Optional[_Encoding]) -> _ParserOrParserClass:
return etree.HTMLParser
def feed(self, markup: _RawMarkup) -> None:
# We know self.soup is set by the time feed() is called.
assert self.soup is not None
encoding = self.soup.original_encoding
try:
self.parser = self.parser_for(encoding)
self.parser.feed(markup)
self.parser.close()
except (UnicodeDecodeError, LookupError, etree.ParserError) as e:
raise ParserRejectedMarkup(e)
def test_fragment_to_document(self, fragment: str) -> str:
"""See `TreeBuilder`."""
return "<html><body>%s</body></html>" % fragment

339
venv/lib/python3.12/site-packages/bs4/css.py Normal file
View File

@@ -0,0 +1,339 @@
"""Integration code for CSS selectors using `Soup Sieve <https://facelessuser.github.io/soupsieve/>`_ (pypi: ``soupsieve``).
Acquire a `CSS` object through the `element.Tag.css` attribute of
the starting point of your CSS selector, or (if you want to run a
selector against the entire document) of the `BeautifulSoup` object
itself.
The main advantage of doing this instead of using ``soupsieve``
functions is that you don't need to keep passing the `element.Tag` to be
selected against, since the `CSS` object is permanently scoped to that
`element.Tag`.
"""
from __future__ import annotations
from types import ModuleType
from typing import (
Any,
cast,
Iterable,
Iterator,
MutableSequence,
Optional,
TYPE_CHECKING,
)
import warnings
from bs4._typing import _NamespaceMapping
if TYPE_CHECKING:
from soupsieve import SoupSieve
from bs4 import element
from bs4.element import ResultSet, Tag
soupsieve: Optional[ModuleType]
try:
import soupsieve
except ImportError:
soupsieve = None
warnings.warn(
"The soupsieve package is not installed. CSS selectors cannot be used."
)
class CSS(object):
"""A proxy object against the ``soupsieve`` library, to simplify its
CSS selector API.
You don't need to instantiate this class yourself; instead, use
`element.Tag.css`.
:param tag: All CSS selectors run by this object will use this as
their starting point.
:param api: An optional drop-in replacement for the ``soupsieve`` module,
intended for use in unit tests.
"""
def __init__(self, tag: element.Tag, api: Optional[ModuleType] = None):
if api is None:
api = soupsieve
if api is None:
raise NotImplementedError(
"Cannot execute CSS selectors because the soupsieve package is not installed."
)
self.api = api
self.tag = tag
def escape(self, ident: str) -> str:
"""Escape a CSS identifier.
This is a simple wrapper around `soupsieve.escape() <https://facelessuser.github.io/soupsieve/api/#soupsieveescape>`_. See the
documentation for that function for more information.
"""
if soupsieve is None:
raise NotImplementedError(
"Cannot escape CSS identifiers because the soupsieve package is not installed."
)
return cast(str, self.api.escape(ident))
def _ns(
self, ns: Optional[_NamespaceMapping], select: str
) -> Optional[_NamespaceMapping]:
"""Normalize a dictionary of namespaces."""
if not isinstance(select, self.api.SoupSieve) and ns is None:
# If the selector is a precompiled pattern, it already has
# a namespace context compiled in, which cannot be
# replaced.
ns = self.tag._namespaces
return ns
def _rs(self, results: MutableSequence[Tag]) -> ResultSet[Tag]:
"""Normalize a list of results to a py:class:`ResultSet`.
A py:class:`ResultSet` is more consistent with the rest of
Beautiful Soup's API, and :py:meth:`ResultSet.__getattr__` has
a helpful error message if you try to treat a list of results
as a single result (a common mistake).
"""
# Import here to avoid circular import
from bs4 import ResultSet
return ResultSet(None, results)
def compile(
self,
select: str,
namespaces: Optional[_NamespaceMapping] = None,
flags: int = 0,
**kwargs: Any,
) -> SoupSieve:
"""Pre-compile a selector and return the compiled object.
:param selector: A CSS selector.
:param namespaces: A dictionary mapping namespace prefixes
used in the CSS selector to namespace URIs. By default,
Beautiful Soup will use the prefixes it encountered while
parsing the document.
:param flags: Flags to be passed into Soup Sieve's
`soupsieve.compile() <https://facelessuser.github.io/soupsieve/api/#soupsievecompile>`_ method.
:param kwargs: Keyword arguments to be passed into Soup Sieve's
`soupsieve.compile() <https://facelessuser.github.io/soupsieve/api/#soupsievecompile>`_ method.
:return: A precompiled selector object.
:rtype: soupsieve.SoupSieve
"""
return self.api.compile(select, self._ns(namespaces, select), flags, **kwargs)
def select_one(
self,
select: str,
namespaces: Optional[_NamespaceMapping] = None,
flags: int = 0,
**kwargs: Any,
) -> element.Tag | None:
"""Perform a CSS selection operation on the current Tag and return the
first result, if any.
This uses the Soup Sieve library. For more information, see
that library's documentation for the `soupsieve.select_one() <https://facelessuser.github.io/soupsieve/api/#soupsieveselect_one>`_ method.
:param selector: A CSS selector.
:param namespaces: A dictionary mapping namespace prefixes
used in the CSS selector to namespace URIs. By default,
Beautiful Soup will use the prefixes it encountered while
parsing the document.
:param flags: Flags to be passed into Soup Sieve's
`soupsieve.select_one() <https://facelessuser.github.io/soupsieve/api/#soupsieveselect_one>`_ method.
:param kwargs: Keyword arguments to be passed into Soup Sieve's
`soupsieve.select_one() <https://facelessuser.github.io/soupsieve/api/#soupsieveselect_one>`_ method.
"""
return self.api.select_one(
select, self.tag, self._ns(namespaces, select), flags, **kwargs
)
def select(
self,
select: str,
namespaces: Optional[_NamespaceMapping] = None,
limit: int = 0,
flags: int = 0,
**kwargs: Any,
) -> ResultSet[element.Tag]:
"""Perform a CSS selection operation on the current `element.Tag`.
This uses the Soup Sieve library. For more information, see
that library's documentation for the `soupsieve.select() <https://facelessuser.github.io/soupsieve/api/#soupsieveselect>`_ method.
:param selector: A CSS selector.
:param namespaces: A dictionary mapping namespace prefixes
used in the CSS selector to namespace URIs. By default,
Beautiful Soup will pass in the prefixes it encountered while
parsing the document.
:param limit: After finding this number of results, stop looking.
:param flags: Flags to be passed into Soup Sieve's
`soupsieve.select() <https://facelessuser.github.io/soupsieve/api/#soupsieveselect>`_ method.
:param kwargs: Keyword arguments to be passed into Soup Sieve's
`soupsieve.select() <https://facelessuser.github.io/soupsieve/api/#soupsieveselect>`_ method.
"""
if limit is None:
limit = 0
return self._rs(
self.api.select(
select, self.tag, self._ns(namespaces, select), limit, flags, **kwargs
)
)
def iselect(
self,
select: str,
namespaces: Optional[_NamespaceMapping] = None,
limit: int = 0,
flags: int = 0,
**kwargs: Any,
) -> Iterator[element.Tag]:
"""Perform a CSS selection operation on the current `element.Tag`.
This uses the Soup Sieve library. For more information, see
that library's documentation for the `soupsieve.iselect()
<https://facelessuser.github.io/soupsieve/api/#soupsieveiselect>`_
method. It is the same as select(), but it returns a generator
instead of a list.
:param selector: A string containing a CSS selector.
:param namespaces: A dictionary mapping namespace prefixes
used in the CSS selector to namespace URIs. By default,
Beautiful Soup will pass in the prefixes it encountered while
parsing the document.
:param limit: After finding this number of results, stop looking.
:param flags: Flags to be passed into Soup Sieve's
`soupsieve.iselect() <https://facelessuser.github.io/soupsieve/api/#soupsieveiselect>`_ method.
:param kwargs: Keyword arguments to be passed into Soup Sieve's
`soupsieve.iselect() <https://facelessuser.github.io/soupsieve/api/#soupsieveiselect>`_ method.
"""
return self.api.iselect(
select, self.tag, self._ns(namespaces, select), limit, flags, **kwargs
)
def closest(
self,
select: str,
namespaces: Optional[_NamespaceMapping] = None,
flags: int = 0,
**kwargs: Any,
) -> Optional[element.Tag]:
"""Find the `element.Tag` closest to this one that matches the given selector.
This uses the Soup Sieve library. For more information, see
that library's documentation for the `soupsieve.closest()
<https://facelessuser.github.io/soupsieve/api/#soupsieveclosest>`_
method.
:param selector: A string containing a CSS selector.
:param namespaces: A dictionary mapping namespace prefixes
used in the CSS selector to namespace URIs. By default,
Beautiful Soup will pass in the prefixes it encountered while
parsing the document.
:param flags: Flags to be passed into Soup Sieve's
`soupsieve.closest() <https://facelessuser.github.io/soupsieve/api/#soupsieveclosest>`_ method.
:param kwargs: Keyword arguments to be passed into Soup Sieve's
`soupsieve.closest() <https://facelessuser.github.io/soupsieve/api/#soupsieveclosest>`_ method.
"""
return self.api.closest(
select, self.tag, self._ns(namespaces, select), flags, **kwargs
)
def match(
self,
select: str,
namespaces: Optional[_NamespaceMapping] = None,
flags: int = 0,
**kwargs: Any,
) -> bool:
"""Check whether or not this `element.Tag` matches the given CSS selector.
This uses the Soup Sieve library. For more information, see
that library's documentation for the `soupsieve.match()
<https://facelessuser.github.io/soupsieve/api/#soupsievematch>`_
method.
:param: a CSS selector.
:param namespaces: A dictionary mapping namespace prefixes
used in the CSS selector to namespace URIs. By default,
Beautiful Soup will pass in the prefixes it encountered while
parsing the document.
:param flags: Flags to be passed into Soup Sieve's
`soupsieve.match()
<https://facelessuser.github.io/soupsieve/api/#soupsievematch>`_
method.
:param kwargs: Keyword arguments to be passed into SoupSieve's
`soupsieve.match()
<https://facelessuser.github.io/soupsieve/api/#soupsievematch>`_
method.
"""
return cast(
bool,
self.api.match(
select, self.tag, self._ns(namespaces, select), flags, **kwargs
),
)
def filter(
self,
select: str,
namespaces: Optional[_NamespaceMapping] = None,
flags: int = 0,
**kwargs: Any,
) -> ResultSet[element.Tag]:
"""Filter this `element.Tag`'s direct children based on the given CSS selector.
This uses the Soup Sieve library. It works the same way as
passing a `element.Tag` into that library's `soupsieve.filter()
<https://facelessuser.github.io/soupsieve/api/#soupsievefilter>`_
method. For more information, see the documentation for
`soupsieve.filter()
<https://facelessuser.github.io/soupsieve/api/#soupsievefilter>`_.
:param namespaces: A dictionary mapping namespace prefixes
used in the CSS selector to namespace URIs. By default,
Beautiful Soup will pass in the prefixes it encountered while
parsing the document.
:param flags: Flags to be passed into Soup Sieve's
`soupsieve.filter()
<https://facelessuser.github.io/soupsieve/api/#soupsievefilter>`_
method.
:param kwargs: Keyword arguments to be passed into SoupSieve's
`soupsieve.filter()
<https://facelessuser.github.io/soupsieve/api/#soupsievefilter>`_
method.
"""
return self._rs(
self.api.filter(
select, self.tag, self._ns(namespaces, select), flags, **kwargs
)
)

1516
venv/lib/python3.12/site-packages/bs4/dammit.py Normal file
View File

File diff suppressed because it is too large Load Diff

268
venv/lib/python3.12/site-packages/bs4/diagnose.py Normal file
View File

@@ -0,0 +1,268 @@
"""Diagnostic functions, mainly for use when doing tech support."""
# Use of this source code is governed by the MIT license.
__license__ = "MIT"
import cProfile
from io import BytesIO
from html.parser import HTMLParser
import bs4
from bs4 import BeautifulSoup, __version__
from bs4.builder import builder_registry
from typing import (
Any,
IO,
List,
Optional,
Tuple,
TYPE_CHECKING,
)
if TYPE_CHECKING:
from bs4._typing import _IncomingMarkup
import pstats
import random
import tempfile
import time
import traceback
import sys
def diagnose(data: "_IncomingMarkup") -> None:
"""Diagnostic suite for isolating common problems.
:param data: Some markup that needs to be explained.
:return: None; diagnostics are printed to standard output.
"""
print(("Diagnostic running on Beautiful Soup %s" % __version__))
print(("Python version %s" % sys.version))
basic_parsers = ["html.parser", "html5lib", "lxml"]
for name in basic_parsers:
for builder in builder_registry.builders:
if name in builder.features:
break
else:
basic_parsers.remove(name)
print(
("I noticed that %s is not installed. Installing it may help." % name)
)
if "lxml" in basic_parsers:
basic_parsers.append("lxml-xml")
try:
from lxml import etree # type:ignore
print(("Found lxml version %s" % ".".join(map(str, etree.LXML_VERSION))))
except ImportError:
print("lxml is not installed or couldn't be imported.")
if "html5lib" in basic_parsers:
try:
import html5lib
print(("Found html5lib version %s" % html5lib.__version__))
except ImportError:
print("html5lib is not installed or couldn't be imported.")
if hasattr(data, "read"):
data = data.read()
for parser in basic_parsers:
print(("Trying to parse your markup with %s" % parser))
success = False
try:
soup = BeautifulSoup(data, features=parser)
success = True
except Exception:
print(("%s could not parse the markup." % parser))
traceback.print_exc()
if success:
print(("Here's what %s did with the markup:" % parser))
print((soup.prettify()))
print(("-" * 80))
def lxml_trace(data: "_IncomingMarkup", html: bool = True, **kwargs: Any) -> None:
"""Print out the lxml events that occur during parsing.
This lets you see how lxml parses a document when no Beautiful
Soup code is running. You can use this to determine whether
an lxml-specific problem is in Beautiful Soup's lxml tree builders
or in lxml itself.
:param data: Some markup.
:param html: If True, markup will be parsed with lxml's HTML parser.
if False, lxml's XML parser will be used.
"""
from lxml import etree
recover = kwargs.pop("recover", True)
if isinstance(data, str):
data = data.encode("utf8")
if not isinstance(data, IO):
reader = BytesIO(data)
for event, element in etree.iterparse(reader, html=html, recover=recover, **kwargs):
print(("%s, %4s, %s" % (event, element.tag, element.text)))
class AnnouncingParser(HTMLParser):
"""Subclass of HTMLParser that announces parse events, without doing
anything else.
You can use this to get a picture of how html.parser sees a given
document. The easiest way to do this is to call `htmlparser_trace`.
"""
def _p(self, s: str) -> None:
print(s)
def handle_starttag(
self,
name: str,
attrs: List[Tuple[str, Optional[str]]],
handle_empty_element: bool = True,
) -> None:
self._p(f"{name} {attrs} START")
def handle_endtag(self, name: str, check_already_closed: bool = True) -> None:
self._p("%s END" % name)
def handle_data(self, data: str) -> None:
self._p("%s DATA" % data)
def handle_charref(self, name: str) -> None:
self._p("%s CHARREF" % name)
def handle_entityref(self, name: str) -> None:
self._p("%s ENTITYREF" % name)
def handle_comment(self, data: str) -> None:
self._p("%s COMMENT" % data)
def handle_decl(self, data: str) -> None:
self._p("%s DECL" % data)
def unknown_decl(self, data: str) -> None:
self._p("%s UNKNOWN-DECL" % data)
def handle_pi(self, data: str) -> None:
self._p("%s PI" % data)
def htmlparser_trace(data: str) -> None:
"""Print out the HTMLParser events that occur during parsing.
This lets you see how HTMLParser parses a document when no
Beautiful Soup code is running.
:param data: Some markup.
"""
parser = AnnouncingParser()
parser.feed(data)
_vowels: str = "aeiou"
_consonants: str = "bcdfghjklmnpqrstvwxyz"
def rword(length: int = 5) -> str:
"""Generate a random word-like string.
:meta private:
"""
s = ""
for i in range(length):
if i % 2 == 0:
t = _consonants
else:
t = _vowels
s += random.choice(t)
return s
def rsentence(length: int = 4) -> str:
"""Generate a random sentence-like string.
:meta private:
"""
return " ".join(rword(random.randint(4, 9)) for i in range(length))
def rdoc(num_elements: int = 1000) -> str:
"""Randomly generate an invalid HTML document.
:meta private:
"""
tag_names = ["p", "div", "span", "i", "b", "script", "table"]
elements = []
for i in range(num_elements):
choice = random.randint(0, 3)
if choice == 0:
# New tag.
tag_name = random.choice(tag_names)
elements.append("<%s>" % tag_name)
elif choice == 1:
elements.append(rsentence(random.randint(1, 4)))
elif choice == 2:
# Close a tag.
tag_name = random.choice(tag_names)
elements.append("</%s>" % tag_name)
return "<html>" + "\n".join(elements) + "</html>"
def benchmark_parsers(num_elements: int = 100000) -> None:
"""Very basic head-to-head performance benchmark."""
print(("Comparative parser benchmark on Beautiful Soup %s" % __version__))
data = rdoc(num_elements)
print(("Generated a large invalid HTML document (%d bytes)." % len(data)))
for parser_name in ["lxml", ["lxml", "html"], "html5lib", "html.parser"]:
success = False
try:
a = time.time()
BeautifulSoup(data, parser_name)
b = time.time()
success = True
except Exception:
print(("%s could not parse the markup." % parser_name))
traceback.print_exc()
if success:
print(("BS4+%s parsed the markup in %.2fs." % (parser_name, b - a)))
from lxml import etree
a = time.time()
etree.HTML(data)
b = time.time()
print(("Raw lxml parsed the markup in %.2fs." % (b - a)))
import html5lib
parser = html5lib.HTMLParser()
a = time.time()
parser.parse(data)
b = time.time()
print(("Raw html5lib parsed the markup in %.2fs." % (b - a)))
def profile(num_elements: int = 100000, parser: str = "lxml") -> None:
"""Use Python's profiler on a randomly generated document."""
filehandle = tempfile.NamedTemporaryFile()
filename = filehandle.name
data = rdoc(num_elements)
vars = dict(bs4=bs4, data=data, parser=parser)
cProfile.runctx("bs4.BeautifulSoup(data, parser)", vars, vars, filename)
stats = pstats.Stats(filename)
# stats.strip_dirs()
stats.sort_stats("cumulative")
stats.print_stats("_html5lib|bs4", 50)
# If this file is run as a script, standard input is diagnosed.
if __name__ == "__main__":
diagnose(sys.stdin.read())

3211
venv/lib/python3.12/site-packages/bs4/element.py Normal file
View File

File diff suppressed because it is too large Load Diff

28
venv/lib/python3.12/site-packages/bs4/exceptions.py Normal file
View File

@@ -0,0 +1,28 @@
"""Exceptions defined by Beautiful Soup itself."""
from typing import Union
class StopParsing(Exception):
"""Exception raised by a TreeBuilder if it's unable to continue parsing."""
class FeatureNotFound(ValueError):
"""Exception raised by the BeautifulSoup constructor if no parser with the
requested features is found.
"""
class ParserRejectedMarkup(Exception):
"""An Exception to be raised when the underlying parser simply
refuses to parse the given markup.
"""
def __init__(self, message_or_exception: Union[str, Exception]):
"""Explain why the parser rejected the given markup, either
with a textual explanation or another exception.
"""
if isinstance(message_or_exception, Exception):
e = message_or_exception
message_or_exception = "%s: %s" % (e.__class__.__name__, str(e))
super(ParserRejectedMarkup, self).__init__(message_or_exception)

764
venv/lib/python3.12/site-packages/bs4/filter.py Normal file
View File

@@ -0,0 +1,764 @@
from __future__ import annotations
from collections import defaultdict
import re
from typing import (
Any,
Callable,
cast,
Dict,
Iterator,
Iterable,
List,
Optional,
Sequence,
Type,
Union,
)
import warnings
from bs4._deprecation import _deprecated
from bs4.element import (
AttributeDict,
NavigableString,
PageElement,
ResultSet,
Tag,
)
from bs4._typing import (
_AtMostOneElement,
_AttributeValue,
_NullableStringMatchFunction,
_OneElement,
_PageElementMatchFunction,
_QueryResults,
_RawAttributeValues,
_RegularExpressionProtocol,
_StrainableAttribute,
_StrainableElement,
_StrainableString,
_StringMatchFunction,
_TagMatchFunction,
)
class ElementFilter(object):
"""`ElementFilter` encapsulates the logic necessary to decide:
1. whether a `PageElement` (a `Tag` or a `NavigableString`) matches a
user-specified query.
2. whether a given sequence of markup found during initial parsing
should be turned into a `PageElement` at all, or simply discarded.
The base class is the simplest `ElementFilter`. By default, it
matches everything and allows all markup to become `PageElement`
objects. You can make it more selective by passing in a
user-defined match function, or defining a subclass.
Most users of Beautiful Soup will never need to use
`ElementFilter`, or its more capable subclass
`SoupStrainer`. Instead, they will use methods like
:py:meth:`Tag.find`, which will convert their arguments into
`SoupStrainer` objects and run them against the tree.
However, if you find yourself wanting to treat the arguments to
Beautiful Soup's find_*() methods as first-class objects, those
objects will be `SoupStrainer` objects. You can create them
yourself and then make use of functions like
`ElementFilter.filter()`.
"""
match_function: Optional[_PageElementMatchFunction]
def __init__(self, match_function: Optional[_PageElementMatchFunction] = None):
"""Pass in a match function to easily customize the behavior of
`ElementFilter.match` without needing to subclass.
:param match_function: A function that takes a `PageElement`
and returns `True` if that `PageElement` matches some criteria.
"""
self.match_function = match_function
@property
def includes_everything(self) -> bool:
"""Does this `ElementFilter` obviously include everything? If so,
the filter process can be made much faster.
The `ElementFilter` might turn out to include everything even
if this returns `False`, but it won't include everything in an
obvious way.
The base `ElementFilter` implementation includes things based on
the match function, so includes_everything is only true if
there is no match function.
"""
return not self.match_function
@property
def excludes_everything(self) -> bool:
"""Does this `ElementFilter` obviously exclude everything? If
so, Beautiful Soup will issue a warning if you try to use it
when parsing a document.
The `ElementFilter` might turn out to exclude everything even
if this returns `False`, but it won't exclude everything in an
obvious way.
The base `ElementFilter` implementation excludes things based
on a match function we can't inspect, so excludes_everything
is always false.
"""
return False
def match(self, element: PageElement, _known_rules:bool=False) -> bool:
"""Does the given PageElement match the rules set down by this
ElementFilter?
The base implementation delegates to the function passed in to
the constructor.
:param _known_rules: Defined for compatibility with
SoupStrainer._match(). Used more for consistency than because
we need the performance optimization.
"""
if not _known_rules and self.includes_everything:
return True
if not self.match_function:
return True
return self.match_function(element)
def filter(self, generator: Iterator[PageElement]) -> Iterator[_OneElement]:
"""The most generic search method offered by Beautiful Soup.
Acts like Python's built-in `filter`, using
`ElementFilter.match` as the filtering function.
"""
# If there are no rules at all, don't bother filtering. Let
# anything through.
if self.includes_everything:
yield from generator
while True:
try:
i = next(generator)
except StopIteration:
break
if i:
if self.match(i, _known_rules=True):
yield cast("_OneElement", i)
def find(self, generator: Iterator[PageElement]) -> _AtMostOneElement:
"""A lower-level equivalent of :py:meth:`Tag.find`.
You can pass in your own generator for iterating over
`PageElement` objects. The first one that matches this
`ElementFilter` will be returned.
:param generator: A way of iterating over `PageElement`
objects.
"""
for match in self.filter(generator):
return match
return None
def find_all(
self, generator: Iterator[PageElement], limit: Optional[int] = None
) -> _QueryResults:
"""A lower-level equivalent of :py:meth:`Tag.find_all`.
You can pass in your own generator for iterating over
`PageElement` objects. Only elements that match this
`ElementFilter` will be returned in the :py:class:`ResultSet`.
:param generator: A way of iterating over `PageElement`
objects.
:param limit: Stop looking after finding this many results.
"""
results = []
for match in self.filter(generator):
results.append(match)
if limit is not None and len(results) >= limit:
break
return ResultSet(self, results)
def allow_tag_creation(
self, nsprefix: Optional[str], name: str, attrs: Optional[_RawAttributeValues]
) -> bool:
"""Based on the name and attributes of a tag, see whether this
`ElementFilter` will allow a `Tag` object to even be created.
By default, all tags are parsed. To change this, subclass
`ElementFilter`.
:param name: The name of the prospective tag.
:param attrs: The attributes of the prospective tag.
"""
return True
def allow_string_creation(self, string: str) -> bool:
"""Based on the content of a string, see whether this
`ElementFilter` will allow a `NavigableString` object based on
this string to be added to the parse tree.
By default, all strings are processed into `NavigableString`
objects. To change this, subclass `ElementFilter`.
:param str: The string under consideration.
"""
return True
class MatchRule(object):
"""Each MatchRule encapsulates the logic behind a single argument
passed in to one of the Beautiful Soup find* methods.
"""
string: Optional[str]
pattern: Optional[_RegularExpressionProtocol]
present: Optional[bool]
exclude_everything: Optional[bool]
# TODO-TYPING: All MatchRule objects also have an attribute
# ``function``, but the type of the function depends on the
# subclass.
def __init__(
self,
string: Optional[Union[str, bytes]] = None,
pattern: Optional[_RegularExpressionProtocol] = None,
function: Optional[Callable] = None,
present: Optional[bool] = None,
exclude_everything: Optional[bool] = None
):
if isinstance(string, bytes):
string = string.decode("utf8")
self.string = string
if isinstance(pattern, bytes):
self.pattern = re.compile(pattern.decode("utf8"))
elif isinstance(pattern, str):
self.pattern = re.compile(pattern)
else:
self.pattern = pattern
self.function = function
self.present = present
self.exclude_everything = exclude_everything
values = [
x
for x in (self.string, self.pattern, self.function, self.present, self.exclude_everything)
if x is not None
]
if len(values) == 0:
raise ValueError(
"Either string, pattern, function, present, or exclude_everything must be provided."
)
if len(values) > 1:
raise ValueError(
"At most one of string, pattern, function, present, and exclude_everything must be provided."
)
def _base_match(self, string: Optional[str]) -> Optional[bool]:
"""Run the 'cheap' portion of a match, trying to get an answer without
calling a potentially expensive custom function.
:return: True or False if we have a (positive or negative)
match; None if we need to keep trying.
"""
# self.exclude_everything matches nothing.
if self.exclude_everything:
return False
# self.present==True matches everything except None.
if self.present is True:
return string is not None
# self.present==False matches _only_ None.
if self.present is False:
return string is None
# self.string does an exact string match.
if self.string is not None:
# print(f"{self.string} ?= {string}")
return self.string == string
# self.pattern does a regular expression search.
if self.pattern is not None:
# print(f"{self.pattern} ?~ {string}")
if string is None:
return False
return self.pattern.search(string) is not None
return None
def matches_string(self, string: Optional[str]) -> bool:
_base_result = self._base_match(string)
if _base_result is not None:
# No need to invoke the test function.
return _base_result
if self.function is not None and not self.function(string):
# print(f"{self.function}({string}) == False")
return False
return True
def __repr__(self) -> str:
cls = type(self).__name__
return f"<{cls} string={self.string} pattern={self.pattern} function={self.function} present={self.present}>"
def __eq__(self, other: Any) -> bool:
return (
isinstance(other, MatchRule)
and self.string == other.string
and self.pattern == other.pattern
and self.function == other.function
and self.present == other.present
)
class TagNameMatchRule(MatchRule):
"""A MatchRule implementing the rules for matches against tag name."""
function: Optional[_TagMatchFunction]
def matches_tag(self, tag: Tag) -> bool:
base_value = self._base_match(tag.name)
if base_value is not None:
return base_value
# The only remaining possibility is that the match is determined
# by a function call. Call the function.
function = cast(_TagMatchFunction, self.function)
if function(tag):
return True
return False
class AttributeValueMatchRule(MatchRule):
"""A MatchRule implementing the rules for matches against attribute value."""
function: Optional[_NullableStringMatchFunction]
class StringMatchRule(MatchRule):
"""A MatchRule implementing the rules for matches against a NavigableString."""
function: Optional[_StringMatchFunction]
class SoupStrainer(ElementFilter):
"""The `ElementFilter` subclass used internally by Beautiful Soup.
A `SoupStrainer` encapsulates the logic necessary to perform the
kind of matches supported by methods such as
:py:meth:`Tag.find`. `SoupStrainer` objects are primarily created
internally, but you can create one yourself and pass it in as
``parse_only`` to the `BeautifulSoup` constructor, to parse a
subset of a large document.
Internally, `SoupStrainer` objects work by converting the
constructor arguments into `MatchRule` objects. Incoming
tags/markup are matched against those rules.
:param name: One or more restrictions on the tags found in a document.
:param attrs: A dictionary that maps attribute names to
restrictions on tags that use those attributes.
:param string: One or more restrictions on the strings found in a
document.
:param kwargs: A dictionary that maps attribute names to restrictions
on tags that use those attributes. These restrictions are additive to
any specified in ``attrs``.
"""
name_rules: List[TagNameMatchRule]
attribute_rules: Dict[str, List[AttributeValueMatchRule]]
string_rules: List[StringMatchRule]
def __init__(
self,
name: Optional[_StrainableElement] = None,
attrs: Optional[Dict[str, _StrainableAttribute]] = None,
string: Optional[_StrainableString] = None,
**kwargs: _StrainableAttribute,
):
if string is None and "text" in kwargs:
string = cast(Optional[_StrainableString], kwargs.pop("text"))
warnings.warn(
"As of version 4.11.0, the 'text' argument to the SoupStrainer constructor is deprecated. Use 'string' instead.",
DeprecationWarning,
stacklevel=2,
)
if name is None and not attrs and not string and not kwargs:
# Special case for backwards compatibility. Instantiating
# a SoupStrainer with no arguments whatsoever gets you one
# that matches all Tags, and only Tags.
self.name_rules = [TagNameMatchRule(present=True)]
else:
self.name_rules = cast(
List[TagNameMatchRule], list(self._make_match_rules(name, TagNameMatchRule))
)
self.attribute_rules = defaultdict(list)
if attrs is None:
attrs = {}
if not isinstance(attrs, dict):
# Passing something other than a dictionary as attrs is
# sugar for matching that thing against the 'class'
# attribute.
attrs = {"class": attrs}
for attrdict in attrs, kwargs:
for attr, value in attrdict.items():
if attr == "class_" and attrdict is kwargs:
# If you pass in 'class_' as part of kwargs, it's
# because class is a Python reserved word. If you
# pass it in as part of the attrs dict, it's
# because you really are looking for an attribute
# called 'class_'.
attr = "class"
if value is None:
value = False
for rule_obj in self._make_match_rules(value, AttributeValueMatchRule):
self.attribute_rules[attr].append(
cast(AttributeValueMatchRule, rule_obj)
)
self.string_rules = cast(
List[StringMatchRule], list(self._make_match_rules(string, StringMatchRule))
)
#: DEPRECATED 4.13.0: You shouldn't need to check this under
#: any name (.string or .text), and if you do, you're probably
#: not taking into account all of the types of values this
#: variable might have. Look at the .string_rules list instead.
self.__string = string
@property
def includes_everything(self) -> bool:
"""Check whether the provided rules will obviously include
everything. (They might include everything even if this returns `False`,
but not in an obvious way.)
"""
return not self.name_rules and not self.string_rules and not self.attribute_rules
@property
def excludes_everything(self) -> bool:
"""Check whether the provided rules will obviously exclude
everything. (They might exclude everything even if this returns `False`,
but not in an obvious way.)
"""
if (self.string_rules and (self.name_rules or self.attribute_rules)):
# This is self-contradictory, so the rules exclude everything.
return True
# If there's a rule that ended up treated as an "exclude everything"
# rule due to creating a logical inconsistency, then the rules
# exclude everything.
if any(x.exclude_everything for x in self.string_rules):
return True
if any(x.exclude_everything for x in self.name_rules):
return True
for ruleset in self.attribute_rules.values():
if any(x.exclude_everything for x in ruleset):
return True
return False
@property
def string(self) -> Optional[_StrainableString]:
":meta private:"
warnings.warn(
"Access to deprecated property string. (Look at .string_rules instead) -- Deprecated since version 4.13.0.",
DeprecationWarning,
stacklevel=2,
)
return self.__string
@property
def text(self) -> Optional[_StrainableString]:
":meta private:"
warnings.warn(
"Access to deprecated property text. (Look at .string_rules instead) -- Deprecated since version 4.13.0.",
DeprecationWarning,
stacklevel=2,
)
return self.__string
def __repr__(self) -> str:
return f"<{self.__class__.__name__} name={self.name_rules} attrs={self.attribute_rules} string={self.string_rules}>"
@classmethod
def _make_match_rules(
cls,
obj: Optional[Union[_StrainableElement, _StrainableAttribute]],
rule_class: Type[MatchRule],
) -> Iterator[MatchRule]:
"""Convert a vaguely-specific 'object' into one or more well-defined
`MatchRule` objects.
:param obj: Some kind of object that corresponds to one or more
matching rules.
:param rule_class: Create instances of this `MatchRule` subclass.
"""
if obj is None:
return
if isinstance(obj, (str, bytes)):
yield rule_class(string=obj)
elif isinstance(obj, bool):
yield rule_class(present=obj)
elif callable(obj):
yield rule_class(function=obj)
elif isinstance(obj, _RegularExpressionProtocol):
yield rule_class(pattern=obj)
elif hasattr(obj, "__iter__"):
if not obj:
# The attribute is being matched against the null set,
# which means it should exclude everything.
yield rule_class(exclude_everything=True)
for o in obj:
if not isinstance(o, (bytes, str)) and hasattr(o, "__iter__"):
# This is almost certainly the user's
# mistake. This list contains another list, which
# opens up the possibility of infinite
# self-reference. In the interests of avoiding
# infinite recursion, we'll treat this as an
# impossible match and issue a rule that excludes
# everything, rather than looking inside.
warnings.warn(
f"Ignoring nested list {o} to avoid the possibility of infinite recursion.",
stacklevel=5,
)
yield rule_class(exclude_everything=True)
continue
for x in cls._make_match_rules(o, rule_class):
yield x
else:
yield rule_class(string=str(obj))
def matches_tag(self, tag: Tag) -> bool:
"""Do the rules of this `SoupStrainer` trigger a match against the
given `Tag`?
If the `SoupStrainer` has any `TagNameMatchRule`, at least one
must match the `Tag` or its `Tag.name`.
If there are any `AttributeValueMatchRule` for a given
attribute, at least one of them must match the attribute
value.
If there are any `StringMatchRule`, at least one must match,
but a `SoupStrainer` that *only* contains `StringMatchRule`
cannot match a `Tag`, only a `NavigableString`.
"""
# If there are no rules at all, let anything through.
#if self.includes_everything:
# return True
# String rules cannot not match a Tag on their own.
if not self.name_rules and not self.attribute_rules:
return False
# Optimization for a very common case where the user is
# searching for a tag with one specific name, and we're
# looking at a tag with a different name.
if (
not tag.prefix
and len(self.name_rules) == 1
and self.name_rules[0].string is not None
and tag.name != self.name_rules[0].string
):
return False
# If there are name rules, at least one must match. It can
# match either the Tag object itself or the prefixed name of
# the tag.
prefixed_name = None
if tag.prefix:
prefixed_name = f"{tag.prefix}:{tag.name}"
if self.name_rules:
name_matches = False
for rule in self.name_rules:
# attrs = " ".join(
# [f"{k}={v}" for k, v in sorted(tag.attrs.items())]
# )
# print(f"Testing <{tag.name} {attrs}>{tag.string}</{tag.name}> against {rule}")
# If the rule contains a function, the function will be called
# with `tag`. It will not be called a second time with
# `prefixed_name`.
if rule.matches_tag(tag) or (
not rule.function and prefixed_name is not None and rule.matches_string(prefixed_name)
):
name_matches = True
break
if not name_matches:
return False
# If there are attribute rules for a given attribute, at least
# one of them must match. If there are rules for multiple
# attributes, each attribute must have at least one match.
for attr, rules in self.attribute_rules.items():
attr_value = tag.get(attr, None)
this_attr_match = self._attribute_match(attr_value, rules)
if not this_attr_match:
return False
# If there are string rules, at least one must match.
if self.string_rules:
_str = tag.string
if _str is None:
return False
if not self.matches_any_string_rule(_str):
return False
return True
def _attribute_match(
self,
attr_value: Optional[_AttributeValue],
rules: Iterable[AttributeValueMatchRule],
) -> bool:
attr_values: Sequence[Optional[str]]
if isinstance(attr_value, list):
attr_values = attr_value
else:
attr_values = [cast(str, attr_value)]
def _match_attribute_value_helper(attr_values: Sequence[Optional[str]]) -> bool:
for rule in rules:
for attr_value in attr_values:
if rule.matches_string(attr_value):
return True
return False
this_attr_match = _match_attribute_value_helper(attr_values)
if not this_attr_match and len(attr_values) != 1:
# Try again but treat the attribute value as a single
# string instead of a list. The result can only be
# different if the list of values contains more or less
# than one item.
# This cast converts Optional[str] to plain str.
#
# We know there can't be any None in the list. Beautiful
# Soup never uses None as a value of a multi-valued
# attribute, and if None is passed in as attr_value, it's
# turned into a list with 1 element, which was excluded by
# the if statement above.
attr_values = cast(Sequence[str], attr_values)
joined_attr_value = " ".join(attr_values)
this_attr_match = _match_attribute_value_helper([joined_attr_value])
return this_attr_match
def allow_tag_creation(
self, nsprefix: Optional[str], name: str, attrs: Optional[_RawAttributeValues]
) -> bool:
"""Based on the name and attributes of a tag, see whether this
`SoupStrainer` will allow a `Tag` object to even be created.
:param name: The name of the prospective tag.
:param attrs: The attributes of the prospective tag.
"""
if self.string_rules:
# A SoupStrainer that has string rules can't be used to
# manage tag creation, because the string rule can't be
# evaluated until after the tag and all of its contents
# have been parsed.
return False
prefixed_name = None
if nsprefix:
prefixed_name = f"{nsprefix}:{name}"
if self.name_rules:
# At least one name rule must match.
name_match = False
for rule in self.name_rules:
for x in name, prefixed_name:
if x is not None:
if rule.matches_string(x):
name_match = True
break
if not name_match:
return False
# For each attribute that has rules, at least one rule must
# match.
if attrs is None:
attrs = AttributeDict()
for attr, rules in self.attribute_rules.items():
attr_value = attrs.get(attr)
if not self._attribute_match(attr_value, rules):
return False
return True
def allow_string_creation(self, string: str) -> bool:
"""Based on the content of a markup string, see whether this
`SoupStrainer` will allow it to be instantiated as a
`NavigableString` object, or whether it should be ignored.
"""
if self.name_rules or self.attribute_rules:
# A SoupStrainer that has name or attribute rules won't
# match any strings; it's designed to match tags with
# certain properties.
return False
if not self.string_rules:
# A SoupStrainer with no string rules will match
# all strings.
return True
if not self.matches_any_string_rule(string):
return False
return True
def matches_any_string_rule(self, string: str) -> bool:
"""See whether the content of a string matches any of
this `SoupStrainer`'s string rules.
"""
if not self.string_rules:
return True
for string_rule in self.string_rules:
if string_rule.matches_string(string):
return True
return False
def match(self, element: PageElement, _known_rules: bool=False) -> bool:
"""Does the given `PageElement` match the rules set down by this
`SoupStrainer`?
The find_* methods rely heavily on this method to find matches.
:param element: A `PageElement`.
:param _known_rules: Set to true in the common case where
we already checked and found at least one rule in this SoupStrainer
that might exclude a PageElement. Without this, we need
to check .includes_everything every time, just to be safe.
:return: `True` if the element matches this `SoupStrainer`'s rules; `False` otherwise.
"""
# If there are no rules at all, let anything through.
if not _known_rules and self.includes_everything:
return True
if isinstance(element, Tag):
return self.matches_tag(element)
assert isinstance(element, NavigableString)
if not (self.name_rules or self.attribute_rules):
# A NavigableString can only match a SoupStrainer that
# does not define any name or attribute rules.
# Then it comes down to the string rules.
return self.matches_any_string_rule(element)
return False
@_deprecated("allow_tag_creation", "4.13.0")
def search_tag(self, name: str, attrs: Optional[_RawAttributeValues]) -> bool:
"""A less elegant version of `allow_tag_creation`. Deprecated as of 4.13.0"""
":meta private:"
return self.allow_tag_creation(None, name, attrs)
@_deprecated("match", "4.13.0")
def search(self, element: PageElement) -> Optional[PageElement]:
"""A less elegant version of match(). Deprecated as of 4.13.0.
:meta private:
"""
return element if self.match(element) else None

276
venv/lib/python3.12/site-packages/bs4/formatter.py Normal file
View File

@@ -0,0 +1,276 @@
from __future__ import annotations
from typing import Callable, Dict, Iterable, Optional, Set, Tuple, TYPE_CHECKING, Union
from typing_extensions import TypeAlias
from bs4.dammit import EntitySubstitution
if TYPE_CHECKING:
from bs4._typing import _AttributeValue
class Formatter(EntitySubstitution):
"""Describes a strategy to use when outputting a parse tree to a string.
Some parts of this strategy come from the distinction between
HTML4, HTML5, and XML. Others are configurable by the user.
Formatters are passed in as the `formatter` argument to methods
like `bs4.element.Tag.encode`. Most people won't need to
think about formatters, and most people who need to think about
them can pass in one of these predefined strings as `formatter`
rather than making a new Formatter object:
For HTML documents:
* 'html' - HTML entity substitution for generic HTML documents. (default)
* 'html5' - HTML entity substitution for HTML5 documents, as
well as some optimizations in the way tags are rendered.
* 'html5-4.12.0' - The version of the 'html5' formatter used prior to
Beautiful Soup 4.13.0.
* 'minimal' - Only make the substitutions necessary to guarantee
valid HTML.
* None - Do not perform any substitution. This will be faster
but may result in invalid markup.
For XML documents:
* 'html' - Entity substitution for XHTML documents.
* 'minimal' - Only make the substitutions necessary to guarantee
valid XML. (default)
* None - Do not perform any substitution. This will be faster
but may result in invalid markup.
"""
#: Constant name denoting HTML markup
HTML: str = "html"
#: Constant name denoting XML markup
XML: str = "xml"
#: Default values for the various constructor options when the
#: markup language is HTML.
HTML_DEFAULTS: Dict[str, Set[str]] = dict(
cdata_containing_tags=set(["script", "style"]),
)
language: Optional[str] #: :meta private:
entity_substitution: Optional[_EntitySubstitutionFunction] #: :meta private:
void_element_close_prefix: str #: :meta private:
cdata_containing_tags: Set[str] #: :meta private:
indent: str #: :meta private:
#: If this is set to true by the constructor, then attributes whose
#: values are sent to the empty string will be treated as HTML
#: boolean attributes. (Attributes whose value is None are always
#: rendered this way.)
empty_attributes_are_booleans: bool
def _default(
self, language: str, value: Optional[Set[str]], kwarg: str
) -> Set[str]:
if value is not None:
return value
if language == self.XML:
# When XML is the markup language in use, all of the
# defaults are the empty list.
return set()
# Otherwise, it depends on what's in HTML_DEFAULTS.
return self.HTML_DEFAULTS[kwarg]
def __init__(
self,
language: Optional[str] = None,
entity_substitution: Optional[_EntitySubstitutionFunction] = None,
void_element_close_prefix: str = "/",
cdata_containing_tags: Optional[Set[str]] = None,
empty_attributes_are_booleans: bool = False,
indent: Union[int,str] = 1,
):
r"""Constructor.
:param language: This should be `Formatter.XML` if you are formatting
XML markup and `Formatter.HTML` if you are formatting HTML markup.
:param entity_substitution: A function to call to replace special
characters with XML/HTML entities. For examples, see
bs4.dammit.EntitySubstitution.substitute_html and substitute_xml.
:param void_element_close_prefix: By default, void elements
are represented as <tag/> (XML rules) rather than <tag>
(HTML rules). To get <tag>, pass in the empty string.
:param cdata_containing_tags: The set of tags that are defined
as containing CDATA in this dialect. For example, in HTML,
<script> and <style> tags are defined as containing CDATA,
and their contents should not be formatted.
:param empty_attributes_are_booleans: If this is set to true,
then attributes whose values are sent to the empty string
will be treated as `HTML boolean
attributes<https://dev.w3.org/html5/spec-LC/common-microsyntaxes.html#boolean-attributes>`_. (Attributes
whose value is None are always rendered this way.)
:param indent: If indent is a non-negative integer or string,
then the contents of elements will be indented
appropriately when pretty-printing. An indent level of 0,
negative, or "" will only insert newlines. Using a
positive integer indent indents that many spaces per
level. If indent is a string (such as "\t"), that string
is used to indent each level. The default behavior is to
indent one space per level.
"""
self.language = language or self.HTML
self.entity_substitution = entity_substitution
self.void_element_close_prefix = void_element_close_prefix
self.cdata_containing_tags = self._default(
self.language, cdata_containing_tags, "cdata_containing_tags"
)
self.empty_attributes_are_booleans = empty_attributes_are_booleans
if indent is None:
indent = 0
indent_str: str
if isinstance(indent, int):
if indent < 0:
indent = 0
indent_str = " " * indent
elif isinstance(indent, str):
indent_str = indent
else:
indent_str = " "
self.indent = indent_str
def substitute(self, ns: str) -> str:
"""Process a string that needs to undergo entity substitution.
This may be a string encountered in an attribute value or as
text.
:param ns: A string.
:return: The same string but with certain characters replaced by named
or numeric entities.
"""
if not self.entity_substitution:
return ns
from .element import NavigableString
if (
isinstance(ns, NavigableString)
and ns.parent is not None
and ns.parent.name in self.cdata_containing_tags
):
# Do nothing.
return ns
# Substitute.
return self.entity_substitution(ns)
def attribute_value(self, value: str) -> str:
"""Process the value of an attribute.
:param ns: A string.
:return: A string with certain characters replaced by named
or numeric entities.
"""
return self.substitute(value)
def attributes(
self, tag: bs4.element.Tag # type:ignore
) -> Iterable[Tuple[str, Optional[_AttributeValue]]]:
"""Reorder a tag's attributes however you want.
By default, attributes are sorted alphabetically. This makes
behavior consistent between Python 2 and Python 3, and preserves
backwards compatibility with older versions of Beautiful Soup.
If `empty_attributes_are_booleans` is True, then
attributes whose values are set to the empty string will be
treated as boolean attributes.
"""
if tag.attrs is None:
return []
items: Iterable[Tuple[str, _AttributeValue]] = list(tag.attrs.items())
return sorted(
(k, (None if self.empty_attributes_are_booleans and v == "" else v))
for k, v in items
)
class HTMLFormatter(Formatter):
"""A generic Formatter for HTML."""
REGISTRY: Dict[Optional[str], HTMLFormatter] = {}
def __init__(
self,
entity_substitution: Optional[_EntitySubstitutionFunction] = None,
void_element_close_prefix: str = "/",
cdata_containing_tags: Optional[Set[str]] = None,
empty_attributes_are_booleans: bool = False,
indent: Union[int,str] = 1,
):
super(HTMLFormatter, self).__init__(
self.HTML,
entity_substitution,
void_element_close_prefix,
cdata_containing_tags,
empty_attributes_are_booleans,
indent=indent
)
class XMLFormatter(Formatter):
"""A generic Formatter for XML."""
REGISTRY: Dict[Optional[str], XMLFormatter] = {}
def __init__(
self,
entity_substitution: Optional[_EntitySubstitutionFunction] = None,
void_element_close_prefix: str = "/",
cdata_containing_tags: Optional[Set[str]] = None,
empty_attributes_are_booleans: bool = False,
indent: Union[int,str] = 1,
):
super(XMLFormatter, self).__init__(
self.XML,
entity_substitution,
void_element_close_prefix,
cdata_containing_tags,
empty_attributes_are_booleans,
indent=indent,
)
# Set up aliases for the default formatters.
HTMLFormatter.REGISTRY["html"] = HTMLFormatter(
entity_substitution=EntitySubstitution.substitute_html
)
HTMLFormatter.REGISTRY["html5"] = HTMLFormatter(
entity_substitution=EntitySubstitution.substitute_html5,
void_element_close_prefix="",
empty_attributes_are_booleans=True,
)
HTMLFormatter.REGISTRY["html5-4.12"] = HTMLFormatter(
entity_substitution=EntitySubstitution.substitute_html,
void_element_close_prefix="",
empty_attributes_are_booleans=True,
)
HTMLFormatter.REGISTRY["minimal"] = HTMLFormatter(
entity_substitution=EntitySubstitution.substitute_xml
)
HTMLFormatter.REGISTRY[None] = HTMLFormatter(entity_substitution=None)
XMLFormatter.REGISTRY["html"] = XMLFormatter(
entity_substitution=EntitySubstitution.substitute_html
)
XMLFormatter.REGISTRY["minimal"] = XMLFormatter(
entity_substitution=EntitySubstitution.substitute_xml
)
XMLFormatter.REGISTRY[None] = XMLFormatter(entity_substitution=None)
# Define type aliases to improve readability.
#
#: A function to call to replace special characters with XML or HTML
#: entities.
_EntitySubstitutionFunction: TypeAlias = Callable[[str], str]
# Many of the output-centered methods take an argument that can either
# be a Formatter object or the name of a Formatter to be looked up.
_FormatterOrName = Union[Formatter, str]

0
venv/lib/python3.12/site-packages/bs4/py.typed Normal file
View File

1
venv/lib/python3.12/site-packages/certifi-2025.11.12.dist-info/INSTALLER Normal file
View File

@@ -0,0 +1 @@
pip

78
venv/lib/python3.12/site-packages/certifi-2025.11.12.dist-info/METADATA Normal file
View File

@@ -0,0 +1,78 @@
Metadata-Version: 2.4
Name: certifi
Version: 2025.11.12
Summary: Python package for providing Mozilla's CA Bundle.
Home-page: https://github.com/certifi/python-certifi
Author: Kenneth Reitz
Author-email: me@kennethreitz.com
License: MPL-2.0
Project-URL: Source, https://github.com/certifi/python-certifi
Classifier: Development Status :: 5 - Production/Stable
Classifier: Intended Audience :: Developers
Classifier: License :: OSI Approved :: Mozilla Public License 2.0 (MPL 2.0)
Classifier: Natural Language :: English
Classifier: Programming Language :: Python
Classifier: Programming Language :: Python :: 3
Classifier: Programming Language :: Python :: 3 :: Only
Classifier: Programming Language :: Python :: 3.7
Classifier: Programming Language :: Python :: 3.8
Classifier: Programming Language :: Python :: 3.9
Classifier: Programming Language :: Python :: 3.10
Classifier: Programming Language :: Python :: 3.11
Classifier: Programming Language :: Python :: 3.12
Classifier: Programming Language :: Python :: 3.13
Classifier: Programming Language :: Python :: 3.14
Requires-Python: >=3.7
License-File: LICENSE
Dynamic: author
Dynamic: author-email
Dynamic: classifier
Dynamic: description
Dynamic: home-page
Dynamic: license
Dynamic: license-file
Dynamic: project-url
Dynamic: requires-python
Dynamic: summary
Certifi: Python SSL Certificates
================================
Certifi provides Mozilla's carefully curated collection of Root Certificates for
validating the trustworthiness of SSL certificates while verifying the identity
of TLS hosts. It has been extracted from the `Requests`_ project.
Installation
------------
``certifi`` is available on PyPI. Simply install it with ``pip``::
$ pip install certifi
Usage
-----
To reference the installed certificate authority (CA) bundle, you can use the
built-in function::
>>> import certifi
>>> certifi.where()
'/usr/local/lib/python3.7/site-packages/certifi/cacert.pem'
Or from the command line::
$ python -m certifi
/usr/local/lib/python3.7/site-packages/certifi/cacert.pem
Enjoy!
.. _`Requests`: https://requests.readthedocs.io/en/master/
Addition/Removal of Certificates
--------------------------------
Certifi does not support any addition/removal or other modification of the
CA trust store content. This project is intended to provide a reliable and
highly portable root of trust to python deployments. Look to upstream projects
for methods to use alternate trust.

14
venv/lib/python3.12/site-packages/certifi-2025.11.12.dist-info/RECORD Normal file
View File

@@ -0,0 +1,14 @@
certifi-2025.11.12.dist-info/INSTALLER,sha256=zuuue4knoyJ-UwPPXg8fezS7VCrXJQrAP7zeNuwvFQg,4
certifi-2025.11.12.dist-info/METADATA,sha256=_JprGu_1lWSdHlruRBKcorXnrfvBDhvX_6KRr8HQbLc,2475
certifi-2025.11.12.dist-info/RECORD,,
certifi-2025.11.12.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
certifi-2025.11.12.dist-info/licenses/LICENSE,sha256=6TcW2mucDVpKHfYP5pWzcPBpVgPSH2-D8FPkLPwQyvc,989
certifi-2025.11.12.dist-info/top_level.txt,sha256=KMu4vUCfsjLrkPbSNdgdekS-pVJzBAJFO__nI8NF6-U,8
certifi/__init__.py,sha256=1BRSxNMnZW7CZ2oJtYWLoJgfHfcB9i273exwiPwfjJM,94
certifi/__main__.py,sha256=xBBoj905TUWBLRGANOcf7oi6e-3dMP4cEoG9OyMs11g,243
certifi/__pycache__/__init__.cpython-312.pyc,,
certifi/__pycache__/__main__.cpython-312.pyc,,
certifi/__pycache__/core.cpython-312.pyc,,
certifi/cacert.pem,sha256=oa1dZD4hxDtb7XTH4IkdzbWPavUcis4eTwINZUqlKhY,283932
certifi/core.py,sha256=XFXycndG5pf37ayeF8N32HUuDafsyhkVMbO4BAPWHa0,3394
certifi/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0

5
venv/lib/python3.12/site-packages/certifi-2025.11.12.dist-info/WHEEL Normal file
View File

@@ -0,0 +1,5 @@
Wheel-Version: 1.0
Generator: setuptools (80.9.0)
Root-Is-Purelib: true
Tag: py3-none-any

20
venv/lib/python3.12/site-packages/certifi-2025.11.12.dist-info/licenses/LICENSE Normal file
View File

@@ -0,0 +1,20 @@
This package contains a modified version of ca-bundle.crt:
ca-bundle.crt -- Bundle of CA Root Certificates
This is a bundle of X.509 certificates of public Certificate Authorities
(CA). These were automatically extracted from Mozilla's root certificates
file (certdata.txt). This file can be found in the mozilla source tree:
https://hg.mozilla.org/mozilla-central/file/tip/security/nss/lib/ckfw/builtins/certdata.txt
It contains the certificates in PEM format and therefore
can be directly used with curl / libcurl / php_curl, or with
an Apache+mod_ssl webserver for SSL client authentication.
Just configure this file as the SSLCACertificateFile.#
***** BEGIN LICENSE BLOCK *****
This Source Code Form is subject to the terms of the Mozilla Public License,
v. 2.0. If a copy of the MPL was not distributed with this file, You can obtain
one at http://mozilla.org/MPL/2.0/.
***** END LICENSE BLOCK *****
@(#) $RCSfile: certdata.txt,v $ $Revision: 1.80 $ $Date: 2011/11/03 15:11:58 $

1
venv/lib/python3.12/site-packages/certifi-2025.11.12.dist-info/top_level.txt Normal file
View File

@@ -0,0 +1 @@
certifi

4
venv/lib/python3.12/site-packages/certifi/__init__.py Normal file
View File

@@ -0,0 +1,4 @@
from .core import contents, where
__all__ = ["contents", "where"]
__version__ = "2025.11.12"

12
venv/lib/python3.12/site-packages/certifi/__main__.py Normal file
View File

@@ -0,0 +1,12 @@
import argparse
from certifi import contents, where
parser = argparse.ArgumentParser()
parser.add_argument("-c", "--contents", action="store_true")
args = parser.parse_args()
if args.contents:
print(contents())
else:
print(where())

BIN
venv/lib/python3.12/site-packages/certifi/__pycache__/__init__.cpython-312.pyc Normal file
View File

Binary file not shown.

BIN
venv/lib/python3.12/site-packages/certifi/__pycache__/__main__.cpython-312.pyc Normal file
View File

Binary file not shown.

BIN
venv/lib/python3.12/site-packages/certifi/__pycache__/core.cpython-312.pyc Normal file
View File

Binary file not shown.

4678
venv/lib/python3.12/site-packages/certifi/cacert.pem Normal file
View File

File diff suppressed because it is too large Load Diff

83
venv/lib/python3.12/site-packages/certifi/core.py Normal file
View File

@@ -0,0 +1,83 @@
"""
certifi.py
~~~~~~~~~~
This module returns the installation location of cacert.pem or its contents.
"""
import sys
import atexit
def exit_cacert_ctx() -> None:
_CACERT_CTX.__exit__(None, None, None) # type: ignore[union-attr]
if sys.version_info >= (3, 11):
from importlib.resources import as_file, files
_CACERT_CTX = None
_CACERT_PATH = None
def where() -> str:
# This is slightly terrible, but we want to delay extracting the file
# in cases where we're inside of a zipimport situation until someone
# actually calls where(), but we don't want to re-extract the file
# on every call of where(), so we'll do it once then store it in a
# global variable.
global _CACERT_CTX
global _CACERT_PATH
if _CACERT_PATH is None:
# This is slightly janky, the importlib.resources API wants you to
# manage the cleanup of this file, so it doesn't actually return a
# path, it returns a context manager that will give you the path
# when you enter it and will do any cleanup when you leave it. In
# the common case of not needing a temporary file, it will just
# return the file system location and the __exit__() is a no-op.
#
# We also have to hold onto the actual context manager, because
# it will do the cleanup whenever it gets garbage collected, so
# we will also store that at the global level as well.
_CACERT_CTX = as_file(files("certifi").joinpath("cacert.pem"))
_CACERT_PATH = str(_CACERT_CTX.__enter__())
atexit.register(exit_cacert_ctx)
return _CACERT_PATH
def contents() -> str:
return files("certifi").joinpath("cacert.pem").read_text(encoding="ascii")
else:
from importlib.resources import path as get_path, read_text
_CACERT_CTX = None
_CACERT_PATH = None
def where() -> str:
# This is slightly terrible, but we want to delay extracting the
# file in cases where we're inside of a zipimport situation until
# someone actually calls where(), but we don't want to re-extract
# the file on every call of where(), so we'll do it once then store
# it in a global variable.
global _CACERT_CTX
global _CACERT_PATH
if _CACERT_PATH is None:
# This is slightly janky, the importlib.resources API wants you
# to manage the cleanup of this file, so it doesn't actually
# return a path, it returns a context manager that will give
# you the path when you enter it and will do any cleanup when
# you leave it. In the common case of not needing a temporary
# file, it will just return the file system location and the
# __exit__() is a no-op.
#
# We also have to hold onto the actual context manager, because
# it will do the cleanup whenever it gets garbage collected, so
# we will also store that at the global level as well.
_CACERT_CTX = get_path("certifi", "cacert.pem")
_CACERT_PATH = str(_CACERT_CTX.__enter__())
atexit.register(exit_cacert_ctx)
return _CACERT_PATH
def contents() -> str:
return read_text("certifi", "cacert.pem", encoding="ascii")

0
venv/lib/python3.12/site-packages/certifi/py.typed Normal file
View File

1
venv/lib/python3.12/site-packages/charset_normalizer-3.4.4.dist-info/INSTALLER Normal file
View File

@@ -0,0 +1 @@
pip

764
venv/lib/python3.12/site-packages/charset_normalizer-3.4.4.dist-info/METADATA Normal file
View File

@@ -0,0 +1,764 @@
Metadata-Version: 2.4
Name: charset-normalizer
Version: 3.4.4
Summary: The Real First Universal Charset Detector. Open, modern and actively maintained alternative to Chardet.
Author-email: "Ahmed R. TAHRI" <tahri.ahmed@proton.me>
Maintainer-email: "Ahmed R. TAHRI" <tahri.ahmed@proton.me>
License: MIT
Project-URL: Changelog, https://github.com/jawah/charset_normalizer/blob/master/CHANGELOG.md
Project-URL: Documentation, https://charset-normalizer.readthedocs.io/
Project-URL: Code, https://github.com/jawah/charset_normalizer
Project-URL: Issue tracker, https://github.com/jawah/charset_normalizer/issues
Keywords: encoding,charset,charset-detector,detector,normalization,unicode,chardet,detect
Classifier: Development Status :: 5 - Production/Stable
Classifier: Intended Audience :: Developers
Classifier: Operating System :: OS Independent
Classifier: Programming Language :: Python
Classifier: Programming Language :: Python :: 3
Classifier: Programming Language :: Python :: 3.7
Classifier: Programming Language :: Python :: 3.8
Classifier: Programming Language :: Python :: 3.9
Classifier: Programming Language :: Python :: 3.10
Classifier: Programming Language :: Python :: 3.11
Classifier: Programming Language :: Python :: 3.12
Classifier: Programming Language :: Python :: 3.13
Classifier: Programming Language :: Python :: 3.14
Classifier: Programming Language :: Python :: 3 :: Only
Classifier: Programming Language :: Python :: Implementation :: CPython
Classifier: Programming Language :: Python :: Implementation :: PyPy
Classifier: Topic :: Text Processing :: Linguistic
Classifier: Topic :: Utilities
Classifier: Typing :: Typed
Requires-Python: >=3.7
Description-Content-Type: text/markdown
License-File: LICENSE
Provides-Extra: unicode-backport
Dynamic: license-file
<h1 align="center">Charset Detection, for Everyone 👋</h1>
<p align="center">
<sup>The Real First Universal Charset Detector</sup><br>
<a href="https://pypi.org/project/charset-normalizer">
<img src="https://img.shields.io/pypi/pyversions/charset_normalizer.svg?orange=blue" />
</a>
<a href="https://pepy.tech/project/charset-normalizer/">
<img alt="Download Count Total" src="https://static.pepy.tech/badge/charset-normalizer/month" />
</a>
<a href="https://bestpractices.coreinfrastructure.org/projects/7297">
<img src="https://bestpractices.coreinfrastructure.org/projects/7297/badge">
</a>
</p>
<p align="center">
<sup><i>Featured Packages</i></sup><br>
<a href="https://github.com/jawah/niquests">
<img alt="Static Badge" src="https://img.shields.io/badge/Niquests-Most_Advanced_HTTP_Client-cyan">
</a>
<a href="https://github.com/jawah/wassima">
<img alt="Static Badge" src="https://img.shields.io/badge/Wassima-Certifi_Replacement-cyan">
</a>
</p>
<p align="center">
<sup><i>In other language (unofficial port - by the community)</i></sup><br>
<a href="https://github.com/nickspring/charset-normalizer-rs">
<img alt="Static Badge" src="https://img.shields.io/badge/Rust-red">
</a>
</p>
> A library that helps you read text from an unknown charset encoding.<br /> Motivated by `chardet`,
> I'm trying to resolve the issue by taking a new approach.
> All IANA character set names for which the Python core library provides codecs are supported.
<p align="center">
>>>>> <a href="https://charsetnormalizerweb.ousret.now.sh" target="_blank">👉 Try Me Online Now, Then Adopt Me 👈 </a> <<<<<
</p>
This project offers you an alternative to **Universal Charset Encoding Detector**, also known as **Chardet**.
| Feature | [Chardet](https://github.com/chardet/chardet) | Charset Normalizer | [cChardet](https://github.com/PyYoshi/cChardet) |
|--------------------------------------------------|:---------------------------------------------:|:--------------------------------------------------------------------------------------------------:|:-----------------------------------------------:|
| `Fast` | ❌ | ✅ | ✅ |
| `Universal**` | ❌ | ✅ | ❌ |
| `Reliable` **without** distinguishable standards | ❌ | ✅ | ✅ |
| `Reliable` **with** distinguishable standards | ✅ | ✅ | ✅ |
| `License` | LGPL-2.1<br>_restrictive_ | MIT | MPL-1.1<br>_restrictive_ |
| `Native Python` | ✅ | ✅ | ❌ |
| `Detect spoken language` | ❌ | ✅ | N/A |
| `UnicodeDecodeError Safety` | ❌ | ✅ | ❌ |
| `Whl Size (min)` | 193.6 kB | 42 kB | ~200 kB |
| `Supported Encoding` | 33 | 🎉 [99](https://charset-normalizer.readthedocs.io/en/latest/user/support.html#supported-encodings) | 40 |
<p align="center">
<img src="https://i.imgflip.com/373iay.gif" alt="Reading Normalized Text" width="226"/><img src="https://media.tenor.com/images/c0180f70732a18b4965448d33adba3d0/tenor.gif" alt="Cat Reading Text" width="200"/>
</p>
*\*\* : They are clearly using specific code for a specific encoding even if covering most of used one*<br>
## ⚡ Performance
This package offer better performance than its counterpart Chardet. Here are some numbers.
| Package | Accuracy | Mean per file (ms) | File per sec (est) |
|-----------------------------------------------|:--------:|:------------------:|:------------------:|
| [chardet](https://github.com/chardet/chardet) | 86 % | 63 ms | 16 file/sec |
| charset-normalizer | **98 %** | **10 ms** | 100 file/sec |
| Package | 99th percentile | 95th percentile | 50th percentile |
|-----------------------------------------------|:---------------:|:---------------:|:---------------:|
| [chardet](https://github.com/chardet/chardet) | 265 ms | 71 ms | 7 ms |
| charset-normalizer | 100 ms | 50 ms | 5 ms |
_updated as of december 2024 using CPython 3.12_
Chardet's performance on larger file (1MB+) are very poor. Expect huge difference on large payload.
> Stats are generated using 400+ files using default parameters. More details on used files, see GHA workflows.
> And yes, these results might change at any time. The dataset can be updated to include more files.
> The actual delays heavily depends on your CPU capabilities. The factors should remain the same.
> Keep in mind that the stats are generous and that Chardet accuracy vs our is measured using Chardet initial capability
> (e.g. Supported Encoding) Challenge-them if you want.
## ✨ Installation
Using pip:
```sh
pip install charset-normalizer -U
```
## 🚀 Basic Usage
### CLI
This package comes with a CLI.
```
usage: normalizer [-h] [-v] [-a] [-n] [-m] [-r] [-f] [-t THRESHOLD]
file [file ...]
The Real First Universal Charset Detector. Discover originating encoding used
on text file. Normalize text to unicode.
positional arguments:
files File(s) to be analysed
optional arguments:
-h, --help show this help message and exit
-v, --verbose Display complementary information about file if any.
Stdout will contain logs about the detection process.
-a, --with-alternative
Output complementary possibilities if any. Top-level
JSON WILL be a list.
-n, --normalize Permit to normalize input file. If not set, program
does not write anything.
-m, --minimal Only output the charset detected to STDOUT. Disabling
JSON output.
-r, --replace Replace file when trying to normalize it instead of
creating a new one.
-f, --force Replace file without asking if you are sure, use this
flag with caution.
-t THRESHOLD, --threshold THRESHOLD
Define a custom maximum amount of chaos allowed in
decoded content. 0. <= chaos <= 1.
--version Show version information and exit.
```
```bash
normalizer ./data/sample.1.fr.srt
```
or
```bash
python -m charset_normalizer ./data/sample.1.fr.srt
```
🎉 Since version 1.4.0 the CLI produce easily usable stdout result in JSON format.
```json
{
"path": "/home/default/projects/charset_normalizer/data/sample.1.fr.srt",
"encoding": "cp1252",
"encoding_aliases": [
"1252",
"windows_1252"
],
"alternative_encodings": [
"cp1254",
"cp1256",
"cp1258",
"iso8859_14",
"iso8859_15",
"iso8859_16",
"iso8859_3",
"iso8859_9",
"latin_1",
"mbcs"
],
"language": "French",
"alphabets": [
"Basic Latin",
"Latin-1 Supplement"
],
"has_sig_or_bom": false,
"chaos": 0.149,
"coherence": 97.152,
"unicode_path": null,
"is_preferred": true
}
```
### Python
*Just print out normalized text*
```python
from charset_normalizer import from_path
results = from_path('./my_subtitle.srt')
print(str(results.best()))
```
*Upgrade your code without effort*
```python
from charset_normalizer import detect
```
The above code will behave the same as **chardet**. We ensure that we offer the best (reasonable) BC result possible.
See the docs for advanced usage : [readthedocs.io](https://charset-normalizer.readthedocs.io/en/latest/)
## 😇 Why
When I started using Chardet, I noticed that it was not suited to my expectations, and I wanted to propose a
reliable alternative using a completely different method. Also! I never back down on a good challenge!
I **don't care** about the **originating charset** encoding, because **two different tables** can
produce **two identical rendered string.**
What I want is to get readable text, the best I can.
In a way, **I'm brute forcing text decoding.** How cool is that ? 😎
Don't confuse package **ftfy** with charset-normalizer or chardet. ftfy goal is to repair Unicode string whereas charset-normalizer to convert raw file in unknown encoding to unicode.
## 🍰 How
- Discard all charset encoding table that could not fit the binary content.
- Measure noise, or the mess once opened (by chunks) with a corresponding charset encoding.
- Extract matches with the lowest mess detected.
- Additionally, we measure coherence / probe for a language.
**Wait a minute**, what is noise/mess and coherence according to **YOU ?**
*Noise :* I opened hundred of text files, **written by humans**, with the wrong encoding table. **I observed**, then
**I established** some ground rules about **what is obvious** when **it seems like** a mess (aka. defining noise in rendered text).
I know that my interpretation of what is noise is probably incomplete, feel free to contribute in order to
improve or rewrite it.
*Coherence :* For each language there is on earth, we have computed ranked letter appearance occurrences (the best we can). So I thought
that intel is worth something here. So I use those records against decoded text to check if I can detect intelligent design.
## ⚡ Known limitations
- Language detection is unreliable when text contains two or more languages sharing identical letters. (eg. HTML (english tags) + Turkish content (Sharing Latin characters))
- Every charset detector heavily depends on sufficient content. In common cases, do not bother run detection on very tiny content.
## ⚠️ About Python EOLs
**If you are running:**
- Python >=2.7,<3.5: Unsupported
- Python 3.5: charset-normalizer < 2.1
- Python 3.6: charset-normalizer < 3.1
- Python 3.7: charset-normalizer < 4.0
Upgrade your Python interpreter as soon as possible.
## 👤 Contributing
Contributions, issues and feature requests are very much welcome.<br />
Feel free to check [issues page](https://github.com/ousret/charset_normalizer/issues) if you want to contribute.
## 📝 License
Copyright © [Ahmed TAHRI @Ousret](https://github.com/Ousret).<br />
This project is [MIT](https://github.com/Ousret/charset_normalizer/blob/master/LICENSE) licensed.
Characters frequencies used in this project © 2012 [Denny Vrandečić](http://simia.net/letters/)
## 💼 For Enterprise
Professional support for charset-normalizer is available as part of the [Tidelift
Subscription][1]. Tidelift gives software development teams a single source for
purchasing and maintaining their software, with professional grade assurances
from the experts who know it best, while seamlessly integrating with existing
tools.
[1]: https://tidelift.com/subscription/pkg/pypi-charset-normalizer?utm_source=pypi-charset-normalizer&utm_medium=readme
[![OpenSSF Best Practices](https://www.bestpractices.dev/projects/7297/badge)](https://www.bestpractices.dev/projects/7297)
# Changelog
All notable changes to charset-normalizer will be documented in this file. This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
## [3.4.4](https://github.com/Ousret/charset_normalizer/compare/3.4.2...3.4.4) (2025-10-13)
### Changed
- Bound `setuptools` to a specific constraint `setuptools>=68,<=81`.
- Raised upper bound of mypyc for the optional pre-built extension to v1.18.2
### Removed
- `setuptools-scm` as a build dependency.
### Misc
- Enforced hashes in `dev-requirements.txt` and created `ci-requirements.txt` for security purposes.
- Additional pre-built wheels for riscv64, s390x, and armv7l architectures.
- Restore ` multiple.intoto.jsonl` in GitHub releases in addition to individual attestation file per wheel.
## [3.4.3](https://github.com/Ousret/charset_normalizer/compare/3.4.2...3.4.3) (2025-08-09)
### Changed
- mypy(c) is no longer a required dependency at build time if `CHARSET_NORMALIZER_USE_MYPYC` isn't set to `1`. (#595) (#583)
- automatically lower confidence on small bytes samples that are not Unicode in `detect` output legacy function. (#391)
### Added
- Custom build backend to overcome inability to mark mypy as an optional dependency in the build phase.
- Support for Python 3.14
### Fixed
- sdist archive contained useless directories.
- automatically fallback on valid UTF-16 or UTF-32 even if the md says it's noisy. (#633)
### Misc
- SBOM are automatically published to the relevant GitHub release to comply with regulatory changes.
Each published wheel comes with its SBOM. We choose CycloneDX as the format.
- Prebuilt optimized wheel are no longer distributed by default for CPython 3.7 due to a change in cibuildwheel.
## [3.4.2](https://github.com/Ousret/charset_normalizer/compare/3.4.1...3.4.2) (2025-05-02)
### Fixed
- Addressed the DeprecationWarning in our CLI regarding `argparse.FileType` by backporting the target class into the package. (#591)
- Improved the overall reliability of the detector with CJK Ideographs. (#605) (#587)
### Changed
- Optional mypyc compilation upgraded to version 1.15 for Python >= 3.8
## [3.4.1](https://github.com/Ousret/charset_normalizer/compare/3.4.0...3.4.1) (2024-12-24)
### Changed
- Project metadata are now stored using `pyproject.toml` instead of `setup.cfg` using setuptools as the build backend.
- Enforce annotation delayed loading for a simpler and consistent types in the project.
- Optional mypyc compilation upgraded to version 1.14 for Python >= 3.8
### Added
- pre-commit configuration.
- noxfile.
### Removed
- `build-requirements.txt` as per using `pyproject.toml` native build configuration.
- `bin/integration.py` and `bin/serve.py` in favor of downstream integration test (see noxfile).
- `setup.cfg` in favor of `pyproject.toml` metadata configuration.
- Unused `utils.range_scan` function.
### Fixed
- Converting content to Unicode bytes may insert `utf_8` instead of preferred `utf-8`. (#572)
- Deprecation warning "'count' is passed as positional argument" when converting to Unicode bytes on Python 3.13+
## [3.4.0](https://github.com/Ousret/charset_normalizer/compare/3.3.2...3.4.0) (2024-10-08)
### Added
- Argument `--no-preemptive` in the CLI to prevent the detector to search for hints.
- Support for Python 3.13 (#512)
### Fixed
- Relax the TypeError exception thrown when trying to compare a CharsetMatch with anything else than a CharsetMatch.
- Improved the general reliability of the detector based on user feedbacks. (#520) (#509) (#498) (#407) (#537)
- Declared charset in content (preemptive detection) not changed when converting to utf-8 bytes. (#381)
## [3.3.2](https://github.com/Ousret/charset_normalizer/compare/3.3.1...3.3.2) (2023-10-31)
### Fixed
- Unintentional memory usage regression when using large payload that match several encoding (#376)
- Regression on some detection case showcased in the documentation (#371)
### Added
- Noise (md) probe that identify malformed arabic representation due to the presence of letters in isolated form (credit to my wife)
## [3.3.1](https://github.com/Ousret/charset_normalizer/compare/3.3.0...3.3.1) (2023-10-22)
### Changed
- Optional mypyc compilation upgraded to version 1.6.1 for Python >= 3.8
- Improved the general detection reliability based on reports from the community
## [3.3.0](https://github.com/Ousret/charset_normalizer/compare/3.2.0...3.3.0) (2023-09-30)
### Added
- Allow to execute the CLI (e.g. normalizer) through `python -m charset_normalizer.cli` or `python -m charset_normalizer`
- Support for 9 forgotten encoding that are supported by Python but unlisted in `encoding.aliases` as they have no alias (#323)
### Removed
- (internal) Redundant utils.is_ascii function and unused function is_private_use_only
- (internal) charset_normalizer.assets is moved inside charset_normalizer.constant
### Changed
- (internal) Unicode code blocks in constants are updated using the latest v15.0.0 definition to improve detection
- Optional mypyc compilation upgraded to version 1.5.1 for Python >= 3.8
### Fixed
- Unable to properly sort CharsetMatch when both chaos/noise and coherence were close due to an unreachable condition in \_\_lt\_\_ (#350)
## [3.2.0](https://github.com/Ousret/charset_normalizer/compare/3.1.0...3.2.0) (2023-06-07)
### Changed
- Typehint for function `from_path` no longer enforce `PathLike` as its first argument
- Minor improvement over the global detection reliability
### Added
- Introduce function `is_binary` that relies on main capabilities, and optimized to detect binaries
- Propagate `enable_fallback` argument throughout `from_bytes`, `from_path`, and `from_fp` that allow a deeper control over the detection (default True)
- Explicit support for Python 3.12
### Fixed
- Edge case detection failure where a file would contain 'very-long' camel cased word (Issue #289)
## [3.1.0](https://github.com/Ousret/charset_normalizer/compare/3.0.1...3.1.0) (2023-03-06)
### Added
- Argument `should_rename_legacy` for legacy function `detect` and disregard any new arguments without errors (PR #262)
### Removed
- Support for Python 3.6 (PR #260)
### Changed
- Optional speedup provided by mypy/c 1.0.1
## [3.0.1](https://github.com/Ousret/charset_normalizer/compare/3.0.0...3.0.1) (2022-11-18)
### Fixed
- Multi-bytes cutter/chunk generator did not always cut correctly (PR #233)
### Changed
- Speedup provided by mypy/c 0.990 on Python >= 3.7
## [3.0.0](https://github.com/Ousret/charset_normalizer/compare/2.1.1...3.0.0) (2022-10-20)
### Added
- Extend the capability of explain=True when cp_isolation contains at most two entries (min one), will log in details of the Mess-detector results
- Support for alternative language frequency set in charset_normalizer.assets.FREQUENCIES
- Add parameter `language_threshold` in `from_bytes`, `from_path` and `from_fp` to adjust the minimum expected coherence ratio
- `normalizer --version` now specify if current version provide extra speedup (meaning mypyc compilation whl)
### Changed
- Build with static metadata using 'build' frontend
- Make the language detection stricter
- Optional: Module `md.py` can be compiled using Mypyc to provide an extra speedup up to 4x faster than v2.1
### Fixed
- CLI with opt --normalize fail when using full path for files
- TooManyAccentuatedPlugin induce false positive on the mess detection when too few alpha character have been fed to it
- Sphinx warnings when generating the documentation
### Removed
- Coherence detector no longer return 'Simple English' instead return 'English'
- Coherence detector no longer return 'Classical Chinese' instead return 'Chinese'
- Breaking: Method `first()` and `best()` from CharsetMatch
- UTF-7 will no longer appear as "detected" without a recognized SIG/mark (is unreliable/conflict with ASCII)
- Breaking: Class aliases CharsetDetector, CharsetDoctor, CharsetNormalizerMatch and CharsetNormalizerMatches
- Breaking: Top-level function `normalize`
- Breaking: Properties `chaos_secondary_pass`, `coherence_non_latin` and `w_counter` from CharsetMatch
- Support for the backport `unicodedata2`
## [3.0.0rc1](https://github.com/Ousret/charset_normalizer/compare/3.0.0b2...3.0.0rc1) (2022-10-18)
### Added
- Extend the capability of explain=True when cp_isolation contains at most two entries (min one), will log in details of the Mess-detector results
- Support for alternative language frequency set in charset_normalizer.assets.FREQUENCIES
- Add parameter `language_threshold` in `from_bytes`, `from_path` and `from_fp` to adjust the minimum expected coherence ratio
### Changed
- Build with static metadata using 'build' frontend
- Make the language detection stricter
### Fixed
- CLI with opt --normalize fail when using full path for files
- TooManyAccentuatedPlugin induce false positive on the mess detection when too few alpha character have been fed to it
### Removed
- Coherence detector no longer return 'Simple English' instead return 'English'
- Coherence detector no longer return 'Classical Chinese' instead return 'Chinese'
## [3.0.0b2](https://github.com/Ousret/charset_normalizer/compare/3.0.0b1...3.0.0b2) (2022-08-21)
### Added
- `normalizer --version` now specify if current version provide extra speedup (meaning mypyc compilation whl)
### Removed
- Breaking: Method `first()` and `best()` from CharsetMatch
- UTF-7 will no longer appear as "detected" without a recognized SIG/mark (is unreliable/conflict with ASCII)
### Fixed
- Sphinx warnings when generating the documentation
## [3.0.0b1](https://github.com/Ousret/charset_normalizer/compare/2.1.0...3.0.0b1) (2022-08-15)
### Changed
- Optional: Module `md.py` can be compiled using Mypyc to provide an extra speedup up to 4x faster than v2.1
### Removed
- Breaking: Class aliases CharsetDetector, CharsetDoctor, CharsetNormalizerMatch and CharsetNormalizerMatches
- Breaking: Top-level function `normalize`
- Breaking: Properties `chaos_secondary_pass`, `coherence_non_latin` and `w_counter` from CharsetMatch
- Support for the backport `unicodedata2`
## [2.1.1](https://github.com/Ousret/charset_normalizer/compare/2.1.0...2.1.1) (2022-08-19)
### Deprecated
- Function `normalize` scheduled for removal in 3.0
### Changed
- Removed useless call to decode in fn is_unprintable (#206)
### Fixed
- Third-party library (i18n xgettext) crashing not recognizing utf_8 (PEP 263) with underscore from [@aleksandernovikov](https://github.com/aleksandernovikov) (#204)
## [2.1.0](https://github.com/Ousret/charset_normalizer/compare/2.0.12...2.1.0) (2022-06-19)
### Added
- Output the Unicode table version when running the CLI with `--version` (PR #194)
### Changed
- Re-use decoded buffer for single byte character sets from [@nijel](https://github.com/nijel) (PR #175)
- Fixing some performance bottlenecks from [@deedy5](https://github.com/deedy5) (PR #183)
### Fixed
- Workaround potential bug in cpython with Zero Width No-Break Space located in Arabic Presentation Forms-B, Unicode 1.1 not acknowledged as space (PR #175)
- CLI default threshold aligned with the API threshold from [@oleksandr-kuzmenko](https://github.com/oleksandr-kuzmenko) (PR #181)
### Removed
- Support for Python 3.5 (PR #192)
### Deprecated
- Use of backport unicodedata from `unicodedata2` as Python is quickly catching up, scheduled for removal in 3.0 (PR #194)
## [2.0.12](https://github.com/Ousret/charset_normalizer/compare/2.0.11...2.0.12) (2022-02-12)
### Fixed
- ASCII miss-detection on rare cases (PR #170)
## [2.0.11](https://github.com/Ousret/charset_normalizer/compare/2.0.10...2.0.11) (2022-01-30)
### Added
- Explicit support for Python 3.11 (PR #164)
### Changed
- The logging behavior have been completely reviewed, now using only TRACE and DEBUG levels (PR #163 #165)
## [2.0.10](https://github.com/Ousret/charset_normalizer/compare/2.0.9...2.0.10) (2022-01-04)
### Fixed
- Fallback match entries might lead to UnicodeDecodeError for large bytes sequence (PR #154)
### Changed
- Skipping the language-detection (CD) on ASCII (PR #155)
## [2.0.9](https://github.com/Ousret/charset_normalizer/compare/2.0.8...2.0.9) (2021-12-03)
### Changed
- Moderating the logging impact (since 2.0.8) for specific environments (PR #147)
### Fixed
- Wrong logging level applied when setting kwarg `explain` to True (PR #146)
## [2.0.8](https://github.com/Ousret/charset_normalizer/compare/2.0.7...2.0.8) (2021-11-24)
### Changed
- Improvement over Vietnamese detection (PR #126)
- MD improvement on trailing data and long foreign (non-pure latin) data (PR #124)
- Efficiency improvements in cd/alphabet_languages from [@adbar](https://github.com/adbar) (PR #122)
- call sum() without an intermediary list following PEP 289 recommendations from [@adbar](https://github.com/adbar) (PR #129)
- Code style as refactored by Sourcery-AI (PR #131)
- Minor adjustment on the MD around european words (PR #133)
- Remove and replace SRTs from assets / tests (PR #139)
- Initialize the library logger with a `NullHandler` by default from [@nmaynes](https://github.com/nmaynes) (PR #135)
- Setting kwarg `explain` to True will add provisionally (bounded to function lifespan) a specific stream handler (PR #135)
### Fixed
- Fix large (misleading) sequence giving UnicodeDecodeError (PR #137)
- Avoid using too insignificant chunk (PR #137)
### Added
- Add and expose function `set_logging_handler` to configure a specific StreamHandler from [@nmaynes](https://github.com/nmaynes) (PR #135)
- Add `CHANGELOG.md` entries, format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) (PR #141)
## [2.0.7](https://github.com/Ousret/charset_normalizer/compare/2.0.6...2.0.7) (2021-10-11)
### Added
- Add support for Kazakh (Cyrillic) language detection (PR #109)
### Changed
- Further, improve inferring the language from a given single-byte code page (PR #112)
- Vainly trying to leverage PEP263 when PEP3120 is not supported (PR #116)
- Refactoring for potential performance improvements in loops from [@adbar](https://github.com/adbar) (PR #113)
- Various detection improvement (MD+CD) (PR #117)
### Removed
- Remove redundant logging entry about detected language(s) (PR #115)
### Fixed
- Fix a minor inconsistency between Python 3.5 and other versions regarding language detection (PR #117 #102)
## [2.0.6](https://github.com/Ousret/charset_normalizer/compare/2.0.5...2.0.6) (2021-09-18)
### Fixed
- Unforeseen regression with the loss of the backward-compatibility with some older minor of Python 3.5.x (PR #100)
- Fix CLI crash when using --minimal output in certain cases (PR #103)
### Changed
- Minor improvement to the detection efficiency (less than 1%) (PR #106 #101)
## [2.0.5](https://github.com/Ousret/charset_normalizer/compare/2.0.4...2.0.5) (2021-09-14)
### Changed
- The project now comply with: flake8, mypy, isort and black to ensure a better overall quality (PR #81)
- The BC-support with v1.x was improved, the old staticmethods are restored (PR #82)
- The Unicode detection is slightly improved (PR #93)
- Add syntax sugar \_\_bool\_\_ for results CharsetMatches list-container (PR #91)
### Removed
- The project no longer raise warning on tiny content given for detection, will be simply logged as warning instead (PR #92)
### Fixed
- In some rare case, the chunks extractor could cut in the middle of a multi-byte character and could mislead the mess detection (PR #95)
- Some rare 'space' characters could trip up the UnprintablePlugin/Mess detection (PR #96)
- The MANIFEST.in was not exhaustive (PR #78)
## [2.0.4](https://github.com/Ousret/charset_normalizer/compare/2.0.3...2.0.4) (2021-07-30)
### Fixed
- The CLI no longer raise an unexpected exception when no encoding has been found (PR #70)
- Fix accessing the 'alphabets' property when the payload contains surrogate characters (PR #68)
- The logger could mislead (explain=True) on detected languages and the impact of one MBCS match (PR #72)
- Submatch factoring could be wrong in rare edge cases (PR #72)
- Multiple files given to the CLI were ignored when publishing results to STDOUT. (After the first path) (PR #72)
- Fix line endings from CRLF to LF for certain project files (PR #67)
### Changed
- Adjust the MD to lower the sensitivity, thus improving the global detection reliability (PR #69 #76)
- Allow fallback on specified encoding if any (PR #71)
## [2.0.3](https://github.com/Ousret/charset_normalizer/compare/2.0.2...2.0.3) (2021-07-16)
### Changed
- Part of the detection mechanism has been improved to be less sensitive, resulting in more accurate detection results. Especially ASCII. (PR #63)
- According to the community wishes, the detection will fall back on ASCII or UTF-8 in a last-resort case. (PR #64)
## [2.0.2](https://github.com/Ousret/charset_normalizer/compare/2.0.1...2.0.2) (2021-07-15)
### Fixed
- Empty/Too small JSON payload miss-detection fixed. Report from [@tseaver](https://github.com/tseaver) (PR #59)
### Changed
- Don't inject unicodedata2 into sys.modules from [@akx](https://github.com/akx) (PR #57)
## [2.0.1](https://github.com/Ousret/charset_normalizer/compare/2.0.0...2.0.1) (2021-07-13)
### Fixed
- Make it work where there isn't a filesystem available, dropping assets frequencies.json. Report from [@sethmlarson](https://github.com/sethmlarson). (PR #55)
- Using explain=False permanently disable the verbose output in the current runtime (PR #47)
- One log entry (language target preemptive) was not show in logs when using explain=True (PR #47)
- Fix undesired exception (ValueError) on getitem of instance CharsetMatches (PR #52)
### Changed
- Public function normalize default args values were not aligned with from_bytes (PR #53)
### Added
- You may now use charset aliases in cp_isolation and cp_exclusion arguments (PR #47)
## [2.0.0](https://github.com/Ousret/charset_normalizer/compare/1.4.1...2.0.0) (2021-07-02)
### Changed
- 4x to 5 times faster than the previous 1.4.0 release. At least 2x faster than Chardet.
- Accent has been made on UTF-8 detection, should perform rather instantaneous.
- The backward compatibility with Chardet has been greatly improved. The legacy detect function returns an identical charset name whenever possible.
- The detection mechanism has been slightly improved, now Turkish content is detected correctly (most of the time)
- The program has been rewritten to ease the readability and maintainability. (+Using static typing)+
- utf_7 detection has been reinstated.
### Removed
- This package no longer require anything when used with Python 3.5 (Dropped cached_property)
- Removed support for these languages: Catalan, Esperanto, Kazakh, Baque, Volapük, Azeri, Galician, Nynorsk, Macedonian, and Serbocroatian.
- The exception hook on UnicodeDecodeError has been removed.
### Deprecated
- Methods coherence_non_latin, w_counter, chaos_secondary_pass of the class CharsetMatch are now deprecated and scheduled for removal in v3.0
### Fixed
- The CLI output used the relative path of the file(s). Should be absolute.
## [1.4.1](https://github.com/Ousret/charset_normalizer/compare/1.4.0...1.4.1) (2021-05-28)
### Fixed
- Logger configuration/usage no longer conflict with others (PR #44)
## [1.4.0](https://github.com/Ousret/charset_normalizer/compare/1.3.9...1.4.0) (2021-05-21)
### Removed
- Using standard logging instead of using the package loguru.
- Dropping nose test framework in favor of the maintained pytest.
- Choose to not use dragonmapper package to help with gibberish Chinese/CJK text.
- Require cached_property only for Python 3.5 due to constraint. Dropping for every other interpreter version.
- Stop support for UTF-7 that does not contain a SIG.
- Dropping PrettyTable, replaced with pure JSON output in CLI.
### Fixed
- BOM marker in a CharsetNormalizerMatch instance could be False in rare cases even if obviously present. Due to the sub-match factoring process.
- Not searching properly for the BOM when trying utf32/16 parent codec.
### Changed
- Improving the package final size by compressing frequencies.json.
- Huge improvement over the larges payload.
### Added
- CLI now produces JSON consumable output.
- Return ASCII if given sequences fit. Given reasonable confidence.
## [1.3.9](https://github.com/Ousret/charset_normalizer/compare/1.3.8...1.3.9) (2021-05-13)
### Fixed
- In some very rare cases, you may end up getting encode/decode errors due to a bad bytes payload (PR #40)
## [1.3.8](https://github.com/Ousret/charset_normalizer/compare/1.3.7...1.3.8) (2021-05-12)
### Fixed
- Empty given payload for detection may cause an exception if trying to access the `alphabets` property. (PR #39)
## [1.3.7](https://github.com/Ousret/charset_normalizer/compare/1.3.6...1.3.7) (2021-05-12)
### Fixed
- The legacy detect function should return UTF-8-SIG if sig is present in the payload. (PR #38)
## [1.3.6](https://github.com/Ousret/charset_normalizer/compare/1.3.5...1.3.6) (2021-02-09)
### Changed
- Amend the previous release to allow prettytable 2.0 (PR #35)
## [1.3.5](https://github.com/Ousret/charset_normalizer/compare/1.3.4...1.3.5) (2021-02-08)
### Fixed
- Fix error while using the package with a python pre-release interpreter (PR #33)
### Changed
- Dependencies refactoring, constraints revised.
### Added
- Add python 3.9 and 3.10 to the supported interpreters
MIT License
Copyright (c) 2025 TAHRI Ahmed R.
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.

35
venv/lib/python3.12/site-packages/charset_normalizer-3.4.4.dist-info/RECORD Normal file
View File

@@ -0,0 +1,35 @@
../../../bin/normalizer,sha256=JkPT5w6gD-7SxFSkAQkAJyO7wx_OKBRIPBnf8QnJMu0,261
charset_normalizer-3.4.4.dist-info/INSTALLER,sha256=zuuue4knoyJ-UwPPXg8fezS7VCrXJQrAP7zeNuwvFQg,4
charset_normalizer-3.4.4.dist-info/METADATA,sha256=jVuUFBti8dav19YLvWissTihVdF2ozUY4KKMw7jdkBQ,37303
charset_normalizer-3.4.4.dist-info/RECORD,,
charset_normalizer-3.4.4.dist-info/WHEEL,sha256=DxRnWQz-Kp9-4a4hdDHsSv0KUC3H7sN9Nbef3-8RjXU,190
charset_normalizer-3.4.4.dist-info/entry_points.txt,sha256=ADSTKrkXZ3hhdOVFi6DcUEHQRS0xfxDIE_pEz4wLIXA,65
charset_normalizer-3.4.4.dist-info/licenses/LICENSE,sha256=bQ1Bv-FwrGx9wkjJpj4lTQ-0WmDVCoJX0K-SxuJJuIc,1071
charset_normalizer-3.4.4.dist-info/top_level.txt,sha256=7ASyzePr8_xuZWJsnqJjIBtyV8vhEo0wBCv1MPRRi3Q,19
charset_normalizer/__init__.py,sha256=OKRxRv2Zhnqk00tqkN0c1BtJjm165fWXLydE52IKuHc,1590
charset_normalizer/__main__.py,sha256=yzYxMR-IhKRHYwcSlavEv8oGdwxsR89mr2X09qXGdps,109
charset_normalizer/__pycache__/__init__.cpython-312.pyc,,
charset_normalizer/__pycache__/__main__.cpython-312.pyc,,
charset_normalizer/__pycache__/api.cpython-312.pyc,,
charset_normalizer/__pycache__/cd.cpython-312.pyc,,
charset_normalizer/__pycache__/constant.cpython-312.pyc,,
charset_normalizer/__pycache__/legacy.cpython-312.pyc,,
charset_normalizer/__pycache__/md.cpython-312.pyc,,
charset_normalizer/__pycache__/models.cpython-312.pyc,,
charset_normalizer/__pycache__/utils.cpython-312.pyc,,
charset_normalizer/__pycache__/version.cpython-312.pyc,,
charset_normalizer/api.py,sha256=V07i8aVeCD8T2fSia3C-fn0i9t8qQguEBhsqszg32Ns,22668
charset_normalizer/cd.py,sha256=WKTo1HDb-H9HfCDc3Bfwq5jzS25Ziy9SE2a74SgTq88,12522
charset_normalizer/cli/__init__.py,sha256=D8I86lFk2-py45JvqxniTirSj_sFyE6sjaY_0-G1shc,136
charset_normalizer/cli/__main__.py,sha256=dMaXG6IJXRvqq8z2tig7Qb83-BpWTln55ooiku5_uvg,12646
charset_normalizer/cli/__pycache__/__init__.cpython-312.pyc,,
charset_normalizer/cli/__pycache__/__main__.cpython-312.pyc,,
charset_normalizer/constant.py,sha256=7UVY4ldYhmQMHUdgQ_sgZmzcQ0xxYxpBunqSZ-XJZ8U,42713
charset_normalizer/legacy.py,sha256=sYBzSpzsRrg_wF4LP536pG64BItw7Tqtc3SMQAHvFLM,2731
charset_normalizer/md.cpython-312-x86_64-linux-gnu.so,sha256=sZ7umtJLjKfA83NFJ7npkiDyr06zDT8cWtl6uIx2MsM,15912
charset_normalizer/md.py,sha256=-_oN3h3_X99nkFfqamD3yu45DC_wfk5odH0Tr_CQiXs,20145
charset_normalizer/md__mypyc.cpython-312-x86_64-linux-gnu.so,sha256=J2WWgLBQiO8sqdFsENp9u5V9uEH0tTwvTLszPdqhsv0,290584
charset_normalizer/models.py,sha256=lKXhOnIPtiakbK3i__J9wpOfzx3JDTKj7Dn3Rg0VaRI,12394
charset_normalizer/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
charset_normalizer/utils.py,sha256=sTejPgrdlNsKNucZfJCxJ95lMTLA0ShHLLE3n5wpT9Q,12170
charset_normalizer/version.py,sha256=nKE4qBNk5WA4LIJ_yIH_aSDfvtsyizkWMg-PUG-UZVk,115

7
venv/lib/python3.12/site-packages/charset_normalizer-3.4.4.dist-info/WHEEL Normal file
View File

@@ -0,0 +1,7 @@
Wheel-Version: 1.0
Generator: setuptools (80.9.0)
Root-Is-Purelib: false
Tag: cp312-cp312-manylinux_2_17_x86_64
Tag: cp312-cp312-manylinux2014_x86_64
Tag: cp312-cp312-manylinux_2_28_x86_64

2
venv/lib/python3.12/site-packages/charset_normalizer-3.4.4.dist-info/entry_points.txt Normal file
View File

@@ -0,0 +1,2 @@
[console_scripts]
normalizer = charset_normalizer.cli:cli_detect

21
venv/lib/python3.12/site-packages/charset_normalizer-3.4.4.dist-info/licenses/LICENSE Normal file
View File

@@ -0,0 +1,21 @@
MIT License
Copyright (c) 2025 TAHRI Ahmed R.
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.

1
venv/lib/python3.12/site-packages/charset_normalizer-3.4.4.dist-info/top_level.txt Normal file
View File

@@ -0,0 +1 @@
charset_normalizer

48
venv/lib/python3.12/site-packages/charset_normalizer/__init__.py Normal file
View File

@@ -0,0 +1,48 @@
"""
Charset-Normalizer
~~~~~~~~~~~~~~
The Real First Universal Charset Detector.
A library that helps you read text from an unknown charset encoding.
Motivated by chardet, This package is trying to resolve the issue by taking a new approach.
All IANA character set names for which the Python core library provides codecs are supported.
Basic usage:
>>> from charset_normalizer import from_bytes
>>> results = from_bytes('Bсеки човек има право на образование. Oбразованието!'.encode('utf_8'))
>>> best_guess = results.best()
>>> str(best_guess)
'Bсеки човек има право на образование. Oбразованието!'
Others methods and usages are available - see the full documentation
at <https://github.com/Ousret/charset_normalizer>.
:copyright: (c) 2021 by Ahmed TAHRI
:license: MIT, see LICENSE for more details.
"""
from __future__ import annotations
import logging
from .api import from_bytes, from_fp, from_path, is_binary
from .legacy import detect
from .models import CharsetMatch, CharsetMatches
from .utils import set_logging_handler
from .version import VERSION, __version__
__all__ = (
"from_fp",
"from_path",
"from_bytes",
"is_binary",
"detect",
"CharsetMatch",
"CharsetMatches",
"__version__",
"VERSION",
"set_logging_handler",
)
# Attach a NullHandler to the top level logger by default
# https://docs.python.org/3.3/howto/logging.html#configuring-logging-for-a-library
logging.getLogger("charset_normalizer").addHandler(logging.NullHandler())

6
venv/lib/python3.12/site-packages/charset_normalizer/__main__.py Normal file
View File

@@ -0,0 +1,6 @@
from __future__ import annotations
from .cli import cli_detect
if __name__ == "__main__":
cli_detect()

BIN
venv/lib/python3.12/site-packages/charset_normalizer/__pycache__/__init__.cpython-312.pyc Normal file
View File

Binary file not shown.

BIN
venv/lib/python3.12/site-packages/charset_normalizer/__pycache__/__main__.cpython-312.pyc Normal file
View File

Binary file not shown.

BIN
venv/lib/python3.12/site-packages/charset_normalizer/__pycache__/api.cpython-312.pyc Normal file
View File

Binary file not shown.

BIN
venv/lib/python3.12/site-packages/charset_normalizer/__pycache__/cd.cpython-312.pyc Normal file
View File

Binary file not shown.

BIN
venv/lib/python3.12/site-packages/charset_normalizer/__pycache__/constant.cpython-312.pyc Normal file
View File

Binary file not shown.

BIN
venv/lib/python3.12/site-packages/charset_normalizer/__pycache__/legacy.cpython-312.pyc Normal file
View File

Binary file not shown.

BIN
venv/lib/python3.12/site-packages/charset_normalizer/__pycache__/md.cpython-312.pyc Normal file
View File

Binary file not shown.

BIN
venv/lib/python3.12/site-packages/charset_normalizer/__pycache__/models.cpython-312.pyc Normal file
View File

Binary file not shown.

BIN
venv/lib/python3.12/site-packages/charset_normalizer/__pycache__/utils.cpython-312.pyc Normal file
View File

Binary file not shown.

BIN
venv/lib/python3.12/site-packages/charset_normalizer/__pycache__/version.cpython-312.pyc Normal file
View File

Binary file not shown.

669
venv/lib/python3.12/site-packages/charset_normalizer/api.py Normal file
View File

@@ -0,0 +1,669 @@
from __future__ import annotations
import logging
from os import PathLike
from typing import BinaryIO
from .cd import (
coherence_ratio,
encoding_languages,
mb_encoding_languages,
merge_coherence_ratios,
)
from .constant import IANA_SUPPORTED, TOO_BIG_SEQUENCE, TOO_SMALL_SEQUENCE, TRACE
from .md import mess_ratio
from .models import CharsetMatch, CharsetMatches
from .utils import (
any_specified_encoding,
cut_sequence_chunks,
iana_name,
identify_sig_or_bom,
is_cp_similar,
is_multi_byte_encoding,
should_strip_sig_or_bom,
)
logger = logging.getLogger("charset_normalizer")
explain_handler = logging.StreamHandler()
explain_handler.setFormatter(
logging.Formatter("%(asctime)s | %(levelname)s | %(message)s")
)
def from_bytes(
sequences: bytes | bytearray,
steps: int = 5,
chunk_size: int = 512,
threshold: float = 0.2,
cp_isolation: list[str] | None = None,
cp_exclusion: list[str] | None = None,
preemptive_behaviour: bool = True,
explain: bool = False,
language_threshold: float = 0.1,
enable_fallback: bool = True,
) -> CharsetMatches:
"""
Given a raw bytes sequence, return the best possibles charset usable to render str objects.
If there is no results, it is a strong indicator that the source is binary/not text.
By default, the process will extract 5 blocks of 512o each to assess the mess and coherence of a given sequence.
And will give up a particular code page after 20% of measured mess. Those criteria are customizable at will.
The preemptive behavior DOES NOT replace the traditional detection workflow, it prioritize a particular code page
but never take it for granted. Can improve the performance.
You may want to focus your attention to some code page or/and not others, use cp_isolation and cp_exclusion for that
purpose.
This function will strip the SIG in the payload/sequence every time except on UTF-16, UTF-32.
By default the library does not setup any handler other than the NullHandler, if you choose to set the 'explain'
toggle to True it will alter the logger configuration to add a StreamHandler that is suitable for debugging.
Custom logging format and handler can be set manually.
"""
if not isinstance(sequences, (bytearray, bytes)):
raise TypeError(
"Expected object of type bytes or bytearray, got: {}".format(
type(sequences)
)
)
if explain:
previous_logger_level: int = logger.level
logger.addHandler(explain_handler)
logger.setLevel(TRACE)
length: int = len(sequences)
if length == 0:
logger.debug("Encoding detection on empty bytes, assuming utf_8 intention.")
if explain: # Defensive: ensure exit path clean handler
logger.removeHandler(explain_handler)
logger.setLevel(previous_logger_level or logging.WARNING)
return CharsetMatches([CharsetMatch(sequences, "utf_8", 0.0, False, [], "")])
if cp_isolation is not None:
logger.log(
TRACE,
"cp_isolation is set. use this flag for debugging purpose. "
"limited list of encoding allowed : %s.",
", ".join(cp_isolation),
)
cp_isolation = [iana_name(cp, False) for cp in cp_isolation]
else:
cp_isolation = []
if cp_exclusion is not None:
logger.log(
TRACE,
"cp_exclusion is set. use this flag for debugging purpose. "
"limited list of encoding excluded : %s.",
", ".join(cp_exclusion),
)
cp_exclusion = [iana_name(cp, False) for cp in cp_exclusion]
else:
cp_exclusion = []
if length <= (chunk_size * steps):
logger.log(
TRACE,
"override steps (%i) and chunk_size (%i) as content does not fit (%i byte(s) given) parameters.",
steps,
chunk_size,
length,
)
steps = 1
chunk_size = length
if steps > 1 and length / steps < chunk_size:
chunk_size = int(length / steps)
is_too_small_sequence: bool = len(sequences) < TOO_SMALL_SEQUENCE
is_too_large_sequence: bool = len(sequences) >= TOO_BIG_SEQUENCE
if is_too_small_sequence:
logger.log(
TRACE,
"Trying to detect encoding from a tiny portion of ({}) byte(s).".format(
length
),
)
elif is_too_large_sequence:
logger.log(
TRACE,
"Using lazy str decoding because the payload is quite large, ({}) byte(s).".format(
length
),
)
prioritized_encodings: list[str] = []
specified_encoding: str | None = (
any_specified_encoding(sequences) if preemptive_behaviour else None
)
if specified_encoding is not None:
prioritized_encodings.append(specified_encoding)
logger.log(
TRACE,
"Detected declarative mark in sequence. Priority +1 given for %s.",
specified_encoding,
)
tested: set[str] = set()
tested_but_hard_failure: list[str] = []
tested_but_soft_failure: list[str] = []
fallback_ascii: CharsetMatch | None = None
fallback_u8: CharsetMatch | None = None
fallback_specified: CharsetMatch | None = None
results: CharsetMatches = CharsetMatches()
early_stop_results: CharsetMatches = CharsetMatches()
sig_encoding, sig_payload = identify_sig_or_bom(sequences)
if sig_encoding is not None:
prioritized_encodings.append(sig_encoding)
logger.log(
TRACE,
"Detected a SIG or BOM mark on first %i byte(s). Priority +1 given for %s.",
len(sig_payload),
sig_encoding,
)
prioritized_encodings.append("ascii")
if "utf_8" not in prioritized_encodings:
prioritized_encodings.append("utf_8")
for encoding_iana in prioritized_encodings + IANA_SUPPORTED:
if cp_isolation and encoding_iana not in cp_isolation:
continue
if cp_exclusion and encoding_iana in cp_exclusion:
continue
if encoding_iana in tested:
continue
tested.add(encoding_iana)
decoded_payload: str | None = None
bom_or_sig_available: bool = sig_encoding == encoding_iana
strip_sig_or_bom: bool = bom_or_sig_available and should_strip_sig_or_bom(
encoding_iana
)
if encoding_iana in {"utf_16", "utf_32"} and not bom_or_sig_available:
logger.log(
TRACE,
"Encoding %s won't be tested as-is because it require a BOM. Will try some sub-encoder LE/BE.",
encoding_iana,
)
continue
if encoding_iana in {"utf_7"} and not bom_or_sig_available:
logger.log(
TRACE,
"Encoding %s won't be tested as-is because detection is unreliable without BOM/SIG.",
encoding_iana,
)
continue
try:
is_multi_byte_decoder: bool = is_multi_byte_encoding(encoding_iana)
except (ModuleNotFoundError, ImportError):
logger.log(
TRACE,
"Encoding %s does not provide an IncrementalDecoder",
encoding_iana,
)
continue
try:
if is_too_large_sequence and is_multi_byte_decoder is False:
str(
(
sequences[: int(50e4)]
if strip_sig_or_bom is False
else sequences[len(sig_payload) : int(50e4)]
),
encoding=encoding_iana,
)
else:
decoded_payload = str(
(
sequences
if strip_sig_or_bom is False
else sequences[len(sig_payload) :]
),
encoding=encoding_iana,
)
except (UnicodeDecodeError, LookupError) as e:
if not isinstance(e, LookupError):
logger.log(
TRACE,
"Code page %s does not fit given bytes sequence at ALL. %s",
encoding_iana,
str(e),
)
tested_but_hard_failure.append(encoding_iana)
continue
similar_soft_failure_test: bool = False
for encoding_soft_failed in tested_but_soft_failure:
if is_cp_similar(encoding_iana, encoding_soft_failed):
similar_soft_failure_test = True
break
if similar_soft_failure_test:
logger.log(
TRACE,
"%s is deemed too similar to code page %s and was consider unsuited already. Continuing!",
encoding_iana,
encoding_soft_failed,
)
continue
r_ = range(
0 if not bom_or_sig_available else len(sig_payload),
length,
int(length / steps),
)
multi_byte_bonus: bool = (
is_multi_byte_decoder
and decoded_payload is not None
and len(decoded_payload) < length
)
if multi_byte_bonus:
logger.log(
TRACE,
"Code page %s is a multi byte encoding table and it appear that at least one character "
"was encoded using n-bytes.",
encoding_iana,
)
max_chunk_gave_up: int = int(len(r_) / 4)
max_chunk_gave_up = max(max_chunk_gave_up, 2)
early_stop_count: int = 0
lazy_str_hard_failure = False
md_chunks: list[str] = []
md_ratios = []
try:
for chunk in cut_sequence_chunks(
sequences,
encoding_iana,
r_,
chunk_size,
bom_or_sig_available,
strip_sig_or_bom,
sig_payload,
is_multi_byte_decoder,
decoded_payload,
):
md_chunks.append(chunk)
md_ratios.append(
mess_ratio(
chunk,
threshold,
explain is True and 1 <= len(cp_isolation) <= 2,
)
)
if md_ratios[-1] >= threshold:
early_stop_count += 1
if (early_stop_count >= max_chunk_gave_up) or (
bom_or_sig_available and strip_sig_or_bom is False
):
break
except (
UnicodeDecodeError
) as e: # Lazy str loading may have missed something there
logger.log(
TRACE,
"LazyStr Loading: After MD chunk decode, code page %s does not fit given bytes sequence at ALL. %s",
encoding_iana,
str(e),
)
early_stop_count = max_chunk_gave_up
lazy_str_hard_failure = True
# We might want to check the sequence again with the whole content
# Only if initial MD tests passes
if (
not lazy_str_hard_failure
and is_too_large_sequence
and not is_multi_byte_decoder
):
try:
sequences[int(50e3) :].decode(encoding_iana, errors="strict")
except UnicodeDecodeError as e:
logger.log(
TRACE,
"LazyStr Loading: After final lookup, code page %s does not fit given bytes sequence at ALL. %s",
encoding_iana,
str(e),
)
tested_but_hard_failure.append(encoding_iana)
continue
mean_mess_ratio: float = sum(md_ratios) / len(md_ratios) if md_ratios else 0.0
if mean_mess_ratio >= threshold or early_stop_count >= max_chunk_gave_up:
tested_but_soft_failure.append(encoding_iana)
logger.log(
TRACE,
"%s was excluded because of initial chaos probing. Gave up %i time(s). "
"Computed mean chaos is %f %%.",
encoding_iana,
early_stop_count,
round(mean_mess_ratio * 100, ndigits=3),
)
# Preparing those fallbacks in case we got nothing.
if (
enable_fallback
and encoding_iana
in ["ascii", "utf_8", specified_encoding, "utf_16", "utf_32"]
and not lazy_str_hard_failure
):
fallback_entry = CharsetMatch(
sequences,
encoding_iana,
threshold,
bom_or_sig_available,
[],
decoded_payload,
preemptive_declaration=specified_encoding,
)
if encoding_iana == specified_encoding:
fallback_specified = fallback_entry
elif encoding_iana == "ascii":
fallback_ascii = fallback_entry
else:
fallback_u8 = fallback_entry
continue
logger.log(
TRACE,
"%s passed initial chaos probing. Mean measured chaos is %f %%",
encoding_iana,
round(mean_mess_ratio * 100, ndigits=3),
)
if not is_multi_byte_decoder:
target_languages: list[str] = encoding_languages(encoding_iana)
else:
target_languages = mb_encoding_languages(encoding_iana)
if target_languages:
logger.log(
TRACE,
"{} should target any language(s) of {}".format(
encoding_iana, str(target_languages)
),
)
cd_ratios = []
# We shall skip the CD when its about ASCII
# Most of the time its not relevant to run "language-detection" on it.
if encoding_iana != "ascii":
for chunk in md_chunks:
chunk_languages = coherence_ratio(
chunk,
language_threshold,
",".join(target_languages) if target_languages else None,
)
cd_ratios.append(chunk_languages)
cd_ratios_merged = merge_coherence_ratios(cd_ratios)
if cd_ratios_merged:
logger.log(
TRACE,
"We detected language {} using {}".format(
cd_ratios_merged, encoding_iana
),
)
current_match = CharsetMatch(
sequences,
encoding_iana,
mean_mess_ratio,
bom_or_sig_available,
cd_ratios_merged,
(
decoded_payload
if (
is_too_large_sequence is False
or encoding_iana in [specified_encoding, "ascii", "utf_8"]
)
else None
),
preemptive_declaration=specified_encoding,
)
results.append(current_match)
if (
encoding_iana in [specified_encoding, "ascii", "utf_8"]
and mean_mess_ratio < 0.1
):
# If md says nothing to worry about, then... stop immediately!
if mean_mess_ratio == 0.0:
logger.debug(
"Encoding detection: %s is most likely the one.",
current_match.encoding,
)
if explain: # Defensive: ensure exit path clean handler
logger.removeHandler(explain_handler)
logger.setLevel(previous_logger_level)
return CharsetMatches([current_match])
early_stop_results.append(current_match)
if (
len(early_stop_results)
and (specified_encoding is None or specified_encoding in tested)
and "ascii" in tested
and "utf_8" in tested
):
probable_result: CharsetMatch = early_stop_results.best() # type: ignore[assignment]
logger.debug(
"Encoding detection: %s is most likely the one.",
probable_result.encoding,
)
if explain: # Defensive: ensure exit path clean handler
logger.removeHandler(explain_handler)
logger.setLevel(previous_logger_level)
return CharsetMatches([probable_result])
if encoding_iana == sig_encoding:
logger.debug(
"Encoding detection: %s is most likely the one as we detected a BOM or SIG within "
"the beginning of the sequence.",
encoding_iana,
)
if explain: # Defensive: ensure exit path clean handler
logger.removeHandler(explain_handler)
logger.setLevel(previous_logger_level)
return CharsetMatches([results[encoding_iana]])
if len(results) == 0:
if fallback_u8 or fallback_ascii or fallback_specified:
logger.log(
TRACE,
"Nothing got out of the detection process. Using ASCII/UTF-8/Specified fallback.",
)
if fallback_specified:
logger.debug(
"Encoding detection: %s will be used as a fallback match",
fallback_specified.encoding,
)
results.append(fallback_specified)
elif (
(fallback_u8 and fallback_ascii is None)
or (
fallback_u8
and fallback_ascii
and fallback_u8.fingerprint != fallback_ascii.fingerprint
)
or (fallback_u8 is not None)
):
logger.debug("Encoding detection: utf_8 will be used as a fallback match")
results.append(fallback_u8)
elif fallback_ascii:
logger.debug("Encoding detection: ascii will be used as a fallback match")
results.append(fallback_ascii)
if results:
logger.debug(
"Encoding detection: Found %s as plausible (best-candidate) for content. With %i alternatives.",
results.best().encoding, # type: ignore
len(results) - 1,
)
else:
logger.debug("Encoding detection: Unable to determine any suitable charset.")
if explain:
logger.removeHandler(explain_handler)
logger.setLevel(previous_logger_level)
return results
def from_fp(
fp: BinaryIO,
steps: int = 5,
chunk_size: int = 512,
threshold: float = 0.20,
cp_isolation: list[str] | None = None,
cp_exclusion: list[str] | None = None,
preemptive_behaviour: bool = True,
explain: bool = False,
language_threshold: float = 0.1,
enable_fallback: bool = True,
) -> CharsetMatches:
"""
Same thing than the function from_bytes but using a file pointer that is already ready.
Will not close the file pointer.
"""
return from_bytes(
fp.read(),
steps,
chunk_size,
threshold,
cp_isolation,
cp_exclusion,
preemptive_behaviour,
explain,
language_threshold,
enable_fallback,
)
def from_path(
path: str | bytes | PathLike, # type: ignore[type-arg]
steps: int = 5,
chunk_size: int = 512,
threshold: float = 0.20,
cp_isolation: list[str] | None = None,
cp_exclusion: list[str] | None = None,
preemptive_behaviour: bool = True,
explain: bool = False,
language_threshold: float = 0.1,
enable_fallback: bool = True,
) -> CharsetMatches:
"""
Same thing than the function from_bytes but with one extra step. Opening and reading given file path in binary mode.
Can raise IOError.
"""
with open(path, "rb") as fp:
return from_fp(
fp,
steps,
chunk_size,
threshold,
cp_isolation,
cp_exclusion,
preemptive_behaviour,
explain,
language_threshold,
enable_fallback,
)
def is_binary(
fp_or_path_or_payload: PathLike | str | BinaryIO | bytes, # type: ignore[type-arg]
steps: int = 5,
chunk_size: int = 512,
threshold: float = 0.20,
cp_isolation: list[str] | None = None,
cp_exclusion: list[str] | None = None,
preemptive_behaviour: bool = True,
explain: bool = False,
language_threshold: float = 0.1,
enable_fallback: bool = False,
) -> bool:
"""
Detect if the given input (file, bytes, or path) points to a binary file. aka. not a string.
Based on the same main heuristic algorithms and default kwargs at the sole exception that fallbacks match
are disabled to be stricter around ASCII-compatible but unlikely to be a string.
"""
if isinstance(fp_or_path_or_payload, (str, PathLike)):
guesses = from_path(
fp_or_path_or_payload,
steps=steps,
chunk_size=chunk_size,
threshold=threshold,
cp_isolation=cp_isolation,
cp_exclusion=cp_exclusion,
preemptive_behaviour=preemptive_behaviour,
explain=explain,
language_threshold=language_threshold,
enable_fallback=enable_fallback,
)
elif isinstance(
fp_or_path_or_payload,
(
bytes,
bytearray,
),
):
guesses = from_bytes(
fp_or_path_or_payload,
steps=steps,
chunk_size=chunk_size,
threshold=threshold,
cp_isolation=cp_isolation,
cp_exclusion=cp_exclusion,
preemptive_behaviour=preemptive_behaviour,
explain=explain,
language_threshold=language_threshold,
enable_fallback=enable_fallback,
)
else:
guesses = from_fp(
fp_or_path_or_payload,
steps=steps,
chunk_size=chunk_size,
threshold=threshold,
cp_isolation=cp_isolation,
cp_exclusion=cp_exclusion,
preemptive_behaviour=preemptive_behaviour,
explain=explain,
language_threshold=language_threshold,
enable_fallback=enable_fallback,
)
return not guesses

395
venv/lib/python3.12/site-packages/charset_normalizer/cd.py Normal file
View File

@@ -0,0 +1,395 @@
from __future__ import annotations
import importlib
from codecs import IncrementalDecoder
from collections import Counter
from functools import lru_cache
from typing import Counter as TypeCounter
from .constant import (
FREQUENCIES,
KO_NAMES,
LANGUAGE_SUPPORTED_COUNT,
TOO_SMALL_SEQUENCE,
ZH_NAMES,
)
from .md import is_suspiciously_successive_range
from .models import CoherenceMatches
from .utils import (
is_accentuated,
is_latin,
is_multi_byte_encoding,
is_unicode_range_secondary,
unicode_range,
)
def encoding_unicode_range(iana_name: str) -> list[str]:
"""
Return associated unicode ranges in a single byte code page.
"""
if is_multi_byte_encoding(iana_name):
raise OSError("Function not supported on multi-byte code page")
decoder = importlib.import_module(f"encodings.{iana_name}").IncrementalDecoder
p: IncrementalDecoder = decoder(errors="ignore")
seen_ranges: dict[str, int] = {}
character_count: int = 0
for i in range(0x40, 0xFF):
chunk: str = p.decode(bytes([i]))
if chunk:
character_range: str | None = unicode_range(chunk)
if character_range is None:
continue
if is_unicode_range_secondary(character_range) is False:
if character_range not in seen_ranges:
seen_ranges[character_range] = 0
seen_ranges[character_range] += 1
character_count += 1
return sorted(
[
character_range
for character_range in seen_ranges
if seen_ranges[character_range] / character_count >= 0.15
]
)
def unicode_range_languages(primary_range: str) -> list[str]:
"""
Return inferred languages used with a unicode range.
"""
languages: list[str] = []
for language, characters in FREQUENCIES.items():
for character in characters:
if unicode_range(character) == primary_range:
languages.append(language)
break
return languages
@lru_cache()
def encoding_languages(iana_name: str) -> list[str]:
"""
Single-byte encoding language association. Some code page are heavily linked to particular language(s).
This function does the correspondence.
"""
unicode_ranges: list[str] = encoding_unicode_range(iana_name)
primary_range: str | None = None
for specified_range in unicode_ranges:
if "Latin" not in specified_range:
primary_range = specified_range
break
if primary_range is None:
return ["Latin Based"]
return unicode_range_languages(primary_range)
@lru_cache()
def mb_encoding_languages(iana_name: str) -> list[str]:
"""
Multi-byte encoding language association. Some code page are heavily linked to particular language(s).
This function does the correspondence.
"""
if (
iana_name.startswith("shift_")
or iana_name.startswith("iso2022_jp")
or iana_name.startswith("euc_j")
or iana_name == "cp932"
):
return ["Japanese"]
if iana_name.startswith("gb") or iana_name in ZH_NAMES:
return ["Chinese"]
if iana_name.startswith("iso2022_kr") or iana_name in KO_NAMES:
return ["Korean"]
return []
@lru_cache(maxsize=LANGUAGE_SUPPORTED_COUNT)
def get_target_features(language: str) -> tuple[bool, bool]:
"""
Determine main aspects from a supported language if it contains accents and if is pure Latin.
"""
target_have_accents: bool = False
target_pure_latin: bool = True
for character in FREQUENCIES[language]:
if not target_have_accents and is_accentuated(character):
target_have_accents = True
if target_pure_latin and is_latin(character) is False:
target_pure_latin = False
return target_have_accents, target_pure_latin
def alphabet_languages(
characters: list[str], ignore_non_latin: bool = False
) -> list[str]:
"""
Return associated languages associated to given characters.
"""
languages: list[tuple[str, float]] = []
source_have_accents = any(is_accentuated(character) for character in characters)
for language, language_characters in FREQUENCIES.items():
target_have_accents, target_pure_latin = get_target_features(language)
if ignore_non_latin and target_pure_latin is False:
continue
if target_have_accents is False and source_have_accents:
continue
character_count: int = len(language_characters)
character_match_count: int = len(
[c for c in language_characters if c in characters]
)
ratio: float = character_match_count / character_count
if ratio >= 0.2:
languages.append((language, ratio))
languages = sorted(languages, key=lambda x: x[1], reverse=True)
return [compatible_language[0] for compatible_language in languages]
def characters_popularity_compare(
language: str, ordered_characters: list[str]
) -> float:
"""
Determine if a ordered characters list (by occurrence from most appearance to rarest) match a particular language.
The result is a ratio between 0. (absolutely no correspondence) and 1. (near perfect fit).
Beware that is function is not strict on the match in order to ease the detection. (Meaning close match is 1.)
"""
if language not in FREQUENCIES:
raise ValueError(f"{language} not available")
character_approved_count: int = 0
FREQUENCIES_language_set = set(FREQUENCIES[language])
ordered_characters_count: int = len(ordered_characters)
target_language_characters_count: int = len(FREQUENCIES[language])
large_alphabet: bool = target_language_characters_count > 26
for character, character_rank in zip(
ordered_characters, range(0, ordered_characters_count)
):
if character not in FREQUENCIES_language_set:
continue
character_rank_in_language: int = FREQUENCIES[language].index(character)
expected_projection_ratio: float = (
target_language_characters_count / ordered_characters_count
)
character_rank_projection: int = int(character_rank * expected_projection_ratio)
if (
large_alphabet is False
and abs(character_rank_projection - character_rank_in_language) > 4
):
continue
if (
large_alphabet is True
and abs(character_rank_projection - character_rank_in_language)
< target_language_characters_count / 3
):
character_approved_count += 1
continue
characters_before_source: list[str] = FREQUENCIES[language][
0:character_rank_in_language
]
characters_after_source: list[str] = FREQUENCIES[language][
character_rank_in_language:
]
characters_before: list[str] = ordered_characters[0:character_rank]
characters_after: list[str] = ordered_characters[character_rank:]
before_match_count: int = len(
set(characters_before) & set(characters_before_source)
)
after_match_count: int = len(
set(characters_after) & set(characters_after_source)
)
if len(characters_before_source) == 0 and before_match_count <= 4:
character_approved_count += 1
continue
if len(characters_after_source) == 0 and after_match_count <= 4:
character_approved_count += 1
continue
if (
before_match_count / len(characters_before_source) >= 0.4
or after_match_count / len(characters_after_source) >= 0.4
):
character_approved_count += 1
continue
return character_approved_count / len(ordered_characters)
def alpha_unicode_split(decoded_sequence: str) -> list[str]:
"""
Given a decoded text sequence, return a list of str. Unicode range / alphabet separation.
Ex. a text containing English/Latin with a bit a Hebrew will return two items in the resulting list;
One containing the latin letters and the other hebrew.
"""
layers: dict[str, str] = {}
for character in decoded_sequence:
if character.isalpha() is False:
continue
character_range: str | None = unicode_range(character)
if character_range is None:
continue
layer_target_range: str | None = None
for discovered_range in layers:
if (
is_suspiciously_successive_range(discovered_range, character_range)
is False
):
layer_target_range = discovered_range
break
if layer_target_range is None:
layer_target_range = character_range
if layer_target_range not in layers:
layers[layer_target_range] = character.lower()
continue
layers[layer_target_range] += character.lower()
return list(layers.values())
def merge_coherence_ratios(results: list[CoherenceMatches]) -> CoherenceMatches:
"""
This function merge results previously given by the function coherence_ratio.
The return type is the same as coherence_ratio.
"""
per_language_ratios: dict[str, list[float]] = {}
for result in results:
for sub_result in result:
language, ratio = sub_result
if language not in per_language_ratios:
per_language_ratios[language] = [ratio]
continue
per_language_ratios[language].append(ratio)
merge = [
(
language,
round(
sum(per_language_ratios[language]) / len(per_language_ratios[language]),
4,
),
)
for language in per_language_ratios
]
return sorted(merge, key=lambda x: x[1], reverse=True)
def filter_alt_coherence_matches(results: CoherenceMatches) -> CoherenceMatches:
"""
We shall NOT return "English—" in CoherenceMatches because it is an alternative
of "English". This function only keeps the best match and remove the em-dash in it.
"""
index_results: dict[str, list[float]] = dict()
for result in results:
language, ratio = result
no_em_name: str = language.replace("—", "")
if no_em_name not in index_results:
index_results[no_em_name] = []
index_results[no_em_name].append(ratio)
if any(len(index_results[e]) > 1 for e in index_results):
filtered_results: CoherenceMatches = []
for language in index_results:
filtered_results.append((language, max(index_results[language])))
return filtered_results
return results
@lru_cache(maxsize=2048)
def coherence_ratio(
decoded_sequence: str, threshold: float = 0.1, lg_inclusion: str | None = None
) -> CoherenceMatches:
"""
Detect ANY language that can be identified in given sequence. The sequence will be analysed by layers.
A layer = Character extraction by alphabets/ranges.
"""
results: list[tuple[str, float]] = []
ignore_non_latin: bool = False
sufficient_match_count: int = 0
lg_inclusion_list = lg_inclusion.split(",") if lg_inclusion is not None else []
if "Latin Based" in lg_inclusion_list:
ignore_non_latin = True
lg_inclusion_list.remove("Latin Based")
for layer in alpha_unicode_split(decoded_sequence):
sequence_frequencies: TypeCounter[str] = Counter(layer)
most_common = sequence_frequencies.most_common()
character_count: int = sum(o for c, o in most_common)
if character_count <= TOO_SMALL_SEQUENCE:
continue
popular_character_ordered: list[str] = [c for c, o in most_common]
for language in lg_inclusion_list or alphabet_languages(
popular_character_ordered, ignore_non_latin
):
ratio: float = characters_popularity_compare(
language, popular_character_ordered
)
if ratio < threshold:
continue
elif ratio >= 0.8:
sufficient_match_count += 1
results.append((language, round(ratio, 4)))
if sufficient_match_count >= 3:
break
return sorted(
filter_alt_coherence_matches(results), key=lambda x: x[1], reverse=True
)

8
venv/lib/python3.12/site-packages/charset_normalizer/cli/__init__.py Normal file
View File

@@ -0,0 +1,8 @@
from __future__ import annotations
from .__main__ import cli_detect, query_yes_no
__all__ = (
"cli_detect",
"query_yes_no",
)

Some files were not shown because too many files have changed in this diff Show More