49 lines
2.0 KiB
Python
49 lines
2.0 KiB
Python
import csv
|
|
import os
|
|
|
|
def import_csv_to_mysql(csv_path, host='localhost', port=3306, user='root', password='', database='crawler_tiktok', table='comments'):
|
|
try:
|
|
import pymysql
|
|
except Exception:
|
|
print('missing dependency: pip install pymysql', flush=True)
|
|
raise SystemExit(1)
|
|
if not os.path.exists(csv_path):
|
|
print('csv not found: ' + csv_path, flush=True)
|
|
raise SystemExit(1)
|
|
conn = pymysql.connect(host=host, port=int(port), user=user, password=password, database=database, charset='utf8mb4')
|
|
cur = conn.cursor()
|
|
cur.execute(f"CREATE TABLE IF NOT EXISTS `{table}` (\n `id` BIGINT AUTO_INCREMENT PRIMARY KEY,\n `username` VARCHAR(255),\n `text` TEXT\n ) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci")
|
|
rows = []
|
|
with open(csv_path, 'r', encoding='utf-8', newline='') as f:
|
|
r = csv.reader(f)
|
|
first = True
|
|
for row in r:
|
|
if first and row and row[0].lower() == 'username':
|
|
first = False
|
|
continue
|
|
first = False
|
|
if not row:
|
|
continue
|
|
username = row[0] if len(row) > 0 else ''
|
|
text = row[1] if len(row) > 1 else ''
|
|
rows.append((username, text))
|
|
if rows:
|
|
cur.executemany(f"INSERT INTO `{table}` (`username`,`text`) VALUES (%s,%s)", rows)
|
|
conn.commit()
|
|
cur.close()
|
|
conn.close()
|
|
print(f"inserted={len(rows)}", flush=True)
|
|
|
|
def create_database_if_not_exists(host='localhost', port=3306, user='root', password='', database='yunque'):
|
|
try:
|
|
import pymysql
|
|
except Exception:
|
|
print('missing dependency: pip install pymysql', flush=True)
|
|
raise SystemExit(1)
|
|
conn = pymysql.connect(host=host, port=int(port), user=user, password=password, charset='utf8mb4')
|
|
cur = conn.cursor()
|
|
cur.execute(f"CREATE DATABASE IF NOT EXISTS `{database}` CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci")
|
|
conn.commit()
|
|
cur.close()
|
|
conn.close()
|
|
print(f"database_ready={database}", flush=True) |