32 lines
1.1 KiB
Python
32 lines
1.1 KiB
Python
|
import scrapy
|
|||
|
|
|
|||
|
|
class MimvpSpider(scrapy.spiders.Spider):
|
|||
|
|
name = "mimvp"
|
|||
|
|
allowed_domains = ["mimvp.com"]
|
|||
|
|
start_urls = [
|
|||
|
|
"http://proxy.mimvp.com/exist.php",
|
|||
|
|
"https://proxy.mimvp.com/exist.php",
|
|||
|
|
]
|
|||
|
|
|
|||
|
|
## 代理设置方式1：直接在代理里设置
|
|||
|
|
def start_requests(self):
|
|||
|
|
urls = [
|
|||
|
|
"http://proxy.mimvp.com/exist.php",
|
|||
|
|
"https://proxy.mimvp.com/exist.php",
|
|||
|
|
]
|
|||
|
|
for url in urls:
|
|||
|
|
meta_proxy = ""
|
|||
|
|
if url.startswith("http://"):
|
|||
|
|
meta_proxy = "http://180.96.27.12:88" # http代理
|
|||
|
|
elif url.startswith("https://"):
|
|||
|
|
meta_proxy = "http://109.108.87.136:53281" # https代理
|
|||
|
|
|
|||
|
|
yield scrapy.Request(url=url, callback=self.parse, meta={'proxy': meta_proxy})
|
|||
|
|
|
|||
|
|
|
|||
|
|
def parse(self, response):
|
|||
|
|
mimvp_url = response.url # 爬取时请求的url
|
|||
|
|
body = response.body # 返回网页内容
|
|||
|
|
|
|||
|
|
print("mimvp_url : " + str(mimvp_url))
|
|||
|
|
print("body : " + str(body))
|