1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
| import scrapy
from kmzyw.items import KmzywItem
class KmzywSpiderSpider(scrapy.Spider):
name = 'kmzyw_spider'
allowed_domains = ['www.kmzyw.com.cn']
start_urls = ['https://www.kmzyw.com.cn']
url_list = [
{
"code": "policy",
"url": "https://www.kmzyw.com.cn/channel_11400/"
},
{
"code": "inf",
"url": "https://www.kmzyw.com.cn/channel_11300/"
}
]
def CodeDictionary(self, code):
methods = {
"inf": "资讯",
"policy": "政策"
}
return methods.get(code)
def MsgList(self, response):
code = response.meta['code']
main_type = self.CodeDictionary(code)
msg_list = response.xpath("//div[@class='km-article-list clearfix']/ul/li//div[@class='title-wrap text-ellipsis']/a/@href").extract()
for item in msg_list:
print(item)
km = KmzywItem()
link = self.start_urls[0] + item
km['link'] = link
km['type'] = main_type
yield scrapy.Request(link, callback=self.Methed, meta={"item": km})
next_link = response.xpath("//div[@class='km-page-box clearfix']/ul/li//a[@aria-label='Next']/@href").extract()
if next_link:
next_link = next_link[0]
yield scrapy.Request('https://www.kmzyw.com.cn/channel_11300/' + next_link, callback=self.MsgList)
def Methed(self, response):
km = response.meta['item']
km['title'] = response.xpath("//div[@class='km-article-cont']/h1/text()").extract()
km['date'] = response.xpath("//div[@class='km-article-cont']//div[@class='source-wrap tips-left lh-30']/span[1]/text()").extract_first()
km['msg'] = response.xpath("//div[@class='km-article-cont']//div[@class='cont-wrap clearfix']//tbody").extract()
yield km
def Parse(self, response):
for item in self.url_list:
yield scrapy.Request(url=item["url"], callback=self.MsgList, meta={'code': item["code"]})
|
