new_link = url[0:-1]+str(int(url[-1])+1)
# 再次发送请求获取下一页数据
yield scrapy.Request(new_link, callback=self.parse)
为新增的四个字段进行网页标签解析
pipelines 文件
import csv
class BiliPipeline:
def __init__(self):
#打开文件,指定方式为写,利用第 3 个参数把 csv 写数据时产生的空
行消除
self.f = open("lyc 大学课程.csv", "a", newline="")
# 设置文件第一行的字段名,注意要跟 spider 传过来的字典 key 名称
相同
self.fieldnames = ["title",
"url","watchnum","dm","uptime","upname"]
# 指定文件的写入方式为 csv 字典写入,参数 1 为指定具体文件,参
数 2 为指定字段名
self.writer = csv.DictWriter(self.f,
fieldnames=self.fieldnames)
# 写入第一行字段名,因为只要写入一次,所以文件放在__init__里
面
self.writer.writeheader()
def process_item(self, item, spider):
# print("title:", item['title'][0])
# print("url:", item['url'][0])
# print("watchnum:",
item['watchnum'][0].replace("\n","").replace(" ",""))
# print("dm:", item['dm'][0].replace("\n", "").replace(" ",
""))
# print("uptime:", item['uptime'][0].replace("\n",
"").replace(" ", ""))
# print("upname:", item['upname'][0])
print("title:", item['title'])
print("url:", item['url'])
print("watchnum:", item['watchnum'])
print("dm:", item['dm'])
print("uptime:", item['uptime'])