Python爬虫：异步加载Jianshu文章信息

# -*- coding: utf-8 -*-
"""
Created on Mon Dec 21 12:49:45 2020

@author: Yuka
"""

#######################异步加载####################
import requests
import re
import time
from lxml import etree

url = "https://www.jianshu.com/?seen_snote_ids%5B%5D=77501094&seen_snote_ids%5B%5D=78620448&seen_snote_ids%5B%5D=80324985&seen_snote_ids%5B%5D=81348233&seen_snote_ids%5B%5D=80348158&seen_snote_ids%5B%5D=81359718&seen_snote_ids%5B%5D=76671403&page=2"
headers = {
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.106 Safari/537.36'
    }
result = requests.get(url=url,headers=headers)
html = etree.HTML(result.text)
li_lst = html.xpath('//ul[@class="note-list"]/li')
info = [["title","content","author","replay"]]
for li in li_lst:
    title = li.xpath('./div/a[1]/text()')[-1]
    content = li.xpath('./div/p[1]/text()')[-1].strip()
    author = li.xpath('./div[1]/div[1]/a[1]/text()')[-1].strip()
    replay = li.xpath('./div[1]/div[1]/a[2]/text()')[-1].strip()
    info.append([title,content,author,replay])
print(info)

#######################存储#########################

import pandas as pd
data = pd.DataFrame(info)
data.to_csv('./test.csv',header=False,index=False)
data.to_excel('./text.xlsx',header=False,index=False)

######################selenium######################
import selenium
from lxml import etree
import time
from selenium import webdriver
import pandas as pd
url = "https://www.jianshu.com/?seen_snote_ids%5B%5D=77501094&seen_snote_ids%5B%5D=78620448&seen_snote_ids%5B%5D=80324985&seen_snote_ids%5B%5D=81348233&seen_snote_ids%5B%5D=80348158&seen_snote_ids%5B%5D=81359718&seen_snote_ids%5B%5D=76671403&page=2"
driver = webdriver.Chrome(executable_path=r'C:\Users\小米帅哥\Desktop\chromedriver_win32\chromedriver.exe')
driver.get(url)
# infos = driver.find_elements_by_xpath('//ul[@class="note-list"]/li')
# for info in infos:
#     title = info.find_element_by_xpath('./div[1]/a[1]').text
#     content = info.find_element_by_xpath('./div[1]/p[1]').text
#     author = info.find_element_by_xpath('./div[1]/div[1]/a[1]').text
#     replay = info.find_element_by_xpath('./div[1]/div[1]/a[2]')[1].text
#     print(author+"\n"+title+"\n"+content+"\n",replay)
lst=[['title','author','content','repaly']]
author = driver.find_elements_by_xpath('//ul[@class="note-list"]/li/div[1]/div[1]/a[1]')
replay = driver.find_elements_by_xpath('//ul[@class="note-list"]/li/div[1]/div[1]/a[2]')
title = driver.find_elements_by_xpath('//ul[@class="note-list"]/li/div[1]/a[1]')
content = driver.find_elements_by_xpath('//ul[@class="note-list"]/li/div[1]/p[1]')
for i in range(len(replay)):
    lst.append([title[i].text,author[i].text,content[i].text,replay[i].text])
data = pd.DataFrame(lst)
data.to_csv('./test.csv',header=False,index=False)