Python爬取当当、京东、亚马逊图书信息代码实例

Python爬取当当、京东、亚马逊图书信息代码实例爬取当当、京东、亚马逊图书信息代码实例

主要介绍了Python爬取当当、京东、亚马逊图书信息代码实例，具有一定借鉴价值，需要的朋友可以参考下。

注：1.本程序采用MSSQLserver数据库存储，请运行程序前手动修改程序开头处的数据库链接信息

2.需要bs4、requests、pymssql库支持

3.支持多线程

from bs4 import BeautifulSoup

import re,requests,pymysql,threading,os,traceback

#返回指定页面的html信息

def getHTMLText(url):

try:

headers = {'User-Agent':'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/56.0.2924.87 Safari/537.36'}

r = requests.get(url,headers = headers)

r.raise_for_status()

r.encoding = r.apparent_encoding

return r.text

except:

return ''

except:

return ''

#获取该关键字在图书网站上的总页数

def getPageLength(webSiteName,url):

try:

soup = getSoupObject(url)

if webSiteName == 'DangDang':

a = soup('a',{'name':'bottom-page-turn'})

return a[-1].string

elif webSiteName == 'Amazon':

class DangDangThread(threading.Thread):

def __init__(self,keyword):

threading.Thread.__init__(self)

self.keyword = keyword

def run(self):

print('提示：开始爬取当当网数据...')

count = 1

length = getPageLength('DangDang','http://search.dangdang.com/?key={}'.format(self.keyword))#总页数

tableName = 'db_{}_dangdang'.format(self.keyword)

url = 'http://search.dangdang.com/?key={}&page_index={}'.format(self.keyword,i)

soup = getSoupObject(url)

lis = soup('li',{'class':re.compile(r'line'),'id':re.compile(r'p')})

for li in lis:

a = li.find_all('a',{'name':'itemlist-title','dd_name':'单品标题'})

pn = li.find_all('span',{'class': 'search_now_price'})

pp = li.find_all('span',{'class': 'search_pre_price'})

if not len(a) == 0:

link = a[0].attrs['href']