import requests from lxml import etree import pandas as pd headers = { 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/76.0.3809.100 Safari/537.36' } url = f'http://b2b.huangye88.com/shenzhen/jixie/pn2/' res = requests.get(url, headers=headers) res.encoding = 'utf-8' html = etree.HTML(res.text) for et in html.xpath('//*[@id="jubao"]/dl'): name = et.xpath('./dt/h4/a/@title') print(name[0]) hrefs = [] href = et.xpath("./dt/h4/a/@href")[0] hrefs.append(href) content = [] # 保存内容 for u in hrefs: msg = requests.get(u).text xmsg = etree.HTML(msg) if len(name) != 0: tel = xmsg.xpath('/ul[@class ="1-txt none"/li') if len(tel) != 0: tel_name = tel.xpath('.//label/text()') if len(tel_name) != 0: tel_iphone=tel.xpath('.//span[@class="secret"]/text()') if len(tel_iphone) != 0: tel = tel[0] print(name[0], tel_name,tel_iphone, '\n')

时间: 2024-04-22 12:25:44 浏览: 13
根据您提供的代码,我注意到您在进行xpath解析时出现了一些错误。以下是我发现的问题和修复建议: 1. 在`tel = xmsg.xpath('/ul[@class ="1-txt none"/li')`这行代码中,XPath表达式的括号未正确闭合。修改为`tel = xmsg.xpath('//ul[@class="1-txt none"]/li')`。 2. 在`tel = tel[0]`这行代码中,您将`tel`变量重新赋值为列表中的第一个元素。但是,在上一行代码中,您已经将`tel`变量用于判断是否存在电话信息。因此,在这里不需要重新赋值。 下面是修复后的代码片段: ```python import requests from lxml import etree import pandas as pd headers = { 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/76.0.3809.100 Safari/537.36' } url = f'http://b2b.huangye88.com/shenzhen/jixie/pn2/' res = requests.get(url, headers=headers) res.encoding = 'utf-8' html = etree.HTML(res.text) for et in html.xpath('//*[@id="jubao"]/dl'): name = et.xpath('./dt/h4/a/@title') print(name[0]) hrefs = [] href = et.xpath("./dt/h4/a/@href")[0] hrefs.append(href) content = [] # 保存内容 for u in hrefs: msg = requests.get(u).text xmsg = etree.HTML(msg) if len(name) != 0: tel = xmsg.xpath('//ul[@class="1-txt none"]/li') if len(tel) != 0: tel_name = tel.xpath('.//label/text()') if len(tel_name) != 0: tel_iphone=tel.xpath('.//span[@class="secret"]/text()') if len(tel_iphone) != 0: print(name[0], tel_name, tel_iphone, '\n') ``` 请注意,这只是对您提供的代码片段进行了修复,并不能保证整个代码的正确性。如果您还遇到其他问题,请提供更多的信息,我将尽力帮助您解决。

相关推荐

帮我将以下代码写注释# coding=gbk # -- coding:uft-8 -- # 贝壳网小区 import requests from lxml import etree from time import sleep import hashlib from urllib import parse import pandas as pd def getPosi(tar): try: ak = 'C8rQZy1askzzMtdY3ChAZUer1P0PRjI0' sk = 'shShi1VLCkH1gGR4v75d2LTnrn2Vm5Mg' add = f'/geocoding/v3/?address={tar}&output=json&ak={ak}&city=大连市' add = parse.quote(add, safe="/:=&?#+!$,;'@()*[]") sn = hashlib.md5(parse.quote_plus(add + sk).encode('utf-8')).hexdigest() url = f'https://api.map.baidu.com{add}&sn={sn}' dic = requests.get(url).json() lat = dic['result']['location']['lat'] lng = dic['result']['location']['lng'] return lat, lng except: return None, None def collect(): items = { 'ganjingzi': 22, 'zhongshan': 19, 'shahekou': 14, 'xigang': 12 } resLs = [] for key in items: for page in range(items[key]): page += 1 url = f'https://dl.ke.com/xiaoqu/{key}/pg{page}/' headers = { 'User-Agent': ua, 'Referer': url } while True: try: res = requests.get(url=url, headers=headers, timeout=(5, 5)).content.decode('utf-8') break except: print('again') tree = etree.HTML(res) for li in tree.xpath('//ul[@class="listContent"]/li'): href = li.xpath('./a/@href')[0] while True: try: res = requests.get(url=href, headers=headers, timeout=(5, 5)).content.decode('utf-8') break except: print('again') tree = etree.HTML(res) dic = { 'href': href, 'key': key, 'name': tree.xpath('//h1/@title')[0], 'price': (tree.xpath('//span[@class="xiaoquUnitPrice"]/text()') + [''])[0], 'property': tree.xpath('//span[@class="xiaoquInfoContent"]/text()')[1].strip(), 'building': tree.xpath('//span[@class="xiaoquInfoContent"]/text()')[4].strip(), 'house': tree.xpath('//span[@class="xiaoquInfoContent"]/text()')[5].strip() } dic['lat'], dic['lng'] = getPosi(dic['name']) print(dic) resLs.append(dic) sleep(3) df = pd.DataFrame(resLs) df.to_excel('贝壳网小区.xlsx', encoding='utf-8', index=False) if name == 'main': ua = 'Mozilla/5.0(WindowsNT10.0;Win64;x64)AppleWebKit/537.36(KHTML,likeGecko)Chrome/91.0.4472.106Safari/537.36' collect()

import requests from lxml import etree import pandas as pd username_list=[] film_critic_list=[] useful_num_list=[] useless_num_list=[] assess_list=[] ttt_all_urls = [] for i in range(191): ttt_page_urls = f'https://movie.douban.com/subject/26430107/reviews?sort=hotest&start={i * 20}' headers={'user-agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36 Edg/114.0.1823.41'} rq=requests.get(url=ttt_page_urls,headers=headers) dom1 = etree.HTML(rq.text) ttt_data = dom1.xpath('//*[@id="content"]/div/div[1]/div[1]/div/@data-cid') for i in ttt_data: a=dom1.xpath(f'//*[@id={i}]/div/h2/a/@href') ttt_all_urls.extend(a) for url1 in ttt_all_urls: for i in ttt_data: rq2 = requests.get(url=url1,headers=headers) dom2=etree.HTML(rq2.text) username= dom2.xpath(f'//*[@id={i}]/header/a[1]/span/text()') print(username) film_critic = dom2.xpath(f'//*[@id="link-report-{i}"]/div[1]/p/text()') useful_num = dom2.xpath(f'*[@id="review-{i}-content"]/div[3]/button[1]/text()') useless_num= dom2.xpath(f'*[@id="review-{i}-content"]/div[3]/button[2]/text()') assess = (dom2.xpath('//*[@id="content"]/div/div[1]/h1/span/text()')) username_list.extend(username) film_critic_list.extend(film_critic) useful_num_list.extend(useful_num) useless_num_list.extend(useless_num) assess_list.extend(assess) data={'username':username_list,'film_critic':film_critic_list,'useful_num':useful_num_list,'useless_num':useless_num_list,'assess':assess_list} df=pd.DataFrame(data) df.to_csv('fimldata.csv',encoding='utf-8',index=None)

import requests from lxml import etree import csv import os import pandas as pd import matplotlib.pyplot as plt class MovieDataCollector: def __init__(self): self.url = "https://movie.douban.com/top250?start=%s&filter=" self.headers = { 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36' } self.urls = ['https://movie.douban.com/top250?start={}&filter='.format(str(i * 25)) for i in range(10)] self.movies_data = [] def get_first_text(self, element_list): try: return element_list[0].strip() except IndexError: return "" def download_image(self, url, title): response = requests.get(url) image_name = f'{title.replace("/", "_")}.jpg' image_path = os.path.join('films_pic', image_name) with open(image_path, 'wb') as f: f.write(response.content) def scrape_movie_data(self): count = 1 for url in self.urls: res = requests.get(url=url, headers=self.headers) print(res.status_code) html = etree.HTML(res.text) lis = html.xpath('//*[@id="content"]/div/div[1]/ol/li') print('当前是第{}页'.format(count)) for li in lis: rank = self.get_first_text(li.xpath('./div/div[1]/em/text()')) title = self.get_first_text(li.xpath('./div/div[2]/div[1]/a/span[1]/text()')) director = self.get_first_text(li.xpath('./div/div[2]/div[2]/p[1]/text()')) score = self.get_first_text(li.xpath('./div/div[2]/div[2]/div/span[2]/text()')) comment = self.get_first_text(li.xpath('./div/div[2]/div[2]/div/span[4]/text()')) # #下载电影图片 # image_url = self.get_first_text(li.xpath('./div/div[1]/a/img/@src')) # self.download_image(image_url, title) self.movies_data.append({ '排名': rank,解释这段代码

优化这段代码:import requests import pandas as pd from bs4 import BeautifulSoup from lxml import etree import time import pymysql from sqlalchemy import create_engine from urllib.parse import urlencode # 编码 URL 字符串 start_time = time.time() #计算程序运行时间 def get_one_page(i): try: headers = { 'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/66.0.3359.181 Safari/537.36' } paras = {'reportTime': '2023-03-23', #可以改报告日期,比如2018-6-30获得的就是该季度的信息 'pageNum': i #页码 } url = 'http://s.askci.com/stock/a/?' + urlencode(paras) response = requests.get(url,headers = headers) if response.status_code == 200: return response.text return None except RequestException: print('爬取失败') def parse_one_page(html): soup = BeautifulSoup(html,'lxml') content = soup.select('#myTable04')[0] #[0]将返回的list改为bs4类型 tbl = pd.read_html(content.prettify(),header = 0)[0] # prettify()优化代码,[0]从pd.read_html返回的list中提取出DataFrame tbl.rename(columns = {'序号':'serial_number', '股票代码':'stock_code', '股票简称':'stock_abbre', '公司名称':'company_name', '省份':'province', '城市':'city', '主营业务收入(201712)':'main_bussiness_income', '净利润(201712)':'net_profit', '员工人数':'employees', '上市日期':'listing_date', '招股书':'zhaogushu', '公司财报':'financial_report', '行业分类':'industry_classification', '产品类型':'industry_type', '主营业务':'main_business'},inplace = True) return tbl def generate_mysql(): conn = pymysql.connect( host='localhost', user='root', password='******', port=3306, charset = 'utf8', db = 'wade') cursor = conn.cursor() sql = 'CREATE TABLE IF NOT EXISTS listed_company (serial_number INT(20) NOT NULL,stock_code INT(20) ,stock_abbre VARCHAR(20) ,company_name VARCHAR(20) ,province VARCHAR(20) ,city VARCHAR(20) ,main_bussiness_income VARCHAR(20) ,net_profit VARCHAR(20) ,employees INT(20) ,listing_date DATETIME(0) ,zhaogushu VARCHAR(20) ,financial_report VARCHAR(20) , industry_classification VARCHAR(20) ,industry_type VARCHAR(100) ,main_business VARCHAR(200) ,PRIMARY KEY (serial_number))' cursor.execute(sql) conn.close() def write_to_sql(tbl, db = 'wade'): engine = create_engine('mysql+pymysql://root:******@localhost:3306/{0}?charset=utf8'.format(db)) try: tbl.to_sql('listed_company2',con = engine,if_exists='append',index=False) # append表示在原有表基础上增加,但该表要有表头 except Exception as e: print(e) def main(page): generate_mysql() for i in range(1,page): html = get_one_page(i) tbl = parse_one_page(html) write_to_sql(tbl) # # 单进程 if __name__ == '__main__': main(178) endtime = time.time()-start_time print('程序运行了%.2f秒' %endtime) # 多进程 from multiprocessing import Pool if __name__ == '__main__': pool = Pool(4) pool.map(main, [i for i in range(1,178)]) #共有178页 emdtime = time.time()-start_time print('程序运行了%.2f秒' %(time.time()-start_time))

最新推荐

recommend-type

pre_o_1csdn63m9a1bs0e1rr51niuu33e.a

pre_o_1csdn63m9a1bs0e1rr51niuu33e.a
recommend-type

matlab建立计算力学课程的笔记和文件.zip

matlab建立计算力学课程的笔记和文件.zip
recommend-type

FT-Prog-v3.12.38.643-FTD USB 工作模式设定及eprom读写

FT_Prog_v3.12.38.643--FTD USB 工作模式设定及eprom读写
recommend-type

matlab基于RRT和人工势场法混合算法的路径规划.zip

matlab基于RRT和人工势场法混合算法的路径规划.zip
recommend-type

matlab基于matlab的两步定位软件定义接收机的开源GNSS直接位置估计插件模块.zip

matlab基于matlab的两步定位软件定义接收机的开源GNSS直接位置估计插件模块.zip
recommend-type

zigbee-cluster-library-specification

最新的zigbee-cluster-library-specification说明文档。
recommend-type

管理建模和仿真的文件

管理Boualem Benatallah引用此版本:布阿利姆·贝纳塔拉。管理建模和仿真。约瑟夫-傅立叶大学-格勒诺布尔第一大学,1996年。法语。NNT:电话:00345357HAL ID:电话:00345357https://theses.hal.science/tel-003453572008年12月9日提交HAL是一个多学科的开放存取档案馆,用于存放和传播科学研究论文,无论它们是否被公开。论文可以来自法国或国外的教学和研究机构,也可以来自公共或私人研究中心。L’archive ouverte pluridisciplinaire
recommend-type

实现实时数据湖架构:Kafka与Hive集成

![实现实时数据湖架构:Kafka与Hive集成](https://img-blog.csdnimg.cn/img_convert/10eb2e6972b3b6086286fc64c0b3ee41.jpeg) # 1. 实时数据湖架构概述** 实时数据湖是一种现代数据管理架构,它允许企业以低延迟的方式收集、存储和处理大量数据。与传统数据仓库不同,实时数据湖不依赖于预先定义的模式,而是采用灵活的架构,可以处理各种数据类型和格式。这种架构为企业提供了以下优势: - **实时洞察:**实时数据湖允许企业访问最新的数据,从而做出更明智的决策。 - **数据民主化:**实时数据湖使各种利益相关者都可
recommend-type

2. 通过python绘制y=e-xsin(2πx)图像

可以使用matplotlib库来绘制这个函数的图像。以下是一段示例代码: ```python import numpy as np import matplotlib.pyplot as plt def func(x): return np.exp(-x) * np.sin(2 * np.pi * x) x = np.linspace(0, 5, 500) y = func(x) plt.plot(x, y) plt.xlabel('x') plt.ylabel('y') plt.title('y = e^{-x} sin(2πx)') plt.show() ``` 运行这段
recommend-type

JSBSim Reference Manual

JSBSim参考手册,其中包含JSBSim简介,JSBSim配置文件xml的编写语法,编程手册以及一些应用实例等。其中有部分内容还没有写完,估计有生之年很难看到完整版了,但是内容还是很有参考价值的。