帮我用python语言写一段可以根据关键词爬取全网信息的爬虫工具代码
时间: 2023-09-12 21:05:31 浏览: 121
基于python开发的网络爬虫代码
### 回答1:
可以使用以下代码来编写一个爬虫工具:import requests
from bs4 import BeautifulSoupkeyword = input("请输入搜索关键词:")
page_num = int(input("请输入页码数:"))# 循环遍历每一页
for i in range(page_num):
url = 'https://search.baidu.com/s?wd=' + keyword + '&pn=' + str(i*10)
response = requests.get(url)
soup = BeautifulSoup(response.text, 'html.parser') # 找到所有搜索结果
results = soup.find_all('div', class_='result')
for result in results:
title = result.find('h3')
print(title.get_text())
### 回答2:
import requests
from bs4 import BeautifulSoup
def spider(keyword):
url = "https://www.baidu.com/s"
params = {'wd': keyword}
try:
response = requests.get(url, params=params)
response.raise_for_status()
soup = BeautifulSoup(response.text, 'html.parser')
search_results = soup.find_all('div', class_='result')
for result in search_results:
title = result.find('h3').get_text()
link = result.find('a')['href']
print(f"标题:{title}")
print(f"链接:{link}")
print()
except Exception as e:
print("爬取失败:", repr(e))
keyword = input("请输入关键词:")
spider(keyword)
阅读全文