python网络爬虫代码

以下是两个Python网络爬虫的例子：

1. 爬取淘宝、天猫等网站的商品信息[^1]。

import requests
from bs4 import BeautifulSoup

def get_product_info(url):
    response = requests.get(url)
    soup = BeautifulSoup(response.text, 'html.parser')
    products = soup.find_all('div', class_='product-item')
    
    for product in products:
        title = product.find('a', class_='product-title').text.strip()
        price = product.find('span', class_='product-price').text.strip()
        print(f"商品名称：{title}，价格：{price}")

# 示例使用
url = 'https://www.taobao.com'
get_product_info(url)

2. 使用Scrapy框架爬取新闻网站的新闻信息。

import scrapy

class NewsSpider(scrapy.Spider):
    name = 'news'
    start_urls = ['https://www.example.com/news']

    def parse(self, response):
        news_list = response.css('.news-list-item')
        
        for news in news_list:
            title = news.css('.news-title::text').get()
            content = news.css('.news-content::text').get()
            comments = news.css('.news-comments::text').get()
            time = news.css('.news-time::text').get()
            
            yield {
                'title': title,
                'content': content,
                'comments': comments,
                'time': time
            }

# 示例使用
scrapy runspider news_spider.py -o news.csv