Python 实现模拟登录和抓取网页数据的技巧与方法

import requests

# 请求 URL
url = 'https://accounts.douban.com/j/mobile/login/basic'

# 设置请求头
headers = {
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.3'}

# 设置请求参数
data = {
    'name': 'pidancode.com',  # 用户名
    'password': '123456',  # 密码
    'remember': 'false',  # 是否记住登录状态
    'ticket': ''  # 验证码
}

# 发送 POST 请求，获取响应
response = requests.post(url, headers=headers, data=data)

# 获取响应 cookie
print(response.cookies)

from selenium import webdriver
from selenium.webdriver.common.keys import Keys
import time

# 创建 WebDriver 对象，指明使用 Chrome 浏览器驱动
driver = webdriver.Chrome()

# 打开登录页面
driver.get('https://pidancode.com/login')

# 找到用户名和密码输入框，输入对应的用户名和密码
username = driver.find_element_by_name('username')
password = driver.find_element_by_name('password')
username.send_keys('pidancode.com')
password.send_keys('123456')

# 找到登录按钮，模拟点击
time.sleep(1)
login_btn = driver.find_element_by_xpath('//button[@type="submit"]')
login_btn.click()

# 获取 cookie
cookies = driver.get_cookies()
print(cookies)

# 关闭浏览器
driver.close()

import requests

# 请求 URL
url = 'https://pidancode.com/'

# 发送 GET 请求，获取响应
response = requests.get(url)

# 获取响应内容
content = response.text
print(content)

from bs4 import BeautifulSoup
import requests

# 请求 URL
url = 'https://pidancode.com/'

# 发送 GET 请求，获取响应
response = requests.get(url)

# 获取响应内容
content = response.text

# 使用 BeautifulSoup 解析 HTML
soup = BeautifulSoup(content, 'html.parser')

# 获取网页标题
title = soup.title.string
print(title)

# 获取首页导航条链接
nav_links = soup.select('.nav-link')
for link in nav_links:
    print(link['href'], link.text)

import scrapy


class PidancodeSpider(scrapy.Spider):
    name = 'pidancode'
    allowed_domains = ['pidancode.com']
    start_urls = ['https://pidancode.com/']

    def parse(self, response):
        # 获取网页标题
        title = response.xpath('//title/text()').get()
        print(title)

        # 获取首页导航条链接
        nav_links = response.css('.nav-link::attr(href)').getall()
        nav_texts = response.css('.nav-link::text').getall()
        for link, text in zip(nav_links, nav_texts):
            print(link, text)