Python 实现自动登录并爬取数据的方法

import requests
from bs4 import BeautifulSoup

# 获取登录界面的 HTML 源码
url_login = 'http://XXX/login.html'
r = requests.get(url_login)
bs = BeautifulSoup(r.text, 'html.parser')

# 解析表单数据
form = bs.find('form', {'id': 'login-form'})
action = form['action']
method = form['method']
inputs = form.find_all('input')
data_login = {}
for input in inputs:
    if input.has_attr('name'):
        name = input['name']
        value = input.get('value', '')
        data_login[name] = value

# 填写用户名和密码
data_login['username'] = 'your_username'
data_login['password'] = 'your_password'

# 发送登录请求
r = requests.post(action, data=data_login)

if r.status_code == 200 and '登录成功' in r.text:
    print('登录成功！')
else:
    print('登录失败！')

# 抓取目标页面的 HTML 源码
url_target = 'http://XXX/target.html'
r = requests.get(url_target)
bs = BeautifulSoup(r.text, 'html.parser')

# 使用 BeautifulSoup 提取需要的数据
data = []
items = bs.find_all('div', {'class': 'item'})
for item in items:
    title = item.find('h2').text
    link = item.find('a')['href']
    data.append({'title': title, 'link': link})

# 打印结果
print(data)

import requests
from bs4 import BeautifulSoup

# 1. 获取登录界面的 HTML 源码并解析表单数据
url_login = 'http://XXX/login.html'
r = requests.get(url_login)
bs = BeautifulSoup(r.text, 'html.parser')

form = bs.find('form', {'id': 'login-form'})
action = form['action']
method = form['method']
inputs = form.find_all('input')
data_login = {}
for input in inputs:
    if input.has_attr('name'):
        name = input['name']
        value = input.get('value', '')
        data_login[name] = value

# 2. 填写表单数据并发送登录请求
data_login['username'] = 'your_username'
data_login['password'] = 'your_password'
r = requests.post(action, data=data_login)

# 3. 判断是否登录成功
if r.status_code == 200 and '登录成功' in r.text:
    print('登录成功！')
else:
    print('登录失败！')
    exit(1)

# 4. 抓取目标页面的 HTML 源码并使用 BeautifulSoup 进行解析
url_target = 'http://XXX/target.html'
r = requests.get(url_target)
bs = BeautifulSoup(r.text, 'html.parser')

data = []
items = bs.find_all('div', {'class': 'item'})
for item in items:
    title = item.find('h2').text
    link = item.find('a')['href']
    data.append({'title': title, 'link': link})

# 5. 打印结果
print(data)