在BeautifulSoup中使用NLP技术进行网页内容的分析和提取

from bs4 import BeautifulSoup
import urllib.request
import nltk
from nltk.corpus import stopwords

# 获取网页内容
url = "https://pidancode.com/"
html = urllib.request.urlopen(url).read()

# 使用BeautifulSoup解析网页内容
soup = BeautifulSoup(html, "html.parser")

# 提取网页中的文本
text = soup.get_text(strip=True)

# 对文本进行分词和词性标注
tokens = nltk.word_tokenize(text)
tags = nltk.pos_tag(tokens)

# 过滤停用词和标点符号
stop_words = set(stopwords.words('english'))
filter_tags = [word for word, tag in tags if word.lower() not in stop_words and tag.isalpha()]

# 统计词频，并排序输出前20个关键词
freq_dist = nltk.FreqDist(filter_tags)
for word, frequency in freq_dist.most_common(20):
    print(f"{word}: {frequency}")

code: 24
Python: 17
programming: 12
learn: 9
language: 5
skills: 5
programming.: 5
web: 5
software: 5
development: 5
PIDA: 4
com: 4
projects: 4
online: 4
courses: 4
make: 4
career: 4
Data: 4
Science: 4
startup: 4