Python使用BeautifulSoup实现网页链接的提取与分析

pip install beautifulsoup4
pip install requests  # 用于获取网页内容

from bs4 import BeautifulSoup
import requests

# 获取网页内容
url = "https://example.com"
response = requests.get(url)
html_content = response.text

# 解析HTML
soup = BeautifulSoup(html_content, 'html.parser')

# 提取所有链接
links = []
for link in soup.find_all('a'):
    href = link.get('href')
    if href:  # 确保href属性存在
        links.append(href)

print(f"Found {len(links)} links:")
for link in links[:10]:  # 打印前10个链接
    print(link)

# 提取特定CSS类的链接
special_links = soup.select('a.special-class')

# 提取特定区域的链接
nav_links = soup.select('nav a')

# 提取特定ID下的链接
footer_links = soup.select('#footer a')

import re

# 只提取包含特定模式的链接
pattern = re.compile(r'\.pdf$')  # 匹配PDF文件
pdf_links = [link for link in links if pattern.search(link)]

from urllib.parse import urljoin

base_url = "https://example.com"
absolute_links = [urljoin(base_url, link) if not link.startswith('http') else link for link in links]

import requests
from bs4 import BeautifulSoup
from urllib.parse import urlparse, urljoin
import pandas as pd
import matplotlib.pyplot as plt
from collections import Counter

def extract_links(url):
    """提取指定URL的所有链接"""
    try:
        response = requests.get(url, timeout=10)
        response.raise_for_status()
    except requests.exceptions.RequestException as e:
        print(f"Error fetching {url}: {e}")
        return []

    soup = BeautifulSoup(response.text, 'html.parser')
    base_url = f"{urlparse(url).scheme}://{urlparse(url).netloc}"

    links = []
    for link in soup.find_all('a'):
        href = link.get('href')
        if href and not href.startswith('javascript'):
            absolute_url = urljoin(base_url, href) if not href.startswith('http') else href
            links.append(absolute_url)

    return links

def analyze_links(links):
    """分析链接数据"""
    # 创建DataFrame
    df = pd.DataFrame(links, columns=['url'])

    # 解析URL组件
    df['domain'] = df['url'].apply(lambda x: urlparse(x).netloc)
    df['path'] = df['url'].apply(lambda x: urlparse(x).path)
    df['extension'] = df['path'].apply(lambda x: x.split('.')[-1] if '.' in x.split('/')[-1] else '')

    # 统计信息
    domain_counts = df['domain'].value_counts()
    extension_counts = df['extension'].value_counts()

    return df, domain_counts, extension_counts

def visualize_data(domain_counts, extension_counts):
    """可视化分析结果"""
    plt.figure(figsize=(12, 6))

    # 域名分布
    plt.subplot(1, 2, 1)
    domain_counts[:10].plot(kind='bar')  # 显示前10个最多出现的域名
    plt.title('Top 10 Domains')
    plt.xlabel('Domain')
    plt.ylabel('Count')

    # 文件类型分布
    plt.subplot(1, 2, 2)
    extension_counts[:5].plot(kind='pie', autopct='%1.1f%%')
    plt.title('File Type Distribution')

    plt.tight_layout()
    plt.show()

if __name__ == "__main__":
    target_url = "https://www.python.org"
    print(f"Extracting links from {target_url}...")

    links = extract_links(target_url)
    print(f"Found {len(links)} links")

    df, domain_counts, extension_counts = analyze_links(links)
    visualize_data(domain_counts, extension_counts)

    # 保存结果
    df.to_csv('extracted_links.csv', index=False)
    print("Analysis complete. Results saved to extracted_links.csv")