python获取网页表格的多种方法汇总

import pandas as pd

# 从URL读取
dfs = pd.read_html('http://example.com/some_page_with_tables.html')

# 从文件读取
dfs = pd.read_html('path_to_your_file.html')

# 访问第一个DataFrame
df = dfs[0]

from bs4 import BeautifulSoup
import pandas as pd

# 假设html_doc是你的HTML内容
html_doc = """
<table>
  <tr>
    <th>Column1</th>
    <th>Column2</th>
  </tr>
  <tr>
    <td>Value1</td>
    <td>Value2</td>
  </tr>
</table>
"""

# 使用BeautifulSoup解析HTML
soup = BeautifulSoup(html_doc, 'html.parser')

# 提取表格
table = soup.find('table')

# 提取表头
headers = [th.text for th in table.find_all('th')]

# 提取表格数据
rows = []
for tr in table.find_all('tr')[1:]:  # 跳过表头
    cells = [td.text for td in tr.find_all('td')]
    rows.append(cells)

# 创建DataFrame
df = pd.DataFrame(rows, columns=headers)

from lxml import html

# 假设html_doc是你的HTML内容
html_doc = """
<table>
  <tr>
    <th>Column1</th>
    <th>Column2</th>
  </tr>
  <tr>
    <td>Value1</td>
    <td>Value2</td>
  </tr>
</table>
"""

# 解析HTML
tree = html.fromstring(html_doc)

# 使用XPath提取表格数据
rows = tree.xpath('//tr')

# 提取表头
headers = [header.text_content() for header in rows[0].xpath('.//th')]

# 提取表格数据
data = []
for row in rows[1:]:
    cells = [cell.text_content() for cell in row.xpath('.//td')]
    data.append(cells)

# 创建DataFrame
df = pd.DataFrame(data, columns=headers)

import scrapy

class MySpider(scrapy.Spider):
    name = 'my_spider'
    start_urls = ['http://example.com/some_page_with_tables.html']

    def parse(self, response):
        for table in response.css('table'):
            for row in table.css('tr'):
                columns = row.css('td::text').getall()
                yield {
                    'Column1': columns[0],
                    'Column2': columns[1],
                }

from selenium import webdriver
from selenium.webdriver.common.by import By

driver = webdriver.Chrome()

driver.get("http://example.com/some_page_with_tables.html")

table = driver.find_element(By.TAG_NAME, 'table')

print(table.get_attribute('outerHTML'))

rows = table.find_elements(By.TAG_NAME, 'tr')
for row in rows:
    cells = row.find_elements(By.TAG_NAME, 'td')
    cell_texts = [cell.text for cell in cells]
    print(cell_texts)

driver.quit()

from selenium import webdriver
from selenium.webdriver.common.by import By

# 创建WebDriver实例
driver = webdriver.Chrome()

# 打开目标网页
driver.get("http://example.com/some_page_with_tables.html")

# 定位表格元素
table = driver.find_element(By.TAG_NAME, 'table')

# 方法1：打印整个表格的HTML
print(table.get_attribute('outerHTML'))

# 方法2：遍历并打印表格的每一行和单元格内容
rows = table.find_elements(By.TAG_NAME, 'tr')
for row in rows:
    cells = row.find_elements(By.TAG_NAME, 'td')
    cell_texts = [cell.text for cell in cells]
    print(cell_texts)

# 关闭浏览器
driver.quit()