利用Python requests库爬取高德地图全国地铁站点信息

import requests

protocol://host[:port]/path/[?query]#fragment
http://www.itcast.cn/index.html?name=andy&age=18#link

# 首先引入所需要的包
import requests
from bs4 import BeautifulSoup
import pandas as pd
import json
# 发送 GET 请求获取网页内容
url = 'http://map.amap.com/subway/index.html'
response = requests.get(url)
# 第一步：爬取两个 div 中的城市数据（包括 ID 和拼音），生成城市集合
if response.status_code == 200:
    # 解码
    response_content = response.content.decode('utf-8')
    # 使用 Beautiful Soup 解析网页内容
    soup = BeautifulSoup(response_content, 'html.parser')
    # 从这里开始，你可以使用 Beautiful Soup 对象（soup）来提取所需的信息
    # 例如，查找标题
    title = soup.title
    # 通过Beautiful Soup来找到城市信息元素，并提取这个元素的信息
    for soup_a in soup.find('div', class_='city-list fl').find_all('a'):
        city_name_py = soup_a['cityname']
        city_id = soup_a['id']
        city_name_ch = soup_a.get_text()
        city_info_list.append({'name_py': city_name_py, 'id': city_id, 'name_ch': city_name_ch})
        # 获取未显示出来的城市列表
    for soup_a in soup.find('div', class_='more-city-list').find_all('a'):
        city_name_py = soup_a['cityname']
        city_id = soup_a['id']
        city_name_ch = soup_a.get_text()
        city_info_list.append({'name_py': city_name_py, 'id': city_id, 'name_ch': city_name_ch})
        print(city_info_list)
else:
    print("无法获取网页内容")
for city_info in city_info_list:
    city_id = city_info.get("id")
    city_name = city_info.get("name_py")
    city_name_ch = city_info.get("name_ch")
    print("开始爬取城市" + city_name_ch + "的数据")
    city_lines_list = []
    # 第二步：遍历城市集合，构造每一个城市的 url,并下载数据
    # 构造每个城市的url
    url = "http://map.amap.com/service/subway?_1717380520536&srhdata=" + city_id + '_drw_' + city_name + '.json'
    res = requests.get(url)
    content = res.content.decode('utf-8')
    # 将内容字符串转换成json对象
    content_json = json.loads(content)
    # 提取该城市的所有地铁线list
    line_info_list = content_json.get("l")
    # 第三步：开始处理每一个地铁线，提取内容到dataframe中
    for line_info in line_info_list:
        # 地铁线名字
        line_name = line_info["kn"]
        # 处理地铁线站点
        df_per_zd = pd.DataFrame(line_info["st"])
        df_per_zd = df_per_zd[['n', 'sl', 'poiid', 'sp', 't', 'su', 'sid']]
        df_per_zd['gd经度'] = df_per_zd['sl'].apply(lambda x: x.split(',')[0])
        df_per_zd['gd纬度'] = df_per_zd['sl'].apply(lambda x: x.split(',')[1])
        df_per_zd.drop('sl', axis=1, inplace=True)
        df_per_zd['路线名称'] = line_info['ln']
        df_per_zd['城市名称'] = city_name_ch
        df_per_zd.rename(columns={"n": "站点名称", "poiid": "POI编号", "sp": "拼音名称", "t": "换乘标志 1:换乘，0:不可换乘", "su": "su", "sid": "sid编号"}, inplace=True)
        # 先将这条地铁线处理过的dataframe存起来，我们后面给他放到一张表里
        city_lines_list.append(df_per_zd)
    # 这段代码就是将地铁线数据列表聚合到一张表里，形成每个城市的地铁站数据
    city_subway_data = pd.concat(city_lines_list, ignore_index=True)
    # 第四步：将处理好的文件保存为xlsx
    city_subway_data.to_excel(city_name_ch + '.xlsx', sheet_name='Sheet1')