利用Python爬取高德地图全国地铁站点信息
利用Python中的requests库进行地铁站点信息的获取,同时将数据保存在本机excel中
# 首先引入所需要的包
import requests
from bs4 import BeautifulSoup
import pandas as pd
import json# 发送 GET 请求获取网页内容
url = 'http://map.amap.com/subway/index.html'
response = requests.get(url)
# 第一步：爬取两个 div 中的城市数据（包括 ID 和拼音），生成城市集合
if response.status_code == 200:# 解码response_content = response.content.decode('utf-8')# 使用 Beautiful Soup 解析网页内容soup = BeautifulSoup(response_content, 'html.parser')# 从这里开始，你可以使用 Beautiful Soup 对象（soup）来提取所需的信息# 例如，查找标题title = soup.title# 通过Beautiful Soup来找到城市信息元素，并提取这个元素的信息for soup_a in soup.find('div', class_='city-list fl').find_all('a'):city_name_py = soup_a['cityname']city_id = soup_a['id']city_name_ch = soup_a.get_text()city_info_list.append({'name_py': city_name_py, 'id': city_id, 'name_ch': city_name_ch})# 获取未显示出来的城市列表for soup_a in soup.find('div', class_='more-city-list').find_all('a'):city_name_py = soup_a['cityname']city_id = soup_a['id']city_name_ch = soup_a.get_text()city_info_list.append({'name_py': city_name_py, 'id': city_id, 'name_ch': city_name_ch})print(city_info_list)
else:print("无法获取网页内容")for city_info in city_info_list:city_id = city_info.get("id")city_name = city_info.get("name_py")city_name_ch = city_info.get("name_ch")print("开始爬取城市" + city_name_ch + "的数据")city_lines_list = []# 第二步：遍历城市集合，构造每一个城市的 url,并下载数据# 构造每个城市的urlurl = "http://map.amap.com/service/subway?_1717380520536&srhdata=" + city_id + '_drw_' + city_name + '.json'res = requests.get(url)content = res.content.decode('utf-8')# 将内容字符串转换成json对象content_json = json.loads(content)# 提取该城市的所有地铁线listline_info_list = content_json.get("l")# 第三步：开始处理每一个地铁线，提取内容到dataframe中for line_info in line_info_list:# 地铁线名字line_name = line_info["kn"]# 处理地铁线站点df_per_zd = pd.DataFrame(line_info["st"])df_per_zd = df_per_zd[['n', 'sl', 'poiid', 'sp', 't', 'su', 'sid']]df_per_zd['gd经度'] = df_per_zd['sl'].apply(lambda x: x.split(',')[0])df_per_zd['gd纬度'] = df_per_zd['sl'].apply(lambda x: x.split(',')[1])df_per_zd.drop('sl', axis=1, inplace=True)df_per_zd['路线名称'] = line_info['ln']df_per_zd['城市名称'] = city_name_chdf_per_zd.rename(columns={"n": "站点名称", "poiid": "POI编号", "sp": "拼音名称", "t": "换乘标志 1:换乘，0:不可换乘", "su": "su", "sid": "sid编号"}, inplace=True)# 先将这条地铁线处理过的dataframe存起来，我们后面给他放到一张表里city_lines_list.append(df_per_zd)# 这段代码就是将地铁线数据列表聚合到一张表里，形成每个城市的地铁站数据city_subway_data = pd.concat(city_lines_list, ignore_index=True)# 第四步：将处理好的文件保存为xlsxcity_subway_data.to_excel(city_name_ch + '.xlsx', sheet_name='Sheet1')