当前位置：首页 > news >正文

python实现http get pos download

news 来源：原创 2024/9/20 0:04:59

python实现http get post download
使用requests, 加上重试机制,超时机制.

#!/usr/bin/python3
# -*- coding: UTF-8 -*-import requests
import sys
import json
import os
import logging
import timedef httpGet(urlStr, headers = None, params = None, data = None, isRaiseNone200=False):print('url:' + urlStr)print('params:', params)MAX_RETRY_COUT = 5# 加个重试次数,可根据需要定制哪些情况下进行重试,如果不重试就把raise的注释去掉for retTryPos in range(MAX_RETRY_COUT):try:# 这里timeout要加,设置连接与访问超时时间,否则可能遇到http卡死挂住的情况res = requests.request('get', urlStr, params=params, headers=headers, data=data, timeout=(15, 20))content = res.textprint(content)if res.status_code != 200:if isRaiseNone200:raise ValueError('http不是200')else:print('重试')time.sleep(5)continueelse:return contentexcept requests.exceptions.ConnectionError as e:logging.error('网络连接异常: ', e)print('重试')time.sleep(5)#raiseexcept requests.exceptions.Timeout as e:logging.error('连接超时: ', e)print('重试')time.sleep(5)#raiseexcept requests.exceptions.RequestException as e:logging.error('请求异常: ', e)time.sleep(5)#raiseexcept requests.exceptions.HTTPError as e:logging.error(f'HTTP错误, 状态码: {e.response.status_code}, {e}')time.sleep(5)#raiseexcept ValueError as e:logging.error('响应解析异常: ', e)print('重试')time.sleep(5)#raiseraise ValueError('超出重试次数')def httpDownload(urlStr, saveFile):print('download url:' + urlStr)savePath = os.path.dirname(saveFile)if not os.path.exists(savePath):print('mkdir:', savePath)os.makedirs(str(savePath))r = requests.get(urlStr, timeout=(15, 30))if r.status_code != 200:print('http返回:', r.content)print('http code:', r.status_code)raise ValueError('code不是200')tmpFile = saveFile + '.tmp'with open(tmpFile, 'wb') as f:f.write(r.content)f.close()os.rename(tmpFile, saveFile)def httpPost(urlStr, headers, postData):print('url:' + urlStr)try:token = getToken()datas = json.dumps(postData)rc = requests.post(url=urlStr, data=datas, headers=headers, timeout=(8, 20))print('post:' + str(datas))content = (rc.text)print(content)if rc.status_code != 200:raise ValueError('code不是200')return contentexcept requests.exceptions.ConnectionError as e:logging.error('网络连接异常: ', e)time.sleep(5)#raiseexcept requests.exceptions.Timeout as e:logging.error('连接超时: ', e)time.sleep(5)#raiseexcept requests.exceptions.RequestException as e:logging.error('请求异常: ', e)time.sleep(5)#raiseexcept requests.exceptions.HTTPError as e:logging.error(f'HTTP错误, 状态码: {e.response.status_code}, {e}')time.sleep(5)#raiseexcept ValueError as e:logging.error('响应解析异常: ', e)time.sleep(5)#raiseraise ValueError('超出重试次数')if __name__ == '__main__':httpGet('http://www.baidu.com')