维护一组浏览器，实现每分钟1000次查询。DriverPool使用变幻版只初始化一次的单例模式。维护每个浏览器的当前是否使用的状态。
不需要等待请求来了，临时开浏览器，开一个浏览器会耽误6秒钟。
可以在程序启动后，随便使用命令杀死slenium,，不怕被别人杀死，不需要重启程序就能保证长久正常运行。
主要使用了 mixin继承、变化版单例模式、鸭子类、桥接模式、上下文管理器，引入了资源池的概念，自动选择一个当前未被使用的浏览器。
使用了池固定了浏览器最大数量，避免了直接开孤立的slenium driver，当并发大的时候代码突然启动几百上千个浏览器，会导致系统突然性能衰竭。
# coding=utf8 """ 浏览器资源池维护。不需要等待有任务来了，再重开浏览器。新开浏览器会耽误6秒时间。 抗杀抗oom，可以随便在程序启动后，批量杀死浏览器，程序会自动开启。 """ import time import os from pathlib import Path from threading import Lock from urllib.error import URLError from selenium.webdriver import DesiredCapabilities from selenium.common.exceptions import WebDriverException from selenium import webdriver from selenium.webdriver.support.wait import WebDriverWait from app.utils_ydf import LoggerMixin, BoundedThreadPoolExecutor, decorators, LogManager class NoAvailableDriverError(Exception): pass class DriverItem: def __init__(self, driver, ): self.driver = driver self.create_time = time.time() self.is_using = False self.last_use_time = time.time() def __str__(self): # noinspection PyRedundantParentheses return (f"{time.strftime('%Y-%m-%d %H:%M:%S',time.localtime(self.create_time))} {self.is_using} {time.strftime('%Y-%m-%d %H:%M:%S',time.localtime(self.last_use_time))} {self.driver}") class PhantomjsItemBuilder(LoggerMixin): # noinspection PyBroadException def create_a_driver_item(self): t0 = time.time() capabilities = DesiredCapabilities.PHANTOMJS.copy() capabilities['platform'] = "WINDOWS" capabilities['version'] = "10" capabilities['phantomjs.page.settings.loadImages'] = False # capabilities['phantomjs.page.settings.userAgent'] = ( # "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) " # "Chrome/49.0.2623.221 Safari/537.36 SE 2.X MetaSr 1.0") capabilities['phantomjs.page.settings.userAgent'] = ( "Mozilla/5.0 (Linux; Android 6.0; Nexus 5 Build/MRA58N) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.132 Mobile Safari/537.36") service_args = ['--load-images=no', '--disk-cache=yes', '--ignore-ssl-errors=true'] self.logger_with_file.info('创建一个driver中。。。。。。') driver = None if os.name == 'posix': # driver = webdriver.PhantomJS(executable_path=Path(__file__).parent / Path('phantomjs'), desired_capabilities=capabilities, service_args=service_args) try: driver = webdriver.PhantomJS(desired_capabilities=capabilities, service_args=service_args) except Exception as e: self.logger.exception(f'从环境变量获取driver路径失败，改为从/usr/local/bin文件夹获取 {e}') try: driver = webdriver.PhantomJS(executable_path='/usr/local/bin/phantomjs', desired_capabilities=capabilities, service_args=service_args) except Exception as e: self.logger.exception(f'从/usr/local/bin/phantomjs启动失败 {e}') else: driver = webdriver.PhantomJS(desired_capabilities=capabilities, service_args=service_args) # driver.maximize_window() driver.set_window_size(390, 713) driver.set_page_load_timeout(10) # driver.implicitly_wait(10) self.logger.info(f'创建一个浏览器耗时{time.time() - t0}') return DriverItem(driver) class ChromeItemBuilder(LoggerMixin): def create_a_driver_item(self): self.logger.info('创建一个driver中。。。。。。') t0 = time.time() chrome_options = webdriver.ChromeOptions() chrome_options.add_argument('--headless') chrome_options.add_argument('--disable-gpu') chrome_options.add_argument('--disable-images') chrome_options.binary_location = r'C:\Users\Administrator\AppData\Local\Google\Chrome\Application\chrome.exe' # prefs = {"profile.managed_default_content_settings.images": 2} prefs = { 'profile.default_content_setting_values': { # 也可以这样写，两种都正确 # 'profile.default_content_settings': { 'images': 2, # 不加载图片 'javascript': 1, # 2不加载JS "User-Agent": 'Mozilla/5.0 (Linux; Android 6.0; Nexus 5 Build/MRA58N) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.132 Mobile Safari/537.36"', # 更换UA } } chrome_options.add_experimental_option("prefs", prefs) chrome_options.add_argument('blink-settings=imagesEnabled=false') # 这句禁用图片才能生效，上面两个禁用图片没起到效果。 driver = webdriver.Chrome(chrome_options=chrome_options) # driver.maximize_window() driver.set_window_size(390, 713) driver.set_page_load_timeout(100) driver.implicitly_wait(100) self.logger.info(f'创建一个浏览器耗时{time.time() - t0}') return DriverItem(driver) class DriverPool(LoggerMixin): lock = Lock() def __new__(cls, *args, **kwargs): if not hasattr(cls, '_instance'): self = super().__new__(cls, ) cls._instance = self self.__custom_init__(*args, **kwargs) return cls._instance def __custom_init__(self, driver_item_num=10, driver_name=1): """ :param driver_item_num:浏览器数量 :param driver_name: 浏览器种类 1为phantomsj，2为chrome :return: """ self.driver_item_list = list() self._driver_item_num = driver_item_num self.driver_item_builder = PhantomjsItemBuilder() if driver_name == 1 else ChromeItemBuilder() self.logger_with_file.info(f'准备初始化{driver_item_num}个浏览器') self._has_init_all_driver_item = False self._init_time = 0 self._init_all_driver_item() def _init_all_driver_item(self): if time.time() - self._init_time > 60: self._init_time = time.time() self.logger.warning('杀死残留的phantomjs进程') # 此处的命令不用怕误杀其它地方的phantomjs，上下文管理器使用被杀的浏览器会自动启动。 if os.name == 'posix': os.system('ps -aux|grep phantomjs|grep -v grep|cut -c 9-15|xargs kill -9') else: os.system('taskkill /F /im phantomjs.exe') t0 = time.time() self.driver_item_list.clear() # 一定需要清空原来的。 def _inner(this: DriverPool): driver_item = this.driver_item_builder.create_a_driver_item() this.driver_item_list.append(driver_item) thread_pool = BoundedThreadPoolExecutor(self._driver_item_num) [thread_pool.submit(_inner, self) for _ in range(self._driver_item_num)] # 亲测多线程创建10个浏览器，比一个接一个的创建速度要快很多。 thread_pool.shutdown() self._has_init_all_drivers = True self.logger.info(f'所有浏览器初始化创建成功,耗时 {time.time() - t0}秒 {len(self.driver_item_list)} {self.driver_item_list}') def borrow_a_driver_item(self): with self.lock: current_using_number = 0 current_not_using_number = 0 for driver_item in self.driver_item_list: if driver_item.is_using: current_using_number += 1 else: current_not_using_number += 1 self.logger.debug(f'当前正在使用的浏览器数量是{current_using_number},闲置的浏览器数量是{current_not_using_number}') for index, driver_item in enumerate(self.driver_item_list): if driver_item.is_using is False: if time.time() - driver_item.create_time > 3600: self.logger.debug('防止phantomjs内存泄漏，关闭并重新创建一个浏览器') self.driver_item_list.pop(index) driver_item.driver.quit() driver_item = self.driver_item_builder.create_a_driver_item() self.driver_item_list.insert(index, driver_item) driver_item.is_using = True return driver_item raise NoAvailableDriverError('当前没有可用的浏览器。。。。。。。。。。。。') @staticmethod def give_back_a_driver_item(driver_item: DriverItem): driver_item.is_using = False driver_item.last_use_time = time.time() class DriverContext: def __init__(self): self.driver_pool = DriverPool() self.driver_item = None self.start_using_time = time.time() def __enter__(self): self.driver_item = self.driver_pool.borrow_a_driver_item() self.driver_pool.logger_with_file.debug(f'当前使用的浏览器是 {self.driver_item}') return self.driver_item.driver def __exit__(self, exc_type, exc_val, exc_tb): self.driver_pool.logger.info(f'此浏览器 {self.driver_item} 占用时间为 {time.time() - self.start_using_time}秒') self.driver_pool.give_back_a_driver_item(self.driver_item) if exc_type == URLError: # 如果phantomjs被被手动杀死或者oom了，再次使用这个phatntomjs会出这个URLError错,重新生成浏览器池。 self.driver_pool._init_all_driver_item() if exc_type and issubclass(exc_type, WebDriverException): self.driver_pool.logger.error(f'selenium发生错误 ,错误类型--> {exc_type} 错误原因--> {exc_val}') # return True if __name__ == '__main__': logger = LogManager('driver_pool_test').get_logger_and_add_handlers() DriverPool(50) if not Path('/picture').exists(): Path('/picture').mkdir() @decorators.tomorrow_threads(40) def f(): with DriverContext() as driver: # 需要使用with语法来使用浏览器，否则需要手动额外处理一些问题和维护浏览器的使用状态。 logger.debug(f'使用的浏览器是--> {driver}') driver.get('http://m.elong.com/ihotel/283904/?inDate=2018-12-12&outDate=2018-12-13&roomPerson=1|2') driver.save_screenshot(f'/picture/{time.time()}.png') WebDriverWait(driver, 10, 0.2).until( lambda driverx: driverx.find_element_by_css_selector('#detail-mapping-box > li:nth-child(1) > div.prodjh_list_box.clearfix > div.detail-mrooom-mapping-product > div.dprodtname')) logger.info(f'页面内容长度是: {len(driver.page_source)}') driver.save_screenshot(f'/picture/{time.time()}.png') [(time.sleep(0.1), f()) for _ in range(50000)]
使用如图，由于不需要对每次请求都频繁创建和摧毁浏览器，所以打开网页速度很快。
