当前位置：首页 > news >正文

Java 爬虫 jvppeteer

news 来源：原创 2024/5/20 11:16:50

开源地址：https://gitee.com/fanyong920/jvppeteer

maven依赖：

        <!-- java爬虫 --><dependency><groupId>io.github.fanyong920</groupId><artifactId>jvppeteer</artifactId><version>1.1.3</version></dependency>

首次先下载Chrome启动器：

//自动下载，第一次下载后不会再下载
//BrowserFetcher.downloadIfNotExist(null);

如果失败按命令提示手动下载安装即可。

常用示例代码：

package com.java.jvppeteer.controller;import com.java.jvppeteer.service.CommonService;
import com.ruiyun.jvppeteer.core.Puppeteer;
import com.ruiyun.jvppeteer.core.browser.Browser;
import com.ruiyun.jvppeteer.core.page.Page;
import com.ruiyun.jvppeteer.options.*;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.http.HttpStatus;
import org.springframework.http.ResponseEntity;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.PostMapping;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RestController;import java.util.ArrayList;@RestController
@RequestMapping(value = "/common")
public class CommonController {@PostMapping("/hello")public ResponseEntity<?> sayHello() {String value = "hello world!";return new ResponseEntity<>(value, HttpStatus.OK);}/*** 启动浏览器*/@GetMapping("/start")public void start() {try {//设置基本的启动配置,这里选择了‘有头’模式启动ArrayList<String> argList = new ArrayList<>();//自动下载，第一次下载后不会再下载//BrowserFetcher.downloadIfNotExist(null);LaunchOptions options = new LaunchOptionsBuilder().withArgs(argList).withHeadless(false).build();argList.add("--no-sandbox");argList.add("--disable-setuid-sandbox");Puppeteer.launch(options);} catch (Exception e) {throw new RuntimeException(e);}}/*** 页面跳转* @param* @return void*/@GetMapping("/go")public void go() {try {//自动下载，第一次下载后不会再下载//BrowserFetcher.downloadIfNotExist(null);ArrayList<String> argList = new ArrayList<>();// withHeadless 是否开启无头模式，无头模式不会显示浏览器LaunchOptions options = new LaunchOptionsBuilder().withArgs(argList).withHeadless(false).build();argList.add("--no-sandbox");argList.add("--disable-setuid-sandbox");Browser browser = Puppeteer.launch(options);Page page = browser.newPage();page.goTo("https://www.baidu.com/");} catch(Exception e){e.printStackTrace();}}/*** 生成pdf* @param* @return void*/@GetMapping("/pdf")public void pdf() {try {ArrayList<String> arrayList = new ArrayList<>();//生成pdf必须在无头模式下才能生效LaunchOptions options = new LaunchOptionsBuilder().withArgs(arrayList).withHeadless(true).build();arrayList.add("--no-sandbox");arrayList.add("--disable-setuid-sandbox");Browser browser = Puppeteer.launch(options);Page page = browser.newPage();page.goTo("https://gitee.com/fanyong920/jvppeteer");PDFOptions pdfOptions = new PDFOptions();pdfOptions.setPath("/Users/mac/BOOTCAMP/jayce/jvppeteer/test.pdf");page.pdf(pdfOptions);page.close();} catch(Exception e){e.printStackTrace();}}/*** 性能分析* @param* @return void*/@GetMapping("/tracing")public void tracing() {try {ArrayList<String> argList = new ArrayList<>();LaunchOptions options = new LaunchOptionsBuilder().withArgs(argList).withHeadless(true).build();argList.add("--no-sandbox");argList.add("--disable-setuid-sandbox");Browser browser = Puppeteer.launch(options);Page page = browser.newPage();//开启追踪page.tracing().start("/Users/mac/BOOTCAMP/jayce/jvppeteer/trace.json");page.goTo("https://www.baidu.com/?tn=98012088_10_dg&ch=3");page.tracing().stop();} catch(Exception e){e.printStackTrace();}}/*** 页面截图*/@GetMapping("/screenshot")public void screenshot() {try {ArrayList<String> arrayList = new ArrayList<>();LaunchOptions options = new LaunchOptionsBuilder().withArgs(arrayList).withHeadless(true).build();arrayList.add("--no-sandbox");arrayList.add("--disable-setuid-sandbox");Browser browser = Puppeteer.launch(options);Page page = browser.newPage();page.goTo("https://www.baidu.com/?tn=98012088_10_dg&ch=3");ScreenshotOptions screenshotOptions = new ScreenshotOptions();//设置截图范围Clip clip = new Clip(1.0,1.56,400,400);screenshotOptions.setClip(clip);//设置存放的路径screenshotOptions.setPath("/Users/mac/BOOTCAMP/jayce/jvppeteer/test.png");page.screenshot(screenshotOptions);} catch(Exception e){e.printStackTrace();}}
}