Spark的开发环境配置

1. 介绍

这里主要记录一下，我们常用的maven配置，方便后期开发配置环境，避免每次都从零开始配置工程。

2. 工程目录

3. pom的配置

<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"><modelVersion>4.0.0</modelVersion><groupId>org.example</groupId><artifactId>sparkdemo01</artifactId><version>1.0-SNAPSHOT</version><properties><maven.compiler.source>8</maven.compiler.source><maven.compiler.target>8</maven.compiler.target><scala.version>2.12.7</scala.version><scala.simple.version>2.12</scala.simple.version><spark.version>3.2.2</spark.version><kafka.version>3.1.0</kafka.version><env.scope>compile</env.scope> <!-- 默认作用域 --></properties><dependencies><!-- Scala Library --><dependency><groupId>org.scala-lang</groupId><artifactId>scala-library</artifactId><version>${scala.version}</version></dependency><!-- Apache Spark --><dependency><groupId>org.apache.spark</groupId><artifactId>spark-core_${scala.simple.version}</artifactId><version>${spark.version}</version><scope>${env.scope}</scope></dependency><!-- Spark SQL --><dependency><groupId>org.apache.spark</groupId><artifactId>spark-sql_${scala.simple.version}</artifactId><version>${spark.version}</version></dependency><!-- Spark 处理kafka的数据 --><dependency><groupId>org.apache.spark</groupId><artifactId>spark-streaming-kafka-0-10_${scala.simple.version}</artifactId><version>${spark.version}</version></dependency><dependency><groupId>org.apache.kafka</groupId><artifactId>kafka-clients</artifactId><version>${kafka.version}</version></dependency></dependencies><profiles><!-- Production profile --><profile><id>prd</id><properties><env.scope>provided</env.scope><env>prd</env></properties></profile><!-- Testing profile --><profile><id>tst</id><properties><env.scope>provided</env.scope><env>tst</env></properties></profile><!-- Development profile --><profile><id>dev</id><properties><env.scope>compile</env.scope><env>dev</env></properties><activation><activeByDefault>true</activeByDefault></activation></profile></profiles><build><finalName>sparkdemo01</finalName><resources><resource><directory>src/main/java</directory></resource><resource><directory>src/main/resources</directory><excludes><exclude>var/*</exclude><exclude>var/*</exclude></excludes></resource><resource><directory>src/main/resources/var</directory><includes><include>${env}.properties</include></includes></resource></resources><plugins><!-- Scala Maven Plugin --><plugin><groupId>net.alchim31.maven</groupId><artifactId>scala-maven-plugin</artifactId><version>4.5.3</version><executions><execution><goals><goal>compile</goal><goal>testCompile</goal></goals></execution></executions><configuration><scalaVersion>${scala.version}</scalaVersion><recompileMode>incremental</recompileMode></configuration></plugin><!-- Maven Compiler Plugin --><plugin><groupId>org.apache.maven.plugins</groupId><artifactId>maven-compiler-plugin</artifactId><version>3.8.1</version><configuration><source>${maven.compiler.source}</source><target>${maven.compiler.target}</target></configuration></plugin><plugin><groupId>org.apache.maven.plugins</groupId><artifactId>maven-assembly-plugin</artifactId><version>2.6</version><configuration><descriptorRefs><descriptorRef>jar-with-dependencies</descriptorRef></descriptorRefs></configuration><executions><execution><id>make-assembly</id><phase>package</phase><goals><goal>single</goal></goals></execution></executions></plugin></plugins></build></project>