wanglihui-ip-learning-graph/ip-learning-spark/src/main/resources/application.properties

#spark任务配置
spark.sql.shuffle.partitions=10
spark.executor.memory=4g
spark.executor.cores=1
spark.cores.max=10
spark.local.dir=./tmp
spark.app.name=test
spark.network.timeout=300s
spark.serializer=org.apache.spark.serializer.KryoSerializer
master=local[*]
#spark读取clickhouse配置
spark.read.clickhouse.url=jdbc:clickhouse://192.168.44.12:8123/tsg_galaxy_v3
spark.read.clickhouse.driver=ru.yandex.clickhouse.ClickHouseDriver
spark.read.clickhouse.user=default
spark.read.clickhouse.password=galaxy2019
spark.read.clickhouse.numPartitions=5
spark.read.clickhouse.fetchsize=10
spark.read.clickhouse.partitionColumn=LAST_FOUND_TIME
spark.read.clickhouse.session.table=session_record
spark.read.clickhouse.radius.table=radius_record
clickhouse.socket.timeout=300000
#arangoDB配置
arangoDB.host=192.168.44.12
arangoDB.port=8529
arangoDB.user=root
arangoDB.password=galaxy_2019
arangoDB.DB.name=tsg_galaxy_v3
arangoDB.ttl=3600

thread.pool.number=10

#读取clickhouse时间范围方式，0：读取过去一小时；1：指定时间范围
clickhouse.time.limit.type=0
read.clickhouse.max.time=1634902508
read.clickhouse.min.time=1631759985

update.arango.batch=10000

distinct.client.ip.num=10000
recent.count.hour=24

update.interval=3600
arangodb.total.num=20000000
#读取radius时间范围,与radius任务执行周期一致,单位:分钟
read.radius.granularity=-30
-												first commit

											
										
										
											2020-06-28 18:27:48 +08:00
+								#spark任务配置
-												根据自定义arangoRDD重构代码

											
										
										
											2020-11-10 16:59:39 +08:00
+								spark.sql.shuffle.partitions=10
-												IP Learning tsg项目 spark版本首次提交

											
										
										
											2020-08-06 16:13:59 +08:00
+								spark.executor.memory=4g
-												资源隔离三个任务，增加spark.executor.cores、spark.cores.max资源控制参数。

											
										
										
											2021-10-29 18:54:18 +08:00
+								spark.executor.cores=1
 								spark.cores.max=10
-												[修复] 修复iplearning查询clickhouse的SQL别名与字段相同引起的查询SSL数据为空问题

											
										
										
											2024-10-24 11:42:28 +08:00
+								spark.local.dir=./tmp
-												first commit

											
										
										
											2020-06-28 18:27:48 +08:00
+								spark.app.name=test
 								spark.network.timeout=300s
 								spark.serializer=org.apache.spark.serializer.KryoSerializer
 								master=local[*]
 								#spark读取clickhouse配置
-												增加vsys_id字段 TSG-11902

											
										
										
											2022-09-19 16:21:33 +08:00
+								spark.read.clickhouse.url=jdbc:clickhouse://192.168.44.12:8123/tsg_galaxy_v3
-												IP Learning tsg项目 spark版本首次提交

											
										
										
											2020-08-06 16:13:59 +08:00
+								spark.read.clickhouse.driver=ru.yandex.clickhouse.ClickHouseDriver
 								spark.read.clickhouse.user=default
-												增加vsys_id字段 TSG-11902

											
										
										
											2022-09-19 16:21:33 +08:00
+								spark.read.clickhouse.password=galaxy2019
-												根据自定义arangoRDD重构代码

											
										
										
											2020-11-10 16:59:39 +08:00
+								spark.read.clickhouse.numPartitions=5
-												增加vsys_id字段 TSG-11902

											
										
										
											2022-09-19 16:21:33 +08:00
+								spark.read.clickhouse.fetchsize=10
-												根据自定义arangoRDD重构代码

											
										
										
											2020-11-10 16:59:39 +08:00
+								spark.read.clickhouse.partitionColumn=LAST_FOUND_TIME
-												统一日志术语，修改表名

											
										
										
											2021-09-16 11:08:01 +08:00
+								spark.read.clickhouse.session.table=session_record
 								spark.read.clickhouse.radius.table=radius_record
-												first commit

											
										
										
											2020-06-28 18:27:48 +08:00
+								clickhouse.socket.timeout=300000
 								#arangoDB配置
-												[修复] 修复iplearning查询clickhouse的SQL别名与字段相同引起的查询SSL数据为空问题

											
										
										
											2024-10-24 11:42:28 +08:00
+								arangoDB.host=192.168.44.12
-												first commit

											
										
										
											2020-06-28 18:27:48 +08:00
+								arangoDB.port=8529
-												格式化代码

											
										
										
											2021-03-23 11:26:55 +08:00
+								arangoDB.user=root
-												增加vsys_id字段 TSG-11902

											
										
										
											2022-09-19 16:21:33 +08:00
+								arangoDB.password=galaxy_2019
 								arangoDB.DB.name=tsg_galaxy_v3
-												first commit

											
										
										
											2020-06-28 18:27:48 +08:00
+								arangoDB.ttl=3600
-												根据自定义arangoRDD重构代码

											
										
										
											2020-11-10 16:59:39 +08:00
+								thread.pool.number=10
-												IP Learning tsg项目 spark版本首次提交

											
										
										
											2020-08-06 16:13:59 +08:00
 								#读取clickhouse时间范围方式，0：读取过去一小时；1：指定时间范围
-												[修复] 修复iplearning查询clickhouse的SQL别名与字段相同引起的查询SSL数据为空问题

											
										
										
											2024-10-24 11:42:28 +08:00
+								clickhouse.time.limit.type=0
-												资源隔离三个任务，增加spark.executor.cores、spark.cores.max资源控制参数。

											
										
										
											2021-10-29 18:54:18 +08:00
+								read.clickhouse.max.time=1634902508
 								read.clickhouse.min.time=1631759985
-												IP Learning tsg项目 spark版本首次提交

											
										
										
											2020-08-06 16:13:59 +08:00
 								update.arango.batch=10000
 								distinct.client.ip.num=10000
 								recent.count.hour=24
-												根据自定义arangoRDD重构代码

											
										
										
											2020-11-10 16:59:39 +08:00
+								update.interval=3600
-												统一日志术语，修改表名

											
										
										
											2021-09-16 11:08:01 +08:00
+								arangodb.total.num=20000000
-												资源隔离三个任务，增加spark.executor.cores、spark.cores.max资源控制参数。

											
										
										
											2021-10-29 18:54:18 +08:00
+								#读取radius时间范围,与radius任务执行周期一致,单位:分钟
 								read.radius.granularity=-30