2020-06-28 18:27:48 +08:00
|
|
|
|
#spark任务配置
|
2020-08-06 16:13:59 +08:00
|
|
|
|
spark.sql.shuffle.partitions=5
|
|
|
|
|
|
spark.executor.memory=4g
|
2020-06-28 18:27:48 +08:00
|
|
|
|
spark.app.name=test
|
|
|
|
|
|
spark.network.timeout=300s
|
|
|
|
|
|
repartitionNumber=36
|
|
|
|
|
|
spark.serializer=org.apache.spark.serializer.KryoSerializer
|
|
|
|
|
|
master=local[*]
|
|
|
|
|
|
#spark读取clickhouse配置
|
2020-08-11 15:18:45 +08:00
|
|
|
|
#spark.read.clickhouse.url=jdbc:clickhouse://192.168.40.186:8123/tsg_galaxy_v3
|
|
|
|
|
|
spark.read.clickhouse.url=jdbc:clickhouse://192.168.40.193:8123/tsg_galaxy_zx
|
2020-08-06 16:13:59 +08:00
|
|
|
|
spark.read.clickhouse.driver=ru.yandex.clickhouse.ClickHouseDriver
|
|
|
|
|
|
spark.read.clickhouse.user=default
|
|
|
|
|
|
spark.read.clickhouse.password=111111
|
|
|
|
|
|
spark.read.clickhouse.numPartitions=144
|
|
|
|
|
|
spark.read.clickhouse.fetchsize=10000
|
2020-08-11 15:18:45 +08:00
|
|
|
|
spark.read.clickhouse.partitionColumn=recv_time
|
2020-06-28 18:27:48 +08:00
|
|
|
|
clickhouse.socket.timeout=300000
|
|
|
|
|
|
#arangoDB配置
|
2020-08-06 16:13:59 +08:00
|
|
|
|
arangoDB.host=192.168.40.182
|
2020-06-28 18:27:48 +08:00
|
|
|
|
arangoDB.port=8529
|
2020-08-06 16:13:59 +08:00
|
|
|
|
arangoDB.user=upsert
|
|
|
|
|
|
arangoDB.password=ceiec2018
|
|
|
|
|
|
#arangoDB.DB.name=insert_iplearn_index
|
|
|
|
|
|
arangoDB.DB.name=ip-learning-test-0
|
2020-06-28 18:27:48 +08:00
|
|
|
|
arangoDB.ttl=3600
|
|
|
|
|
|
|
2020-08-06 16:13:59 +08:00
|
|
|
|
thread.pool.number=5
|
|
|
|
|
|
|
|
|
|
|
|
#读取clickhouse时间范围方式，0：读取过去一小时；1：指定时间范围
|
2020-08-11 15:18:45 +08:00
|
|
|
|
clickhouse.time.limit.type=1
|
|
|
|
|
|
read.clickhouse.max.time=1571241720
|
|
|
|
|
|
read.clickhouse.min.time=1571241600
|
2020-08-06 16:13:59 +08:00
|
|
|
|
|
|
|
|
|
|
#读取arangoDB时间范围方式，0：正常读；1：指定时间范围
|
|
|
|
|
|
arango.time.limit.type=0
|
|
|
|
|
|
read.arango.max.time=1571245320
|
|
|
|
|
|
read.arango.min.time=1571245200
|
|
|
|
|
|
|
|
|
|
|
|
arangoDB.read.limit=
|
|
|
|
|
|
update.arango.batch=10000
|
|
|
|
|
|
|
|
|
|
|
|
distinct.client.ip.num=10000
|
|
|
|
|
|
recent.count.hour=24
|
|
|
|
|
|
|
2020-08-10 18:38:15 +08:00
|
|
|
|
update.interval=10800
|
