IP Learning tsg项目 spark版本首次提交

ip-learning-spark/src/main/resources/application.properties

@@ -1,25 +1,45 @@
 #spark任务配置
-spark.sql.shuffle.partitions=144
-spark.sql.read.fetchsize=10000
-spark.executor.memory=120g
+spark.sql.shuffle.partitions=5
+spark.executor.memory=4g
 spark.app.name=test
 spark.network.timeout=300s
 repartitionNumber=36
 spark.serializer=org.apache.spark.serializer.KryoSerializer
 master=local[*]
 #spark读取clickhouse配置
-numPartitions=144
-maxPoolSize=40
-minTime=1571245199
-maxTime=1571284799
+spark.read.clickhouse.url=jdbc:clickhouse://192.168.40.186:8123/tsg_galaxy_v3
+spark.read.clickhouse.driver=ru.yandex.clickhouse.ClickHouseDriver
+spark.read.clickhouse.user=default
+spark.read.clickhouse.password=111111
+spark.read.clickhouse.numPartitions=144
+spark.read.clickhouse.fetchsize=10000
+spark.read.clickhouse.partitionColumn=common_recv_time
 clickhouse.socket.timeout=300000
 #arangoDB配置
-arangoDB.host=192.168.40.127
+arangoDB.host=192.168.40.182
 arangoDB.port=8529
-arangoDB.user=root
-arangoDB.password=111111
-arangoDB.DB.name=insert_iplearn_index
-arangoDB.batch=100000
+arangoDB.user=upsert
+arangoDB.password=ceiec2018
+#arangoDB.DB.name=insert_iplearn_index
+arangoDB.DB.name=ip-learning-test-0
 arangoDB.ttl=3600
 
-thread.pool.number=10
+thread.pool.number=5
+
+#读取clickhouse时间范围方式，0：读取过去一小时；1：指定时间范围
+clickhouse.time.limit.type=0
+read.clickhouse.max.time=1571245220
+read.clickhouse.min.time=1571245210
+
+#读取arangoDB时间范围方式，0：正常读；1：指定时间范围
+arango.time.limit.type=0
+read.arango.max.time=1571245320
+read.arango.min.time=1571245200
+
+arangoDB.read.limit=
+update.arango.batch=10000
+
+distinct.client.ip.num=10000
+recent.count.hour=24
+
+update.interval=3600