资源隔离三个任务,增加spark.executor.cores、spark.cores.max资源控制参数。

This commit is contained in:
wanglihui
2021-10-29 18:54:18 +08:00
parent 264afdaa3e
commit bf707f8b2e
9 changed files with 341 additions and 158 deletions

View File

@@ -1,6 +1,8 @@
#spark任务配置
spark.sql.shuffle.partitions=10
spark.executor.memory=4g
spark.executor.cores=1
spark.cores.max=10
spark.app.name=test
spark.network.timeout=300s
spark.serializer=org.apache.spark.serializer.KryoSerializer
@@ -17,22 +19,19 @@ spark.read.clickhouse.session.table=session_record
spark.read.clickhouse.radius.table=radius_record
clickhouse.socket.timeout=300000
#arangoDB配置
#arangoDB.host=192.168.40.223
arangoDB.host=192.168.44.12
arangoDB.port=8529
arangoDB.user=root
#arangoDB.password=galaxy_2019
arangoDB.password=ceiec2019
arangoDB.DB.name=tsg_galaxy_v3_test
#arangoDB.DB.name=iplearn_media_domain
arangoDB.ttl=3600
thread.pool.number=10
#读取clickhouse时间范围方式0读取过去一小时1指定时间范围
clickhouse.time.limit.type=0
read.clickhouse.max.time=1608518990
read.clickhouse.min.time=1604851201
clickhouse.time.limit.type=1
read.clickhouse.max.time=1634902508
read.clickhouse.min.time=1631759985
update.arango.batch=10000
@@ -40,5 +39,6 @@ distinct.client.ip.num=10000
recent.count.hour=24
update.interval=3600
arangodb.total.num=20000000
#读取radius时间范围,与radius任务执行周期一致,单位:分钟
read.radius.granularity=-30