gfwleak/wanglihui-ip-learning-graph
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

支持arangoDb故障转移

Browse Source
This commit is contained in:
wanglihui
2020-10-13 16:24:55 +08:00
parent e0f5b20ab6
commit 5cb7327f7b
2 changed files with 24 additions and 12 deletions
Download Patch File Download Diff File Expand all files Collapse all files

17
ip-learning-spark/src/main/java/cn/ac/iie/utils/ArangoDBConnect.java
View File

@@ -27,14 +27,27 @@ public class ArangoDBConnect {
} }
private static void getArangoDatabase(){ private static void getArangoDatabase(){
arangoDB = new ArangoDB.Builder() ArangoDB.Builder host = getArangoHost();
arangoDB = host
.maxConnections(ApplicationConfig.THREAD_POOL_NUMBER()) .maxConnections(ApplicationConfig.THREAD_POOL_NUMBER())
.host(ApplicationConfig.ARANGODB_HOST(), ApplicationConfig.ARANGODB_PORT()) .acquireHostList(true)
.user(ApplicationConfig.ARANGODB_USER()) .user(ApplicationConfig.ARANGODB_USER())
.password(ApplicationConfig.ARANGODB_PASSWORD()) .password(ApplicationConfig.ARANGODB_PASSWORD())
.build(); .build();
} }
private static ArangoDB.Builder getArangoHost(){
String hostList = ApplicationConfig.ARANGODB_HOST();
String[] split = hostList.split(",");
ArangoDB.Builder host = new ArangoDB.Builder();
for (String hostStr : split) {
host.host(hostStr, ApplicationConfig.ARANGODB_PORT());
LOG.warn("arangoDB host {} 已添加",hostStr);
}
LOG.warn("获取arangoDB host成功");
return host;
}
public static synchronized ArangoDBConnect getInstance(){ public static synchronized ArangoDBConnect getInstance(){
if (null == conn){ if (null == conn){
conn = new ArangoDBConnect(); conn = new ArangoDBConnect();

19
ip-learning-spark/src/main/resources/application.properties
View File

@@ -3,11 +3,11 @@ spark.sql.shuffle.partitions=10
spark.executor.memory=4g spark.executor.memory=4g
spark.app.name=test spark.app.name=test
spark.network.timeout=300s spark.network.timeout=300s
repartitionNumber=36
spark.serializer=org.apache.spark.serializer.KryoSerializer spark.serializer=org.apache.spark.serializer.KryoSerializer
#spark.serializer=org.apache.spark.serializer.JavaSerializer
master=local[*] master=local[*]
#spark读取clickhouse配置 #spark读取clickhouse配置
spark.read.clickhouse.url=jdbc:clickhouse://192.168.44.12:8123/tsg_galaxy_v3 spark.read.clickhouse.url=jdbc:clickhouse://192.168.44.10:8123/tsg_galaxy_v3
spark.read.clickhouse.driver=ru.yandex.clickhouse.ClickHouseDriver spark.read.clickhouse.driver=ru.yandex.clickhouse.ClickHouseDriver
spark.read.clickhouse.user=default spark.read.clickhouse.user=default
spark.read.clickhouse.password=ceiec2019 spark.read.clickhouse.password=ceiec2019
@@ -16,21 +16,20 @@ spark.read.clickhouse.fetchsize=10000
spark.read.clickhouse.partitionColumn=LAST_FOUND_TIME spark.read.clickhouse.partitionColumn=LAST_FOUND_TIME
clickhouse.socket.timeout=300000 clickhouse.socket.timeout=300000
#arangoDB配置 #arangoDB配置
arangoDB.host=192.168.40.182 arangoDB.host=192.168.40.123,192.168.40.223,192.168.40.222
#arangoDB.host=192.168.40.223
arangoDB.port=8529 arangoDB.port=8529
arangoDB.user=upsert arangoDB.user=upsert
arangoDB.password=ceiec2018 arangoDB.password=ceiec2019
#arangoDB.DB.name=insert_iplearn_index arangoDB.DB.name=tsg_galaxy_v3
arangoDB.DB.name=ip-learning-test-0
#arangoDB.DB.name=ip-learning-test
arangoDB.ttl=3600 arangoDB.ttl=3600
thread.pool.number=10 thread.pool.number=10
#读取clickhouse时间范围方式，0：读取过去一小时；1：指定时间范围 #读取clickhouse时间范围方式，0：读取过去一小时；1：指定时间范围
clickhouse.time.limit.type=0 clickhouse.time.limit.type=1
read.clickhouse.max.time=1600246160 read.clickhouse.max.time=1600916194
read.clickhouse.min.time=1597197469 read.clickhouse.min.time=1599197648
arangoDB.read.sepNum=10 arangoDB.read.sepNum=10
update.arango.batch=10000 update.arango.batch=10000