抽象document父类
This commit is contained in:
wanglihui
@@ -1,74 +1,25 @@
|
||||
package cn.ac.iie.test;
|
||||
|
||||
import cn.ac.iie.dao.BaseArangoData;
|
||||
import cn.ac.iie.dao.BaseClickhouseData;
|
||||
import cn.ac.iie.utils.ArangoDBConnect;
|
||||
import cn.ac.iie.utils.ExecutorThreadPool;
|
||||
import cn.ac.iie.dao.UpdateGraphData;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
import java.util.concurrent.CountDownLatch;
|
||||
|
||||
public class IpLearningApplicationTest {
|
||||
private static final Logger LOG = LoggerFactory.getLogger(IpLearningApplicationTest.class);
|
||||
|
||||
public static void main(String[] args) {
|
||||
long startA = System.currentTimeMillis();
|
||||
long start = System.currentTimeMillis();
|
||||
LOG.info("Ip Learning Application开始运行");
|
||||
BaseArangoData baseArangoData = new BaseArangoData();
|
||||
baseArangoData.baseDocumentDataMap();
|
||||
|
||||
LOG.info("历史数据读取完成,开始更新数据");
|
||||
UpdateGraphData updateGraphData = new UpdateGraphData();
|
||||
updateGraphData.updateArango();
|
||||
long last = System.currentTimeMillis();
|
||||
LOG.info("共计运行时间："+(last - start));
|
||||
|
||||
long lastA = System.currentTimeMillis();
|
||||
System.out.println("读取ArangoDb时间："+(lastA - startA));
|
||||
|
||||
|
||||
/*
|
||||
|
||||
long startC = System.currentTimeMillis();
|
||||
CountDownLatch countDownLatch = new CountDownLatch(4);
|
||||
new Thread(new Runnable() {
|
||||
@Override
|
||||
public void run() {
|
||||
BaseClickhouseData.BaseVFqdn();
|
||||
countDownLatch.countDown();
|
||||
}
|
||||
}).start();
|
||||
|
||||
new Thread(new Runnable() {
|
||||
@Override
|
||||
public void run() {
|
||||
BaseClickhouseData.BaseVIp();
|
||||
countDownLatch.countDown();
|
||||
}
|
||||
}).start();
|
||||
|
||||
new Thread(new Runnable() {
|
||||
@Override
|
||||
public void run() {
|
||||
BaseClickhouseData.BaseEFqdnAddressIp();
|
||||
countDownLatch.countDown();
|
||||
}
|
||||
}).start();
|
||||
|
||||
new Thread(new Runnable() {
|
||||
@Override
|
||||
public void run() {
|
||||
BaseClickhouseData.BaseEIpVisitFqdn();
|
||||
countDownLatch.countDown();
|
||||
}
|
||||
}).start();
|
||||
try {
|
||||
countDownLatch.await();
|
||||
}catch (Exception e){
|
||||
e.printStackTrace();
|
||||
}
|
||||
long lastC = System.currentTimeMillis();
|
||||
System.out.println("更新ArangoDb时间："+(lastC - startC));
|
||||
|
||||
*/
|
||||
|
||||
System.out.println(BaseArangoData.v_Fqdn_Map.size());
|
||||
System.out.println(BaseArangoData.v_Ip_Map.size());
|
||||
System.out.println(BaseArangoData.e_Fqdn_Address_Ip_Map.size());
|
||||
System.out.println(BaseArangoData.e_Ip_Visit_Fqdn_Map.size());
|
||||
|
||||
ArangoDBConnect.clean();
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,95 +0,0 @@
|
||||
package cn.ac.iie.test;
|
||||
|
||||
import cn.ac.iie.config.ApplicationConfig;
|
||||
import com.arangodb.ArangoCursor;
|
||||
import com.arangodb.ArangoDB;
|
||||
import com.arangodb.ArangoDatabase;
|
||||
import com.arangodb.entity.BaseDocument;
|
||||
import com.arangodb.model.AqlQueryOptions;
|
||||
import com.arangodb.util.MapBuilder;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.concurrent.ConcurrentHashMap;
|
||||
import java.util.concurrent.ExecutorService;
|
||||
import java.util.concurrent.Executors;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
|
||||
public class ReadArangoDBThreadTest {
|
||||
|
||||
private static ConcurrentHashMap<String,BaseDocument> fqdnMap = new ConcurrentHashMap<String,BaseDocument>();
|
||||
public static void main(String[] args) throws Exception {
|
||||
final ArangoDB arangoDB = new ArangoDB.Builder()
|
||||
.maxConnections(ApplicationConfig.THREAD_POOL_NUMBER)
|
||||
.host(ApplicationConfig.ARANGODB_HOST, ApplicationConfig.ARANGODB_PORT)
|
||||
.user(ApplicationConfig.ARANGODB_USER)
|
||||
.password(ApplicationConfig.ARANGODB_PASSWORD)
|
||||
.build();
|
||||
Map<String, Object> bindVars = new MapBuilder().get();
|
||||
AqlQueryOptions options = new AqlQueryOptions().ttl(ApplicationConfig.ARANGODB_TTL);
|
||||
|
||||
String sql = "LET FQDN = (FOR doc IN V_FQDN RETURN doc) return {max_time:MAX(FQDN[*].FQDN_FIRST_FOUND_TIME),min_time:MIN(FQDN[*].FQDN_FIRST_FOUND_TIME)}";
|
||||
// String sql = "LET IP = (FOR doc IN V_IP RETURN doc) return {max_time:MAX(IP[*].FIRST_FOUND_TIME),min_time:MIN(IP[*].FIRST_FOUND_TIME)}";
|
||||
final ArangoDatabase db = arangoDB.db("insert_iplearn_index");
|
||||
long startTime = System.currentTimeMillis();
|
||||
ArangoCursor<BaseDocument> timeDoc = db.query(sql, bindVars, options, BaseDocument.class);
|
||||
long maxTime =0L;
|
||||
long minTime =0L;
|
||||
while (timeDoc.hasNext()){
|
||||
BaseDocument doc = timeDoc.next();
|
||||
maxTime = Long.parseLong(doc.getAttribute("max_time").toString()) + ApplicationConfig.THREAD_POOL_NUMBER;
|
||||
minTime = Long.parseLong(doc.getAttribute("min_time").toString());
|
||||
}
|
||||
long lastTime = System.currentTimeMillis();
|
||||
System.out.println("查询最大最小时间用时："+(lastTime-startTime));
|
||||
System.out.println(maxTime + "--" + minTime);
|
||||
final long diffTime = (maxTime - minTime) / ApplicationConfig.THREAD_POOL_NUMBER;
|
||||
ExecutorService pool = Executors.newFixedThreadPool(ApplicationConfig.THREAD_POOL_NUMBER);
|
||||
|
||||
for (int i = 0; i < ApplicationConfig.THREAD_POOL_NUMBER; i++) {
|
||||
// long finalMaxTime = maxTime;
|
||||
final long finalMinTime = minTime;
|
||||
pool.execute(new Runnable() {
|
||||
|
||||
public void run() {
|
||||
String name = Thread.currentThread().getName();
|
||||
ArangoDatabase insert_iplearn_index = arangoDB.db("insert_iplearn_index");
|
||||
Map<String, Object> bindVars = new MapBuilder().get();
|
||||
AqlQueryOptions options = new AqlQueryOptions().ttl(ApplicationConfig.ARANGODB_TTL);
|
||||
String[] split = name.split("-");
|
||||
Long threadNum = Long.parseLong(split[3]);
|
||||
long maxThreadTime = finalMinTime + threadNum * diffTime;
|
||||
long minThreadTime = finalMinTime + (threadNum-1)*diffTime;
|
||||
String query = "FOR doc IN V_FQDN filter doc.FQDN_FIRST_FOUND_TIME >= "+minThreadTime+" and doc.FQDN_FIRST_FOUND_TIME <= "+maxThreadTime+" RETURN doc";
|
||||
// String query = "FOR doc IN V_IP filter doc.FIRST_FOUND_TIME >= "+minThreadTime+" and doc.FIRST_FOUND_TIME <= "+maxThreadTime+" RETURN doc";
|
||||
System.out.println(name+":"+query);
|
||||
long s = System.currentTimeMillis();
|
||||
ArangoCursor<BaseDocument> fqdnDoc = insert_iplearn_index.query(query, bindVars, options, BaseDocument.class);
|
||||
List<BaseDocument> baseDocuments = fqdnDoc.asListRemaining();
|
||||
int i = 0;
|
||||
for (BaseDocument doc:baseDocuments){
|
||||
String key = doc.getKey();
|
||||
// System.out.println(name+":"+key);
|
||||
fqdnMap.put(key,doc);
|
||||
i++;
|
||||
}
|
||||
/*
|
||||
while (fqdnDoc.hasNext()){
|
||||
BaseDocument doc = fqdnDoc.next();
|
||||
}
|
||||
*/
|
||||
System.out.println(name+":"+ i);
|
||||
long l = System.currentTimeMillis();
|
||||
System.out.println(name+"运行时间："+(l-s));
|
||||
}
|
||||
});
|
||||
}
|
||||
pool.shutdown();
|
||||
while (!pool.awaitTermination(20, TimeUnit.SECONDS)){
|
||||
|
||||
}
|
||||
System.out.println(fqdnMap.size());
|
||||
arangoDB.shutdown();
|
||||
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user