gfwleak/wanglihui-ip-learning-graph
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

IP Learning tsg项目 spark版本首次提交

Browse Source
This commit is contained in:
wanglihui
2020-08-07 10:39:22 +08:00
parent 6ea2518ee4
commit 2a4d6dda4a
1 changed files with 68 additions and 57 deletions
Download Patch File Download Diff File Expand all files Collapse all files

13
ip-learning-spark/src/main/scala/cn/ac/iie/service/update/UpdateDocument.scala
View File

@@ -1,5 +1,7 @@
package cn.ac.iie.service.update package cn.ac.iie.service.update
import java.util import java.util
import java.util.concurrent.ConcurrentHashMap
import cn.ac.iie.config.ApplicationConfig import cn.ac.iie.config.ApplicationConfig
import cn.ac.iie.dao.BaseArangoData import cn.ac.iie.dao.BaseArangoData
@@ -10,6 +12,7 @@ import cn.ac.iie.utils.ArangoDBConnect
import cn.ac.iie.utils.SparkSessionUtil.spark import cn.ac.iie.utils.SparkSessionUtil.spark
import com.arangodb.entity.{BaseDocument, BaseEdgeDocument} import com.arangodb.entity.{BaseDocument, BaseEdgeDocument}
import org.apache.spark.TaskContext import org.apache.spark.TaskContext
import org.apache.spark.broadcast.Broadcast
import org.apache.spark.rdd.RDD import org.apache.spark.rdd.RDD
import org.apache.spark.sql.Row import org.apache.spark.sql.Row
import org.slf4j.LoggerFactory import org.slf4j.LoggerFactory
@@ -22,9 +25,17 @@ object UpdateDocument {
private val LOG = LoggerFactory.getLogger(UpdateDocument.getClass) private val LOG = LoggerFactory.getLogger(UpdateDocument.getClass)
private val baseArangoData = new BaseArangoData() private val baseArangoData = new BaseArangoData()
def updateDocument(): Unit ={ def updateDocument[T <: BaseDocument](collName: String,
historyMap: ConcurrentHashMap[Integer, ConcurrentHashMap[String, T]],
getDocumentRow: Row => T,
getNewDataRdd: Unit => RDD[Row]
): Unit = {
baseArangoData.readHistoryData(collName, historyMap, classOf[T])
val hisBc = spark.sparkContext.broadcast(historyMap)
try {
} }
}
def updateVertexFqdn(): Unit = { def updateVertexFqdn(): Unit = {
baseArangoData.readHistoryData("FQDN", historyVertexFqdnMap, classOf[BaseDocument]) baseArangoData.readHistoryData("FQDN", historyVertexFqdnMap, classOf[BaseDocument])