IP Learning tsg项目 spark版本首次提交

ip-learning-spark/src/main/scala/cn/ac/iie/service/update/UpdateDocument.scala

@@ -1,5 +1,7 @@
 package cn.ac.iie.service.update
+
 import java.util
+import java.util.concurrent.ConcurrentHashMap
 
 import cn.ac.iie.config.ApplicationConfig
 import cn.ac.iie.dao.BaseArangoData
@@ -10,6 +12,7 @@ import cn.ac.iie.utils.ArangoDBConnect
 import cn.ac.iie.utils.SparkSessionUtil.spark
 import com.arangodb.entity.{BaseDocument, BaseEdgeDocument}
 import org.apache.spark.TaskContext
+import org.apache.spark.broadcast.Broadcast
 import org.apache.spark.rdd.RDD
 import org.apache.spark.sql.Row
 import org.slf4j.LoggerFactory
@@ -22,9 +25,17 @@ object UpdateDocument {
   private val LOG = LoggerFactory.getLogger(UpdateDocument.getClass)
   private val baseArangoData = new BaseArangoData()
 
-  def updateDocument(): Unit ={
+  def updateDocument[T <: BaseDocument](collName: String,
+                                        historyMap: ConcurrentHashMap[Integer, ConcurrentHashMap[String, T]],
+                                        getDocumentRow: Row => T,
+                                        getNewDataRdd: Unit => RDD[Row]
+                                       ): Unit = {
+    baseArangoData.readHistoryData(collName, historyMap, classOf[T])
+    val hisBc = spark.sparkContext.broadcast(historyMap)
+    try {
 
     }
+  }
 
   def updateVertexFqdn(): Unit = {
     baseArangoData.readHistoryData("FQDN", historyVertexFqdnMap, classOf[BaseDocument])