diff --git a/ip-learning-spark/src/main/scala/cn/ac/iie/spark/rdd/ArangoRdd.scala b/ip-learning-spark/src/main/scala/cn/ac/iie/spark/rdd/ArangoRdd.scala index adf3e1b..4162e76 100644 --- a/ip-learning-spark/src/main/scala/cn/ac/iie/spark/rdd/ArangoRdd.scala +++ b/ip-learning-spark/src/main/scala/cn/ac/iie/spark/rdd/ArangoRdd.scala @@ -46,7 +46,7 @@ class ArangoRdd[T: ClassTag](@transient override val sparkContext: SparkContext, LOG.info(sql) arangoCursor = arangoDB.db(options.database).query(sql,clazz.runtimeClass.asInstanceOf[Class[T]]) }catch { - case e: Exception => LOG.error("创建Cursor异常") + case e: Exception => LOG.error(s"创建Cursor异常:${e.getMessage}") }finally { arangoDB.shutdown() } @@ -66,13 +66,13 @@ class ArangoRdd[T: ClassTag](@transient override val sparkContext: SparkContext, private def getCountTotal: Long = { val arangoDB = spark.createArangoBuilder(options).build() var cnt = 0L - val sql = "RETURN LENGTH(" + options.collection + ")" + val sql = s"RETURN LENGTH(${options.collection})" LOG.info(sql) try { val longs = arangoDB.db(options.database).query(sql, classOf[Long]) while (longs.hasNext) cnt = longs.next } catch { - case e: Exception => LOG.error(sql + "执行异常") + case e: Exception => LOG.error(sql + s"执行异常:${e.getMessage}") }finally { arangoDB.shutdown() } diff --git a/ip-learning-spark/src/test/scala/cn/ac/iie/spark/RDDTest.scala b/ip-learning-spark/src/test/scala/cn/ac/iie/spark/RDDTest.scala index 08cdcf3..4936cd4 100644 --- a/ip-learning-spark/src/test/scala/cn/ac/iie/spark/RDDTest.scala +++ b/ip-learning-spark/src/test/scala/cn/ac/iie/spark/RDDTest.scala @@ -27,6 +27,9 @@ object RDDTest { doc.addAttribute("abc", 1) doc }) + + value.map(doc => {(doc.getKey,doc)}) + value.persist(StorageLevel.MEMORY_AND_DISK) value.foreach(row => println(row.toString))