spark.read.format("csv")
.option("nullValue", "")
.option("quote", "")
.option("header", "true")
.load(s"file:/root/test/dir")         //HDFS :       .load(s"/root/test/networks.csv")
.createOrReplaceTempView("v1")

spark.table("v1").show(5)
val l = spark.table("v1").count()
println(s"l = ${l}")

spark.sql("INSERT OVERWRITE TABLE xxxx select * from v1")


res.coalesce(1).write.option("header", "true").csv(s"/root/test/res")
res.coalesce(1).write.option("header", "true").csv(s"hdfs://xxxxxx:9000/test/network/res")

 /test/makt

val res2 =
spark.sql(
s"""
|select * from 
|xxxx.xxxx limit 1;
|""".stripMargin)


res2.coalesce(1).write.option("header", "true").csv(s"file:/root/test/res2")

res2.coalesce(1).write.csv("/root/test/res.csv")


hdfs dfs -get /root/test/res/xxxx /root/test

更多推荐