dataframe - 找不到表或视图 - Spark Scala

Question

我是大数据的初学者，我正在使用 spark-scala。我使用数据框并让我清楚地了解我使用多个 scala 对象来编写我的代码。所有类都有运行代码的主要方法。第一个 scala 对象用于将文件加载到数据帧中，其他 scala 对象进行统计计算。这是第一个的一些代码

object LoadFiles {



  //classes for datasets

  case class T(X: Option[String], P: Option[String],Y:Option[String])
     println("Load File 1 into dataframe")

def main(args: Array[String]){

 val sc = new SparkContext("local[*]", "LoadFiles1") 
 val sqlContext = new org.apache.spark.sql.SQLContext(sc)
 val warehouseLocation = new File("spark-warehouse").getAbsolutePath
 val spark = SparkSession
 .builder()
 .appName("Spark Hive Example")
 .config("spark.sql.warehouse.dir", warehouseLocation)
 .enableHiveSupport()
 .getOrCreate()


import sqlContext.implicits._
import org.apache.spark.sql._
import org.apache.spark.sql.Row;
import org.apache.spark.sql.types.{StructType,StructField,StringType};


  val dataframe1 = sc.textFile("file1.ttl").map(_.split(" |\\ . ")).map(p => 
 T(Try(p(0).toString()).toOption,Try(p(1).toString()).toOption,Try(p(2).toString()).toOption)).toDF()


  dataframe1
  .write
  .partitionBy("Predicate")
  .mode(SaveMode.Overwrite)
  .saveAsTable("dataframe1")
   }}

其他 scala 对象用于从加载的数据帧进行许多计算并创建其他数据帧

这是第二个

object Statistics1 {


  def main(args: Array[String]) {
     val sc = new SparkContext("local[*]", "LoadFiles1") 
     val sqlContext = new org.apache.spark.sql.SQLContext(sc)
     val warehouseLocation = new File("spark-warehouse").getAbsolutePath
     val spark = SparkSession
     .builder()
     .appName("Spark Hive Example")
     .config("spark.sql.warehouse.dir", warehouseLocation)
     .enableHiveSupport()
     .getOrCreate()


import sqlContext.implicits._
import org.apache.spark.sql._
import org.apache.spark.sql.Row;
import org.apache.spark.sql.types.{StructType,StructField,StringType};


   // subject query
    val Query1 = spark.sql("SELECT X As Res, P as Pred, COUNT(Y) As nbrFROM dataframe1 GROUP BY X, P")
    .write
    .mode(SaveMode.Overwrite)
    .saveAsTable("stat1") }}

我得到了错误Exception in thread "main" org.apache.spark.sql.AnalysisException: Table or view not found: dataframe1; line 1 pos 75

我怎样才能解决这个问题 ？

标签： dataframeapache-spark-sql