yanggao1119 / simple-scala-spark-gradle Goto Github PK

Hi, if I try replicating the code from SimpleSparkJob in LibrarySuite, it gives me the following error :

Error while instantiating 'org.apache.spark.sql.internal.SessionStateBuilder':
java.lang.IllegalArgumentException: Error while instantiating 'org.apache.spark.sql.internal.SessionStateBuilder':
at org.apache.spark.sql.SparkSession$.org$apache$spark$sql$SparkSession$$instantiateSessionState(SparkSession.scala:1107)
at org.apache.spark.sql.SparkSession.$anonfun$sessionState$2(SparkSession.scala:146)
at scala.Option.getOrElse(Option.scala:189)
at org.apache.spark.sql.SparkSession.sessionState$lzycompute(SparkSession.scala:144)
at org.apache.spark.sql.SparkSession.sessionState(SparkSession.scala:141)
at or

LibrarySuite code I've used :
package com.tutorial.spark
import org.scalatest.funsuite.AnyFunSuite

import org.apache.spark.sql.SparkSession
import org.junit.runner.RunWith
import org.scalatestplus.junit.JUnitRunner

@RunWith(classOf[JUnitRunner])
class LibrarySuite extends AnyFunSuite {

test("someLibraryMethod is always true") {
val logFile = "D:\\Downloads\\gradlescalaspark\\simple-scala-spark-gradle\\README.md" // Should be some file on your system
val spark = SparkSession.builder.appName("Simple Application").master("local[2]").getOrCreate()
val logData = spark.read.textFile(logFile).cache()
val numAs = logData.filter(line => line.contains("a")).count()
val numBs = logData.filter(line => line.contains("b")).count()
println(s"Lines with a: $numAs, Lines with b: $numBs")
  println(spark)
spark.stop()
 def library = new Library()
 assert(library.someLibraryMethod)
 }

}

Any idea what could be causing this?