phatak-dev / anatomy_of_spark_dataframe_api Goto Github PK

I am running StepsInQueryPlanning.scala file.
It works fine with spark version *1.4.0 but with spark version 1.6.0* it is giving exception.

Exception in thread "main" org.apache.spark.sql.catalyst.analysis.UnresolvedException: Invalid call to exprId on unresolved object, tree: 'c2
at org.apache.spark.sql.catalyst.analysis.UnresolvedAttribute.exprId(unresolved.scala:58)
at org.apache.spark.sql.catalyst.expressions.AttributeMap.get(AttributeMap.scala:34)
at org.apache.spark.sql.catalyst.expressions.AttributeMap.get(AttributeMap.scala:31)
at scala.collection.MapLike$class.contains(MapLike.scala:150)
at org.apache.spark.sql.catalyst.expressions.AttributeMap.contains(AttributeMap.scala:31)
at org.apache.spark.sql.catalyst.optimizer.PushPredicateThroughProject$$anonfun$apply$14$$anonfun$21$$anonfun$apply$15.applyOrElse(Optimizer.scala:605)
at org.apache.spark.sql.catalyst.optimizer.PushPredicateThroughProject$$anonfun$apply$14$$anonfun$21$$anonfun$apply$15.applyOrElse(Optimizer.scala:604)
at scala.PartialFunction$Lifted.apply(PartialFunction.scala:218)
at scala.PartialFunction$Lifted.apply(PartialFunction.scala:214)
at org.apache.spark.sql.catalyst.trees.TreeNode$$anonfun$collect$1.apply(TreeNode.scala:134)
at org.apache.spark.sql.catalyst.trees.TreeNode$$anonfun$collect$1.apply(TreeNode.scala:134)
at org.apache.spark.sql.catalyst.trees.TreeNode.foreach(TreeNode.scala:93)
at org.apache.spark.sql.catalyst.trees.TreeNode$$anonfun$foreach$1.apply(TreeNode.scala:94)
at org.apache.spark.sql.catalyst.trees.TreeNode$$anonfun$foreach$1.apply(TreeNode.scala:94)
at scala.collection.immutable.List.foreach(List.scala:318)
at org.apache.spark.sql.catalyst.trees.TreeNode.foreach(TreeNode.scala:94)
at org.apache.spark.sql.catalyst.trees.TreeNode.collect(TreeNode.scala:134)
at org.apache.spark.sql.catalyst.optimizer.PushPredicateThroughProject$$anonfun$apply$14$$anonfun$21.apply(Optimizer.scala:604)
at org.apache.spark.sql.catalyst.optimizer.PushPredicateThroughProject$$anonfun$apply$14$$anonfun$21.apply(Optimizer.scala:606)
at scala.collection.TraversableLike$$anonfun$partition$1.apply(TraversableLike.scala:321)
at scala.collection.TraversableLike$$anonfun$partition$1.apply(TraversableLike.scala:321)
at scala.collection.immutable.List.foreach(List.scala:318)
at scala.collection.TraversableLike$class.partition(TraversableLike.scala:321)
at scala.collection.AbstractTraversable.partition(Traversable.scala:105)
at org.apache.spark.sql.catalyst.optimizer.PushPredicateThroughProject$$anonfun$apply$14.applyOrElse(Optimizer.scala:604)
at org.apache.spark.sql.catalyst.optimizer.PushPredicateThroughProject$$anonfun$apply$14.applyOrElse(Optimizer.scala:592)
at org.apache.spark.sql.catalyst.trees.TreeNode$$anonfun$3.apply(TreeNode.scala:227)
at org.apache.spark.sql.catalyst.trees.TreeNode$$anonfun$3.apply(TreeNode.scala:227)
at org.apache.spark.sql.catalyst.trees.CurrentOrigin$.withOrigin(TreeNode.scala:51)
at org.apache.spark.sql.catalyst.trees.TreeNode.transformDown(TreeNode.scala:226)
at org.apache.spark.sql.catalyst.trees.TreeNode$$anonfun$transformDown$1.apply(TreeNode.scala:232)
at org.apache.spark.sql.catalyst.trees.TreeNode$$anonfun$transformDown$1.apply(TreeNode.scala:232)
at org.apache.spark.sql.catalyst.trees.TreeNode$$anonfun$4.apply(TreeNode.scala:249)
at scala.collection.Iterator$$anon$11.next(Iterator.scala:328)
at scala.collection.Iterator$class.foreach(Iterator.scala:727)
at scala.collection.AbstractIterator.foreach(Iterator.scala:1157)
at scala.collection.generic.Growable$class.$plus$plus$eq(Growable.scala:48)
at scala.collection.mutable.ArrayBuffer.$plus$plus$eq(ArrayBuffer.scala:103)
at scala.collection.mutable.ArrayBuffer.$plus$plus$eq(ArrayBuffer.scala:47)
at scala.collection.TraversableOnce$class.to(TraversableOnce.scala:273)
at scala.collection.AbstractIterator.to(Iterator.scala:1157)
at scala.collection.TraversableOnce$class.toBuffer(TraversableOnce.scala:265)
at scala.collection.AbstractIterator.toBuffer(Iterator.scala:1157)
at scala.collection.TraversableOnce$class.toArray(TraversableOnce.scala:252)
at scala.collection.AbstractIterator.toArray(Iterator.scala:1157)
at org.apache.spark.sql.catalyst.trees.TreeNode.transformChildren(TreeNode.scala:279)
at org.apache.spark.sql.catalyst.trees.TreeNode.transformDown(TreeNode.scala:232)
at org.apache.spark.sql.catalyst.trees.TreeNode.transform(TreeNode.scala:217)
at org.apache.spark.sql.catalyst.optimizer.PushPredicateThroughProject$.apply(Optimizer.scala:592)
at com.madhukaraphatak.spark.dataframe.StepsInQueryPlanning$.main(StepsInQueryPlanning.scala:66)
at com.madhukaraphatak.spark.dataframe.StepsInQueryPlanning.main(StepsInQueryPlanning.scala)

In Spark 2.0, the function Util.getAnalyzer will throw java.util.NoSuchElementException: next on empty iterator

def fields = members.filter(_.typeSignature <:< typeOf[Analyzer]) // no clazz called Analyzer in SQLContext field.
val symbol = fields.head // empty fields.

http://stackoverflow.com/questions/33544385/how-to-change-binary-file-into-rdd-or-dataframe

I will delete this question later.