التجميعات

import org.apache.spark.sql.functions._

df.agg(
  count("*").as("rows"),
  avg("age").as("avg_age")
).show()

import org.apache.spark.sql.functions._

val byCity = df.groupBy("city").agg(count("*").as("people"))
byCity.show()

import org.apache.spark.sql.functions._

df.groupBy("department").agg(
  sum("salary").as("total"),
  avg("salary").as("avg"),
  max("salary").as("top")
).show()

import org.apache.spark.sql.functions._

df.agg(countDistinct("city").as("unique_cities")).show()

import org.apache.spark.sql.functions._

df.groupBy("city")
  .agg(count("*").as("n"))
  .filter(col("n") > 100)
  .show()

df.createOrReplaceTempView("people")
spark.sql(
  "SELECT city, COUNT(*) AS n FROM people GROUP BY city HAVING COUNT(*) > 100"
).show()

import org.apache.spark.sql.functions._

sales.groupBy("region")
     .pivot("quarter")
     .agg(sum("amount"))
     .show()

import org.apache.spark.sql.expressions.Window
import org.apache.spark.sql.functions._

val w = Window.partitionBy("dept").orderBy(col("salary").desc)
df.withColumn("rank", rank().over(w)).show()

val pairs = sc.parallelize(Seq(("a", 10), ("a", 20), ("b", 5)))
val sums  = pairs.reduceByKey(_ + _)
// (a, 30), (b, 5)

object Main {
  def main(args: Array[String]): Unit = {
    val data = Seq(("a", 10), ("a", 20), ("b", 5))
    val sums = data.groupBy(_._1).map { case (k, v) => k -> v.map(_._2).sum }
    sums.toSeq.sortBy(_._1).foreach { case (k, s) => println(s"$k: $s") }
  }
}