iteratively improving spark application performance
play

Iteratively Improving Spark Application Performance William C. - PowerPoint PPT Presentation

Feb 11, 2023 •210 likes •1.5k views

Iteratively Improving Spark Application Performance William C. Benton Red Hat, Inc. Forecast Background: Spark, RDDs, and Sparks execution model Case study overview Improving our prototype Background Apache Spark Introduced

  1. trait ActivitySliding { import org.apache.spark.rdd.RDD import com.freevariable.surlaplaque.data.{Trackpoint => TP} def windowsForActivities[U](data: RDD[TP], period: Int, data: RDD[TP] period: Int xform: (TP => U) = identity _) = { xform: (TP => U) = identity _ val pairs = data.groupBy((tp:TP) => tp.activity.getOrElse("UNKNOWN")) data.groupBy((tp:TP) => tp.activity.getOrElse("UNKNOWN")) pairs.flatMap { case (activity:String, stp:Seq[TP]) => (stp sliding period).zipWithIndex.map { case (s,i) => ((activity, i), s.map(xform)) } } } def identity(tp: Trackpoint) = tp }

  2. trait ActivitySliding { import org.apache.spark.rdd.RDD import com.freevariable.surlaplaque.data.{Trackpoint => TP} def windowsForActivities[U](data: RDD[TP], period: Int, data: RDD[TP] period: Int xform: (TP => U) = identity _) = { xform: (TP => U) = identity _ val pairs = data.groupBy((tp:TP) => tp.activity.getOrElse("UNKNOWN")) data.groupBy((tp:TP) => tp.activity.getOrElse("UNKNOWN")) pairs.flatMap { pairs.flatMap case (activity:String, stp:Seq[TP]) => (stp sliding period).zipWithIndex.map { case (s,i) => ((activity, i), s.map(xform)) } } } def identity(tp: Trackpoint) = tp }

  3. trait ActivitySliding { import org.apache.spark.rdd.RDD import com.freevariable.surlaplaque.data.{Trackpoint => TP} def windowsForActivities[U](data: RDD[TP], period: Int, data: RDD[TP] period: Int xform: (TP => U) = identity _) = { xform: (TP => U) = identity _ val pairs = data.groupBy((tp:TP) => tp.activity.getOrElse("UNKNOWN")) data.groupBy((tp:TP) => tp.activity.getOrElse("UNKNOWN")) pairs.flatMap { pairs.flatMap case (activity:String, stp:Seq[TP]) => (stp sliding period).zipWithIndex.map { (stp sliding period).zipWithIndex.map { case (s,i) => ((activity, i), s.map(xform)) case (s,i) => ((activity, i), s.map(xform)) } } } } def identity(tp: Trackpoint) = tp }

  4. Transform an RDD of TRACKPOINTS … trait ActivitySliding { import org.apache.spark.rdd.RDD import com.freevariable.surlaplaque.data.{Trackpoint => TP} def windowsForActivities[U](data: RDD[TP], period: Int, data: RDD[TP] period: Int xform: (TP => U) = identity _) = { xform: (TP => U) = identity _ val pairs = data.groupBy((tp:TP) => tp.activity.getOrElse("UNKNOWN")) data.groupBy((tp:TP) => tp.activity.getOrElse("UNKNOWN")) pairs.flatMap { pairs.flatMap case (activity:String, stp:Seq[TP]) => (stp sliding period).zipWithIndex.map { (stp sliding period).zipWithIndex.map { case (s,i) => ((activity, i), s.map(xform)) case (s,i) => ((activity, i), s.map(xform)) } } } } def identity(tp: Trackpoint) = tp } …TO an RDD of WINDOW ID S and SAMPLE WINDOWS

  5. def bestsForPeriod(data: RDD[TP], period: Int, app: SLP, model: KMeansModel) = { val windowedSamples = windowsForActivities(data, period, minify _).cache val clusterPairs = windowedSamples.map { case ((act, idx), samples) => ((act, idx), clusterPairsForWindow(samples, model)) } val mmps = windowedSamples.map { case ((act, idx), samples) => ((act, idx), samples.map(_.watts).reduce(_ + _) / samples.size) } // continued...

  6. def bestsForPeriod(data: RDD[TP], period: Int, app: SLP, model: KMeansModel) = { val windowedSamples = windowsForActivities(data, period, minify _).cache val windowedSamples = windowsForActivities(data, period, minify _).cache val clusterPairs = windowedSamples.map { case ((act, idx), samples) => ((act, idx), clusterPairsForWindow(samples, model)) } val mmps = windowedSamples.map { case ((act, idx), samples) => ((act, idx), samples.map(_.watts).reduce(_ + _) / samples.size) } // continued... Divide the input data into overlapping windows , keyed by ACTIVITY and OFFSET (we’ll call this key a WINDOW ID )

  7. def bestsForPeriod(data: RDD[TP], period: Int, app: SLP, model: KMeansModel) = { val windowedSamples = windowsForActivities(data, period, minify _).cache val clusterPairs = windowedSamples.map { val clusterPairs = windowedSamples.map { case ((act, idx), samples) => case ((act, idx), samples) => ((act, idx), clusterPairsForWindow(samples, model)) ((act, idx), clusterPairsForWindow(samples, model)) } } val mmps = windowedSamples.map { case ((act, idx), samples) => ((act, idx), samples.map(_.watts).reduce(_ + _) / samples.size) } // continued... Identify the spatial clusters that each window starts and ends in

  8. def bestsForPeriod(data: RDD[TP], period: Int, app: SLP, model: KMeansModel) = { val windowedSamples = windowsForActivities(data, period, minify _).cache val clusterPairs = windowedSamples.map { case ((act, idx), samples) => ((act, idx), clusterPairsForWindow(samples, model)) } val mmps = windowedSamples.map { val mmps = windowedSamples.map { case ((act, idx), samples) => case ((act, idx), samples) => ((act, idx), samples.map(_.watts).reduce(_ + _) / samples.size) ((act, idx), samples.map(_.watts).reduce(_ + _) / samples.size) } } // continued... Identify the MEAN WATTAGE for each window of samples

  9. def bestsForPeriod(data: RDD[TP], period: Int, app: SLP, model: KMeansModel) = { val windowedSamples = /* window IDs and raw sample windows */ val clusterPairs = /* window IDs and spatial cluster pairs */ val mmps = /* window IDs and mean wattages for each window */ val top20 = mmps.join(clusterPairs) .map { case ((act, idx), (watts, (cls1, cls2))) => ((cls1, cls2), (watts, (act, idx))) } .reduceByKey ((a, b) => if (a._1 > b._1) a else b) .map { case ((cls1, cls2), (watts, (act, idx))) => (watts, (act, idx)) } .sortByKey(false) .take(20) app.context.parallelize(top20) .map { case (watts, (act, idx)) => ((act, idx), watts) } .join (windowedSamples) .map { case ((act, idx), (watts, samples)) => (watts, samples) } .collect }

  10. def bestsForPeriod(data: RDD[TP], period: Int, app: SLP, model: KMeansModel) = { val windowedSamples = /* window IDs and raw sample windows */ val clusterPairs = /* window IDs and spatial cluster pairs */ val mmps = /* window IDs and mean wattages for each window */ val top20 = mmps.join(clusterPairs) mmps.join(clusterPairs) .map { case ((act, idx), (watts, (cls1, cls2))) => ((cls1, cls2), (watts, (act, idx))) } .reduceByKey ((a, b) => if (a._1 > b._1) a else b) .map { case ((cls1, cls2), (watts, (act, idx))) => (watts, (act, idx)) } .sortByKey(false) .take(20) app.context.parallelize(top20) .map { case (watts, (act, idx)) => ((act, idx), watts) } .join (windowedSamples) .map { case ((act, idx), (watts, samples)) => (watts, samples) } .collect } for each window ID , JOIN its mean wattages with its spatial clusters

  11. def bestsForPeriod(data: RDD[TP], period: Int, app: SLP, model: KMeansModel) = { val windowedSamples = /* window IDs and raw sample windows */ val clusterPairs = /* window IDs and spatial cluster pairs */ val mmps = /* window IDs and mean wattages for each window */ val top20 = mmps.join(clusterPairs) .map { case ((act, idx), (watts, (cls1, cls2))) => .map { case ((act, idx), (watts, (cls1, cls2))) => ((cls1, cls2), (watts, (act, idx))) } ((cls1, cls2), (watts, (act, idx))) } .reduceByKey ((a, b) => if (a._1 > b._1) a else b) .map { case ((cls1, cls2), (watts, (act, idx))) => (watts, (act, idx)) } .sortByKey(false) .take(20) app.context.parallelize(top20) .map { case (watts, (act, idx)) => ((act, idx), watts) } .join (windowedSamples) .map { case ((act, idx), (watts, samples)) => (watts, samples) } .collect } transpose these tuples so they are keyed by spatial cluster pairs

  12. def bestsForPeriod(data: RDD[TP], period: Int, app: SLP, model: KMeansModel) = { val windowedSamples = /* window IDs and raw sample windows */ val clusterPairs = /* window IDs and spatial cluster pairs */ val mmps = /* window IDs and mean wattages for each window */ val top20 = mmps.join(clusterPairs) .map { case ((act, idx), (watts, (cls1, cls2))) => ((cls1, cls2), (watts, (act, idx))) } .reduceByKey ((a, b) => if (a._1 > b._1) a else b) .reduceByKey ((a, b) => if (a._1 > b._1) a else b) .map { case ((cls1, cls2), (watts, (act, idx))) => (watts, (act, idx)) } .sortByKey(false) .take(20) app.context.parallelize(top20) .map { case (watts, (act, idx)) => ((act, idx), watts) } .join (windowedSamples) .map { case ((act, idx), (watts, samples)) => (watts, samples) } .collect } KEEP ONLY the BEST wattage for each spatial cluster pair

  13. def bestsForPeriod(data: RDD[TP], period: Int, app: SLP, model: KMeansModel) = { val windowedSamples = /* window IDs and raw sample windows */ val clusterPairs = /* window IDs and spatial cluster pairs */ val mmps = /* window IDs and mean wattages for each window */ val top20 = mmps.join(clusterPairs) .map { case ((act, idx), (watts, (cls1, cls2))) => ((cls1, cls2), (watts, (act, idx))) } .reduceByKey ((a, b) => if (a._1 > b._1) a else b) .map { case ((cls1, cls2), (watts, (act, idx))) => (watts, (act, idx)) } .map { case ((cls1, cls2), (watts, (act, idx))) => (watts, (act, idx)) } .sortByKey(false) .take(20) app.context.parallelize(top20) .map { case (watts, (act, idx)) => ((act, idx), watts) } .join (windowedSamples) .map { case ((act, idx), (watts, samples)) => (watts, samples) } .collect } project away the cluster centers

  14. def bestsForPeriod(data: RDD[TP], period: Int, app: SLP, model: KMeansModel) = { val windowedSamples = /* window IDs and raw sample windows */ val clusterPairs = /* window IDs and spatial cluster pairs */ val mmps = /* window IDs and mean wattages for each window */ val top20 = mmps.join(clusterPairs) .map { case ((act, idx), (watts, (cls1, cls2))) => ((cls1, cls2), (watts, (act, idx))) } .reduceByKey ((a, b) => if (a._1 > b._1) a else b) .map { case ((cls1, cls2), (watts, (act, idx))) => (watts, (act, idx)) } .sortByKey(false) .sortByKey(false) .take(20) .take(20) app.context.parallelize(top20) .map { case (watts, (act, idx)) => ((act, idx), watts) } .join (windowedSamples) .map { case ((act, idx), (watts, samples)) => (watts, samples) } .collect } SORT by wattage in descending order ; keep the best twenty

  15. def bestsForPeriod(data: RDD[TP], period: Int, app: SLP, model: KMeansModel) = { val windowedSamples = /* window IDs and raw sample windows */ val clusterPairs = /* window IDs and spatial cluster pairs */ val mmps = /* window IDs and mean wattages for each window */ val top20 = mmps.join(clusterPairs) .map { case ((act, idx), (watts, (cls1, cls2))) => ((cls1, cls2), (watts, (act, idx))) } .reduceByKey ((a, b) => if (a._1 > b._1) a else b) .map { case ((cls1, cls2), (watts, (act, idx))) => (watts, (act, idx)) } .sortByKey(false) .take(20) app.context.parallelize(top20) .map { case (watts, (act, idx)) => ((act, idx), watts) } .map { case (watts, (act, idx)) => ((act, idx), watts) } .join (windowedSamples) .map { case ((act, idx), (watts, samples)) => (watts, samples) } .collect } Re-key the best efforts by window id

  16. def bestsForPeriod(data: RDD[TP], period: Int, app: SLP, model: KMeansModel) = { val windowedSamples = /* window IDs and raw sample windows */ val clusterPairs = /* window IDs and spatial cluster pairs */ val mmps = /* window IDs and mean wattages for each window */ val top20 = mmps.join(clusterPairs) .map { case ((act, idx), (watts, (cls1, cls2))) => ((cls1, cls2), (watts, (act, idx))) } .reduceByKey ((a, b) => if (a._1 > b._1) a else b) .map { case ((cls1, cls2), (watts, (act, idx))) => (watts, (act, idx)) } .sortByKey(false) .take(20) app.context.parallelize(top20) .map { case (watts, (act, idx)) => ((act, idx), watts) } .join (windowedSamples) .join (windowedSamples) .map { case ((act, idx), (watts, samples)) => (watts, samples) } .map { case ((act, idx), (watts, samples)) => (watts, samples) } .collect } get the actual sample windows for each effort ; project away IDs

  17. def bestsForPeriod(data: RDD[TP], period: Int, app: SLP, model: KMeansModel) = { val windowedSamples = /* window IDs and raw sample windows */ val clusterPairs = /* window IDs and spatial cluster pairs */ val mmps = /* window IDs and mean wattages for each window */ val top20 = mmps.join(clusterPairs) .map { case ((act, idx), (watts, (cls1, cls2))) => ((cls1, cls2), (watts, (act, idx))) } .reduceByKey ((a, b) => if (a._1 > b._1) a else b) .map { case ((cls1, cls2), (watts, (act, idx))) => (watts, (act, idx)) } .sortByKey(false) .take(20) app.context.parallelize(top20) .map { case (watts, (act, idx)) => ((act, idx), watts) } .join (windowedSamples) .map { case ((act, idx), (watts, samples)) => (watts, samples) } .collect .collect }

  18. Improving the prototype

  19. Broadcast large static data

  20. Broadcast variables // phoneBook maps (given name, surname) -> phone number digits val phoneBook: Map[(String, String), String] = initPhoneBook() val names: RDD[(String, String)] = /* ... */ val directory = names.map { case name @ (first, last) => (name, phoneBook.getOrElse("555-1212")) }

  21. Broadcast variables // phoneBook maps (given name, surname) -> phone number digits val phoneBook: Map[(String, String), String] = initPhoneBook() val names: RDD[(String, String)] = /* ... */ val directory = names.map { case name @ (first, last) => (name, phoneBook.getOrElse("555-1212")) } phoneBook will be copied and deserialized for each task !

  22. Broadcast variables // phoneBook maps (given name, surname) -> phone number digits val phoneBook: Map[(String, String), String] = initPhoneBook() val names: RDD[(String, String)] = /* ... */ val pbb = sparkContext.broadcast(phoneBook) val directory = names.map { case name @ (first, last) => (name, pbb.value.getOrElse(“555-1212")) } Broadcasting phoneBook means it can be deserialized once and cached on each Node !

  23. def bestsForPeriod(data: RDD[Trackpoint], period: Int, app: SLP, model: KMeansModel) = { val windowedSamples = windowsForActivities(data, period, minify _) val clusterPairs = windowedSamples.map { case ((act, idx), samples) => ((act, idx), clusterPairsForWindow(samples, model)) // ... }

  24. def bestsForPeriod(data: RDD[Trackpoint], period: Int, app: SLP, model: Broadcast[KMeansModel]) = { val windowedSamples = windowsForActivities(data, period, minify _) val clusterPairs = windowedSamples.map { case ((act, idx), samples) => ((act, idx), clusterPairsForWindow(samples, model.value)) // rest of function unchanged }

  25. Cache only when necessary

  26. def bestsForPeriod(data: RDD[TP], period: Int, app: SLP, model: KMeansModel) = { val windowedSamples = windowsForActivities(data, period).cache // ... val top20 = mmps.join(clusterPairs) .map { case ((act, idx), (watts, (cls1, cls2))) => ((cls1, cls2), (watts, (act, idx))) } .reduceByKey ((a, b) => if (a._1 > b._1) a else b) .map { case ((cls1, cls2), (watts, (act, idx))) => (watts, (act, idx)) } .sortByKey(false) .take(20) // ... }

  27. Keeping EVERY WINDOW IN MEMORY… def bestsForPeriod(data: RDD[TP], period: Int, app: SLP, model: KMeansModel) = { val windowedSamples = windowsForActivities(data, period).cache // ... val top20 = mmps.join(clusterPairs) .map { case ((act, idx), (watts, (cls1, cls2))) => ((cls1, cls2), (watts, (act, idx))) } .reduceByKey ((a, b) => if (a._1 > b._1) a else b) .map { case ((cls1, cls2), (watts, (act, idx))) => (watts, (act, idx)) } .sortByKey(false) .take(20) // ... }

  28. Keeping EVERY WINDOW IN MEMORY… def bestsForPeriod(data: RDD[TP], period: Int, app: SLP, model: KMeansModel) = { val windowedSamples = windowsForActivities(data, period).cache // ... val top20 = mmps.join(clusterPairs) .map { case ((act, idx), (watts, (cls1, cls2))) => ((cls1, cls2), (watts, (act, idx))) } .reduceByKey ((a, b) => if (a._1 > b._1) a else b) .map { case ((cls1, cls2), (watts, (act, idx))) => (watts, (act, idx)) } .sortByKey(false) .take(20) // ... } …EVEN THOUGH RECOMPUTING windows IS incredibly CHEAP AND YOU’ll need only a tiny fraction of windows?

  29. def bestsForPeriod(data: RDD[TP], period: Int, app: SLP, model: KMeansModel) = { val windowedSamples = windowsForActivities(data, period).cache // ... val top20 = mmps.join(clusterPairs) .map { case ((act, idx), (watts, (cls1, cls2))) => ((cls1, cls2), (watts, (act, idx))) } .reduceByKey ((a, b) => if (a._1 > b._1) a else b) .map { case ((cls1, cls2), (watts, (act, idx))) => (watts, (act, idx)) } .sortByKey(false) .take(20) // ... } Eliminating unnecessary memory pressure can lead to a substantial speedup!

  30. Avoid shu ffl es when possible

  31. tasks

  32. stages

  33. stages we want to Avoid all unnecessary shuffles

  34. def bestsForPeriod(data: RDD[Trackpoint], period: Int, app: SLP, model: Broadcast[KMeansModel]) = { val windowedSamples = windowsForActivities(data, period, minify _) val bests = windowedSamples.map { case ((act, idx), samples) => ( clusterPairsForWindow(samples, model.value), ((act, idx), samples.map(_.watts).reduce(_ + _) / samples.size) ) }.cache val top20 = bests.reduceByKey ((a, b) => if (a._2 > b._2) a else b) .map { case ((_, _), keep) => keep } .takeOrdered(20)(Ordering.by[((String, Int), Double), Double] { case ((_, _), watts) => -watts }) app.context.parallelize(top20) .join (windowedSamples) .map { case ((act, idx), (watts, samples)) => (watts, samples) } .collect }

  35. def bestsForPeriod(data: RDD[Trackpoint], period: Int, app: SLP, model: Broadcast[KMeansModel]) = { val windowedSamples = windowsForActivities(data, period, minify _) val bests = windowedSamples.map { { case ((act, idx), samples) => ( case ((act, idx), samples) => ( clusterPairsForWindow(samples, model.value), clusterPairsForWindow(samples, model.value), ((act, idx), samples.map(_.watts).reduce(_ + _) / samples.size) ((act, idx), samples.map(_.watts).reduce(_ + _) / samples.size) ) ) } }.cache val top20 = bests.reduceByKey ((a, b) => if (a._2 > b._2) a else b) .map { case ((_, _), keep) => keep } .takeOrdered(20)(Ordering.by[((String, Int), Double), Double] { case ((_, _), watts) => -watts }) app.context.parallelize(top20) .join (windowedSamples) .map { case ((act, idx), (watts, samples)) => (watts, samples) } .collect }

  36. def bestsForPeriod(data: RDD[Trackpoint], period: Int, app: SLP, model: Broadcast[KMeansModel]) = { val windowedSamples = windowsForActivities(data, period, minify _) start and end clusters { val bests = windowedSamples.map { case ((act, idx), samples) => ( case ((act, idx), samples) => ( clusterPairsForWindow(samples, model.value), clusterPairsForWindow(samples, model.value), ((act, idx), samples.map(_.watts).reduce(_ + _) / samples.size) ((act, idx), samples.map(_.watts).reduce(_ + _) / samples.size) ) ) }.cache } val top20 = bests.reduceByKey ((a, b) => if (a._2 > b._2) a else b) .map { case ((_, _), keep) => keep } .takeOrdered(20)(Ordering.by[((String, Int), Double), Double] { case ((_, _), watts) => -watts }) app.context.parallelize(top20) .join (windowedSamples) .map { case ((act, idx), (watts, samples)) => (watts, samples) } .collect }

  37. def bestsForPeriod(data: RDD[Trackpoint], period: Int, app: SLP, model: Broadcast[KMeansModel]) = { val windowedSamples = windowsForActivities(data, period, minify _) start and end clusters val bests = windowedSamples.map { { case ((act, idx), samples) => ( case ((act, idx), samples) => ( clusterPairsForWindow(samples, model.value), clusterPairsForWindow(samples, model.value), ((act, idx), samples.map(_.watts).reduce(_ + _) / samples.size) ((act, idx), samples.map(_.watts).reduce(_ + _) / samples.size) window id s and mean wattages ) ) }.cache } val top20 = bests.reduceByKey ((a, b) => if (a._2 > b._2) a else b) .map { case ((_, _), keep) => keep } .takeOrdered(20)(Ordering.by[((String, Int), Double), Double] { case ((_, _), watts) => -watts }) app.context.parallelize(top20) .join (windowedSamples) .map { case ((act, idx), (watts, samples)) => (watts, samples) } .collect }

  38. def bestsForPeriod(data: RDD[Trackpoint], period: Int, app: SLP, model: Broadcast[KMeansModel]) = { val windowedSamples = windowsForActivities(data, period, minify _) val bests = windowedSamples.map { case ((act, idx), samples) => ( clusterPairsForWindow(samples, model.value), ((act, idx), samples.map(_.watts).reduce(_ + _) / samples.size) ) eliminate a join and a transpose }.cache bests.reduceByKey ((a, b) => if (a._2 > b._2) a else b) val top20 = bests.reduceByKey ((a, b) => if (a._2 > b._2) a else b) .map { case ((_, _), keep) => keep } .map { case ((_, _), keep) => keep } .takeOrdered(20)(Ordering.by[((String, Int), Double), Double] { case ((_, _), watts) => -watts }) app.context.parallelize(top20) .join (windowedSamples) .map { case ((act, idx), (watts, samples)) => (watts, samples) } .collect }

  39. def bestsForPeriod(data: RDD[Trackpoint], period: Int, app: SLP, model: Broadcast[KMeansModel]) = { val windowedSamples = windowsForActivities(data, period, minify _) val bests = windowedSamples.map { case ((act, idx), samples) => ( clusterPairsForWindow(samples, model.value), ((act, idx), samples.map(_.watts).reduce(_ + _) / samples.size) ) }.cache val top20 = bests.reduceByKey ((a, b) => if (a._2 > b._2) a else b) .map { case ((_, _), keep) => keep } .takeOrdered(20)(Ordering.by[((String, Int), Double), Double] { .takeOrdered(20)(Ordering.by[((String, Int), Double), Double] { case ((_, _), watts) => -watts case ((_, _), watts) => -watts (use the right API calls!) }) } app.context.parallelize(top20) .join (windowedSamples) .map { case ((act, idx), (watts, samples)) => (watts, samples) } .collect }

  40. def bestsForPeriod(data: RDD[Trackpoint], period: Int, app: SLP, model: Broadcast[KMeansModel]) = { val windowedSamples = windowsForActivities(data, period, minify _) val bests = windowedSamples.map { case ((act, idx), samples) => ( clusterPairsForWindow(samples, model.value), ((act, idx), samples.map(_.watts).reduce(_ + _) / samples.size) ) }.cache val top20 = bests.reduceByKey ((a, b) => if (a._2 > b._2) a else b) .map { case ((_, _), keep) => keep } .takeOrdered(20)(Ordering.by[((String, Int), Double), Double] { case ((_, _), watts) => -watts }) app.context.parallelize(top20) app.context.parallelize(top20) .join (windowedSamples) .join (windowedSamples) .map { case ((act, idx), (watts, samples)) => (watts, samples) } .map { case ((act, idx), (watts, samples)) => (watts, samples) } .collect .collect } eliminate a transpose

  41. def bestsForPeriod(data: RDD[Trackpoint], period: Int, app: SLP, model: Broadcast[KMeansModel]) = { val windowedSamples = windowsForActivities(data, period, minify _) val bests = windowedSamples.map { case ((act, idx), samples) => ( clusterPairsForWindow(samples, model.value), ((act, idx), samples.map(_.watts).reduce(_ + _) / samples.size) ) }.cache val top20 = bests.reduceByKey ((a, b) => if (a._2 > b._2) a else b) .map { case ((_, _), keep) => keep } .takeOrdered(20)(Ordering.by[((String, Int), Double), Double] { case ((_, _), watts) => -watts }) app.context.parallelize(top20) app.context.parallelize(top20) app.context.parallelize(top20) .join (windowedSamples) .join (windowedSamples) .join (windowedSamples) .collect .map { case ((act, idx), (watts, samples)) => (watts, samples) } .map { case ((act, idx), (watts, samples)) => (watts, samples) } .map { case ((act, idx), (watts, samples)) => (watts, samples) } .collect .collect } eliminate a transpose …or two!

  42. Embrace laziness

  43. Embrace laziness (only pay for what you use)

  44. Windowed processing redux 20140909.tcx

Recommend

Improving Spark Performance with Zero-copy Buffer Management and RDMA Hu Li, Charley Chen and Wei

Improving Spark Performance with Zero-copy Buffer Management and RDMA Hu Li, Charley Chen and Wei

Improving Spark Performance with Zero-copy Buffer Management and RDMA Hu Li, Charley Chen and Wei Xu Institute for Interdisciplinary Information Sciences Tsinghua University, China Latency matters in big data Impala Query Dremel Query [2012]

541 views • 17 slides
Spark Technology 1. Spark main objectives 2. RDD concepts and operations 3. SPARK application

Spark Technology 1. Spark main objectives 2. RDD concepts and operations 3. SPARK application

10/05/2019 Big Data : Informatique pour les donnes et calculs massifs 7 SPARK technology Stphane Vialle Stephane.Vialle@centralesupelec.fr http://www.metz.supelec.fr/~vialle Spark Technology 1. Spark main objectives 2. RDD concepts

818 views • 39 slides
Intr Intro o to Spark to Spark and Spark and Spark SQL SQL AMP Camp 2014 Michael Armbrust -

Intr Intro o to Spark to Spark and Spark and Spark SQL SQL AMP Camp 2014 Michael Armbrust -

Intr Intro o to Spark to Spark and Spark and Spark SQL SQL AMP Camp 2014 Michael Armbrust - @michaelarmbrust What is Apache Spark? Fast and general cluster computing system, interoperable with Hadoop, included in all major distros

667 views • 43 slides
Spark Code Camp Discover Spark Streaming & Spark SQL Project Overview Focus on Spark

Spark Code Camp Discover Spark Streaming & Spark SQL Project Overview Focus on Spark

Spark Code Camp Discover Spark Streaming & Spark SQL Project Overview Focus on Spark Streaming and Spark SQL Explored Streaming API of Apache Spark on Ukko Cluster Window based Stream Content Direct Stream content

221 views • 9 slides
High Integrity Ada with SPARK Praxis Critical Systems 1 SPARK and the SPARK Examiner What is

High Integrity Ada with SPARK Praxis Critical Systems 1 SPARK and the SPARK Examiner What is

High Integrity Ada with SPARK Praxis Critical Systems 1 SPARK and the SPARK Examiner What is SPARK? A sub-language of Ada 83 and 95 with particular properties that make it ideally suited to the most critical of applications: completely

848 views • 10 slides
Exploring the Performance of Spark for a Scientific Use Case Saba Sehrish (ssehrish@fnal.gov), Jim

Exploring the Performance of Spark for a Scientific Use Case Saba Sehrish (ssehrish@fnal.gov), Jim

Exploring the Performance of Spark for a Scientific Use Case Saba Sehrish (ssehrish@fnal.gov), Jim Kowalkowski and Marc Paterno IEEE International Workshop on High-Performance Big Data Computing (HPBDC) In conjunction with the 30 th IEEE IPDPS

600 views • 22 slides
BUILDING APACHE SPARK APPLICATION PIPELINES FOR THE KUBERNETES ECOSYSTEM Michael McCune 14

BUILDING APACHE SPARK APPLICATION PIPELINES FOR THE KUBERNETES ECOSYSTEM Michael McCune 14

BUILDING APACHE SPARK APPLICATION PIPELINES FOR THE KUBERNETES ECOSYSTEM Michael McCune 14 November 2016 1 INTRODUCTION A little about me Embedded to Orchestration Red Hat emerging technologies OpenStack Sahara Oshinko project for

453 views • 31 slides
Improving Scalability and Fault Improving Scalability and Fault Tolerance in an Application

Improving Scalability and Fault Improving Scalability and Fault Tolerance in an Application

Improving Scalability and Fault Improving Scalability and Fault Tolerance in an Application Tolerance in an Application Management Infrastructure Management Infrastructure Nikolay Topilski , Jeannie Albrecht, and Amin Vahdat Williams College

560 views • 18 slides
Company Introduction Improving the security and performance of your data centres, networks and

Company Introduction Improving the security and performance of your data centres, networks and

SOLUTIONS & SUPPORT PROFESSIONAL & MANAGED SERVICES Company Introduction Improving the security and performance of your data centres, networks and applications Network Data Cyber Application Network Test www.phoenixdatacom.com

476 views • 10 slides
Spark Processing 101 September 10, 2015 Justin Sun Overview What is Spark? SparkContext

Spark Processing 101 September 10, 2015 Justin Sun Overview What is Spark? SparkContext

Spark Processing 101 September 10, 2015 Justin Sun Overview What is Spark? SparkContext Resilient Distributed Datasets (RDDs) Transformations Actions Code Examples Resources What is Spark? General cluster

356 views • 10 slides
SPARK G N I T E R R A M E V I T A E C u s t o m L o g o P r o j e c t Introduction

SPARK G N I T E R R A M E V I T A E C u s t o m L o g o P r o j e c t Introduction

C K PROPOSAL SPARK G N I T E R R A M E V I T A E C u s t o m L o g o P r o j e c t Introduction We are excited to quote on your logo design project and believe that Spark is a perfect fit for your needs. Concept Presentation

554 views • 30 slides
Using Adobe Spark Video in the Classroom Jessica McKenna jmckenna@glencoveschools.org &

Using Adobe Spark Video in the Classroom Jessica McKenna jmckenna@glencoveschools.org &

Using Adobe Spark Video in the Classroom Jessica McKenna jmckenna@glencoveschools.org & Giselle Taylor gjtaylor@glencoveschools.org Glen Cove City School District What is Adobe Spark Video? Story-telling video making application

331 views • 9 slides
SPARK NEW ZEALAND ANNUAL MEETING 2015 Spark New Zealand 2015 Spark New Zealand 2015 2 Order of

SPARK NEW ZEALAND ANNUAL MEETING 2015 Spark New Zealand 2015 Spark New Zealand 2015 2 Order of

SPARK NEW ZEALAND ANNUAL MEETING 2015 Spark New Zealand 2015 Spark New Zealand 2015 2 Order of Meeting: Introductions and formalities Chairmans address Managing Director update Resolutions Shareholder questions Conduct of polls Meeting

421 views • 38 slides
Improving Performance We want to improve the performance of our computation Unit 17

Improving Performance We want to improve the performance of our computation Unit 17

17.1 17.2 Improving Performance We want to improve the performance of our computation Unit 17 Question: What are we referring to when we say "performance"? __________________ Improving Performance __________________

337 views • 5 slides
Distributing Matrix Computations with Spark MLlib Reza Zadeh A General Platform Standard libraries

Distributing Matrix Computations with Spark MLlib Reza Zadeh A General Platform Standard libraries

Distributing Matrix Computations with Spark MLlib Reza Zadeh A General Platform Standard libraries included with Spark Spark MLlib Spark SQL GraphX Streaming machine structured graph learning real-time Spark Core Outline Introduction to

682 views • 40 slides
The Art of CPU-Pinning: Evaluating and Improving the Performance of Virtualization and

The Art of CPU-Pinning: Evaluating and Improving the Performance of Virtualization and

The Art of CPU-Pinning: Evaluating and Improving the Performance of Virtualization and Containerization Platforms Davood GhatrehSamani, Chavit Denninart Joseph Bacik Mohsen Amini Salehi High Performance Cloud Computing Lab (HPCC)

408 views • 30 slides
Apache Spark Streaming, Kafka and HarmonicIO: A Performance Benchmark and Architecture Comparison

Apache Spark Streaming, Kafka and HarmonicIO: A Performance Benchmark and Architecture Comparison

Apache Spark Streaming, Kafka and HarmonicIO: A Performance Benchmark and Architecture Comparison for Enterprise and Scientific Computing Ben Blamey, Andreas Hellander and Salman Toor Uppsala University, Sweden Ben.Blamey@it.uu.se Bench19,

474 views • 20 slides
Flex 4 - Spark Containers Ryan Frishberg Software Consultant, Lab49 http://www.frishy.com Spark

Flex 4 - Spark Containers Ryan Frishberg Software Consultant, Lab49 http://www.frishy.com Spark

Flex 4 - Spark Containers Ryan Frishberg Software Consultant, Lab49 http://www.frishy.com Spark Architecture From MX to Spark MX Rich, styleable components Heavy components => Easy to use (most of the time) Spark introduces

500 views • 30 slides
Improving the performance of data servers on multicore architectures Fabien Gaud Grenoble

Improving the performance of data servers on multicore architectures Fabien Gaud Grenoble

Improving the performance of data servers on multicore architectures Fabien Gaud Grenoble University Advisors: Jean-Bernard Stefani, Renaud Lachaize and Vivien Qu ema Sardes (INRIA/LIG) December 2, 2010 Improving the performance of data

693 views • 67 slides
Spark starts here. Spark New Zealand Annual Results 2014 Investor Presentation Spark is more

Spark starts here. Spark New Zealand Annual Results 2014 Investor Presentation Spark is more

Spark starts here. Spark New Zealand Annual Results 2014 Investor Presentation Spark is more than a name change. It It reflects enormous change for our customers fl t h f t and our business. Our ambition is to be a winning business,

667 views • 30 slides
Agenda Overall Enterprise Application Performance Factors Enterprise Application Best Practice

Agenda Overall Enterprise Application Performance Factors Enterprise Application Best Practice

Agenda Overall Enterprise Application Performance Factors Enterprise Application Best Practice for generic Enterprise Application Performance Factors Best Practice for 3-tiers Enterprise Application Hardware Load Balancer Basic Unix Tuning

336 views • 9 slides
Big Data Meets Machine Learning Apache Spark MLlib 1 MLlib Spark MLlib Graphx

Big Data Meets Machine Learning Apache Spark MLlib 1 MLlib Spark MLlib Graphx

Big Data Meets Machine Learning Apache Spark MLlib 1 MLlib Spark MLlib Graphx Streaming Spark Dataframe Spark Core (RDD) 2 Machine Learning Algorithms Supervised learning Given a set of features and labels Builds a model that

590 views • 24 slides
Spark & Spark SQL High-Speed In-Memory Analytics over

Spark & Spark SQL High-Speed In-Memory Analytics over

CSE 6242 / CX 4242 Data and Visual Analytics | Georgia Tech Spark & Spark SQL High-Speed In-Memory Analytics over Hadoop and Hive Data Instructor: Duen Horng (Polo) Chau 1 Slides adopted

792 views • 64 slides
Progress with improving electricity Progress with improving electricity industry performance 1

Progress with improving electricity Progress with improving electricity industry performance 1

Progress with improving electricity Progress with improving electricity industry performance 1 April 2014 There are many sub-markets in electricity 27 x Retail 3 x Hedge Transport 1 x Spot 9 x AS Retail On-going rises in residential

1.17k views • 47 slides

More recommend