本文共 2725 字，大约阅读时间需要 9 分钟。

1. wordcount: 统计词频， 排序
2. 历年温度最值： max, min, avg

part1: spark rdd 之 wordcount

spark-shell

//word.txtvar file="/home/wang/txt/word.txt" //a b c e f e fimport  org.apache.spark.rdd.RDD var rdd1:RDD[String]=sc.textFile(file)     				 // a b c e f e fvar rdd2:RDD[(String,Integer)]=rdd1.flatMap(_.split(" ") ).filter(_.trim !="" ).map((_,1))   // ("",6) (a,1)var rdd3:RDD[(String,Integer)] =rdd2.reduceByKey(_+_)   //(b,1), (yre,1), (46,1)var arr=rdd3.collect									//按单词升序var wordsRdd:RDD[(String,Integer)]=rdd3.sortByKey(true) // (435,1), (46,1), (5,1), (:,1), (a,1)var arr1:Array[(String,Integer)]=wordsRdd.collect		//按词频降序var countRdd:RDD[(String,Integer)]=rdd3.sortBy(x=>x._2,false) // (e,4), (f,2), (er,2)var arr2:Array[(String,Integer)]=countRdd.collect

scala api

   	
               
    
     org.apache.spark
                
    
     spark-core_2.11
                
    
     2.1.0
            
   

object Wc {  def main(args: Array[String]): Unit = {    //conf    var conf=new SparkConf()    conf.setMaster("local")    conf.setAppName("wc")    //context    var sc=new SparkContext(conf)    //上面的spark-shell代码.......  }}

part2: spark rdd 之 maxTemp

思路1：groupByKey

//（ 年， 温度）=> groupByKey//tmp.txtvar file="/home/wang/txt/tmp.txt"  //1982,-8  import org.apache.spark.rdd.RDDvar rdd1:RDD[(Integer,Integer)]=sc.textFile(file).map(x=> { var arr=x.split(","); (arr(0).toInt, arr(1).toInt)  //(1982,-8)  })//rdd2:RDD[(Integer, Iterable[Integer])]var rdd2:RDD[(Integer,Iterable[Integer])]=rdd1.groupByKey   //(1984,CompactBuffer(-4, 26))//(1984,26,-4,11.0),(1964,22,-11,5.5)var rdd3:RDD[(Integer,Integer,Integer,Double)]=rdd2.map(x=>{ var arr=x._2; var sum=arr.reduce(_+_);  var avg=sum/ arr.size.toDouble; (x._1,  arr.max,arr.min,  avg )})var arr:Array[(Integer,Integer,Integer,Double)]=rdd3.collect

思路2： reduceByKey

//  (年， max温度, min温度, 温度sum, 温度count )=> reduceByKey(聚合函数)var file="/home/wang/txt/tmp.txt"  //1982,-8  import org.apache.spark.rdd.RDDvar rdd1:RDD[(Integer, (Integer,Integer,Integer,Integer))]=sc.textFile(file).map(x=> {var arr=x.split(",");var tmp=arr(1).toInt;var year=arr(0).toInt;(year,  (tmp, tmp,tmp, 1 )) //(1982,-8)})var rdd2:RDD[(Integer, (Integer,Integer,Integer,Integer))]=rdd1.reduceByKey((x,y)=>{var tup1=xvar tup2=yvar maxTmp=max(tup1._1, tup2._1)var minTmp=min(tup1._2, tup2._2)var size=   tup1._3+ tup2._3var sum=    tup1._4 + tup2._4(maxTmp,minTmp,size,sum)  //(1984,(26,-4,22,2))})var rdd3:RDD[(Integer, (Integer,Integer,Double))]=rdd2.map(x=>{var year=x._1var tup1=x._2var maxTmp=tup1._1var minTmp=tup1._2var size= tup1._3var sum=  tup1._4 var avg= (sum/size.toDouble).formatted("%.3f").toDouble(year,(maxTmp,minTmp,avg))  //(1984,(26,-4,0.091))})var arr:Array[(Integer,Integer,Double)]=rdd3.collect

转载地址：http://rpdef.baihongyu.com/