spark wordcount

发表于 | 分类于 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
package com.scala.test.core

import org.apache.spark.{SparkConf, SparkContext}

object WordCount {
  def main(args : Array[String]): Unit = {
    val conf = new SparkConf().setAppName("wordCount").setMaster("local[*]")
    val sc = new SparkContext(conf)
    val line = sc.textFile("/Users/lifei/Desktop/ab/a.txt");//todo wholeTextFiles 可以读取目录
//    直接打印
//    line.flatMap(_.split(" ")).map((_,1)).reduceByKey(_+_).collect().foreach(println)

//    保存到文件
    line.flatMap(_.split(" ")).map((_,1)).reduceByKey(_+_).saveAsTextFile("/Users/lifei/Desktop/ab/lala")

    sc.stop();
  }
}