DStream窗口操作-DStream_ReduceBykeyAndWindow

package org.hnsw

import org.apache.spark.SparkConf
import org.apache.spark.streaming.{Seconds, StreamingContext}

object DStream_ReduceBykeyAndWindow {
  def main(args: Array[String]): Unit = {
    //1、初始化Context上下文
    val conf = new SparkConf().setAppName("jxq").setMaster("local[*]")
    val sc = new StreamingContext(conf, Seconds(1))
    //2、指定采集服务器ip和端口
    //设置切换
//    sc.checkpoint("out") //执行countByWindow前需要设置checkpoint
    val dStream = sc.socketTextStream("192.168.3.66",8888)
    //3、业务逻辑: 窗口函数操作
    //1) 分割字符，取到单词，装换成key-value结构
    //窗口类rdd数据的数量
    val wolds = dStream.flatMap((x)=>{
      x.split(" ")
    }).map((x)=>{
      (x,1)  //设置单词初始计算值为1
    })
    //2)reduceByKeyWindow方法 对窗口内数据进行规约聚合操作
    // 输入参数需要显示的指定类型
    val reduceWords = wolds.reduceByKeyAndWindow((x:Int,y:Int)=>{
      x + y
    },Seconds(4),Seconds(2)) //指定窗口大小 和 滑动频率 必须是批处理时间的整数倍
    //3）结果打印
    reduceWords.print()
    //4、启动stream
    sc.start()
    //5、挂起stream
    sc.awaitTermination()
  }
}

标签：窗口,val,Seconds,ReduceBykeyAndWindow,sc,org,DStream
来源： https://www.cnblogs.com/857weir09432/p/16362783.html