欢迎关注微信公众号：小满锅

这里flume+kafka+Flink Cluster总体架构就是这个样子。

我们将多个Tomcat Server的Web服务器，或者其他的网站服务器都看你而定，没台节点上都配置了Flume Agent，用来收集本地日志目录。然后在另一台节点配置了Flume Consolidation Agent，用来收集各个Web服务器的Flume Agent的数据，然后送入Kafka Cluster。然后Kafka Cluster的数据交由Flink Cluster消费，从而将数据展示在Web前端

flume1

ZooKeeper -server host:port cmd args
#主要作用是监听文件中的新增数据，采集到数据之后，输出到avro
#    注意：Flume agent的运行，主要就是配置source channel sink
#  下面的a1就是agent的代号，source叫r1 channel叫c1 sink叫k1
a1.sources = r1
a1.sinks = k1
a1.channels = c1

#对于source的配置描述 监听文件中的新增数据 exec
a1.sources.r1.type = exec
a1.sources.r1.command  = tail -F /home/uplooking/data/data-clean/data-access.log

#对于sink的配置描述 使用avro日志做数据的消费
a1.sinks.k1.type = avro
a1.sinks.k1.hostname = localhost
a1.sinks.k1.port = 9000

#对于channel的配置描述 使用文件做数据的临时缓存 这种的安全性要高
a1.channels.c1.type = file
a1.channels.c1.checkpointDir = /home/uplooking/data/flume/checkpoint
a1.channels.c1.dataDirs = /home/uplooking/data/flume/data

#通过channel c1将source r1和sink k1关联起来
a1.sources.r1.channels = c1
a1.sinks.k1.channel = c1

 执行命令
flume-ng agent --conf conf -n a1 -f 上面这个文件的位置 >/dev/null 2>&1 &

flume2同上，都是一台web服务器

flume3

#主要作用是监听目录中的新增文件，采集到数据之后，输出到kafka
#    注意：Flume agent的运行，主要就是配置source channel sink
#  下面的a1就是agent的代号，source叫r1 channel叫c1 sink叫k1
a1.sources = r1
a1.sinks = k1
a1.channels = c1

#对于source的配置描述 监听avro
a1.sources.r1.type = avro
a1.sources.r1.bind = 0.0.0.0
a1.sources.r1.port = 9000

#对于sink的配置描述 使用kafka做数据的消费
a1.sinks.k1.type = org.apache.flume.sink.kafka.KafkaSink
a1.sinks.k1.topic = f-k-s
a1.sinks.k1.brokerList = 114.116.219.197:5008,114.116.220.98:5008,114.116.199.154:5008
a1.sinks.k1.requiredAcks = 1
a1.sinks.k1.batchSize = 20

#对于channel的配置描述 使用内存缓冲区域做数据的临时缓存
a1.channels.c1.type = memory
a1.channels.c1.capacity = 1000
a1.channels.c1.transactionCapacity = 100

#通过channel c1将source r1和sink k1关联起来
a1.sources.r1.channels = c1
a1.sinks.k1.channel = c1

flume-ng agent --conf conf -n a1 -f 上面这个文件的位置 >/dev/null 2>&1 &

前面所有的过程都需要你能够把kafka，flume搭建好，这里就做一下flume+kafka+flink的整合

由于Web暂时还没写，我们直接往日志目录里面写数据，先看看flink代码是怎么样的，这是一段关于电商的热门商品统计的实时数据分析

package com.ongbo.hotAnalysis

import java.sql.Timestamp
import java.util.Properties

import org.apache.flink.api.common.functions.AggregateFunction
import org.apache.flink.api.common.serialization.SimpleStringSchema
import org.apache.flink.api.common.state.{ListState, ListStateDescriptor}
import org.apache.flink.api.java.tuple.Tuple
import org.apache.flink.configuration.Configuration
import org.apache.flink.streaming.api.TimeCharacteristic
import org.apache.flink.streaming.api.functions.KeyedProcessFunction
import org.apache.flink.streaming.api.scala._
import org.apache.flink.streaming.api.scala.function.WindowFunction
import org.apache.flink.streaming.api.windowing.time.Time
import org.apache.flink.streaming.api.windowing.windows.TimeWindow
import org.apache.flink.streaming.connectors.kafka.FlinkKafkaConsumer
import org.apache.flink.util.Collector

import scala.collection.mutable.ListBuffer

/*
*定义输入数据的样例类
 */
case class UserBehavior(userId: Long, itemId: Long, cateGoryId: Int,behavior: String, timestamp: Long)
//定义窗口聚合结果样例类
case class ItemViewCount(itemId: Long, windowEnd: Long, count: Long)

object HotItems {
  def main(args: Array[String]): Unit = {
    //1:创建执行环境
    val env = StreamExecutionEnvironment.getExecutionEnvironment
    env.setParallelism(1)
    //设置为事件事件
    env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime)
    //2:读取数据

    /*kafka源*/
    val properties = new Properties()
    properties.setProperty("bootstrap.servers","114.116.219.197:5008,114.116.220.98:5008,114.116.199.154:5008")
    properties.setProperty("group.id","web-consumer-group")
    properties.setProperty("key.deserializer","org.apache.kafka.common.serialization.StringDeserializer")
    properties.setProperty("value.deserializer","org.apache.kafka.common.serialization.StringDeserializer")
    properties.setProperty("auto.offset.reset","latest")
    val dataStream = env.addSource(new FlinkKafkaConsumer[String]("weblog", new SimpleStringSchema(),properties))
//    val dataStream = env.readTextFile("/Users/ongbo/Maven/bin/UserBehaviorAnalysis/HotItemAnalysis/src/main/resources/UserBehavior.csv")
      .map(data =>{
        System.out.println("data:"+data)
        val dataArray = data.split(",")
//        if(dataArray(0).equals("ij"))
        UserBehavior(dataArray(0).trim.toLong, dataArray(1).trim.toLong, dataArray(2).trim.toInt, dataArray(3).trim, dataArray(4).trim.toLong)

      })
      .assignAscendingTimestamps(_.timestamp * 1000L)

    //3:transform处理数据
    val processStream = dataStream
      //筛选出埋点pv数据
      .filter(_.behavior.equals("pv"))
      //先对itemID进行分组
      .keyBy(_.itemId)
      //然后设置timeWindow，size为1小时，步长为5分钟的滑动窗口
      .timeWindow(Time.seconds(20), Time.seconds(10))
      //窗口聚合
      .aggregate(new CountAgg(), new WindowResult())
      .keyBy(_.windowEnd)      //按照窗口分组

      .process(new TopNHotItems(10))


    //sink：输出数据
    processStream.print("processStream::")
//    dataStream.print()
    //执行
    env.execute("hot Items Job")



  }
}

/*自定义预聚合函数*/
class CountAgg() extends AggregateFunction[UserBehavior, Long, Long]{
  //累加器初始值
  override def createAccumulator(): Long = 0
  //每来一次就加一
  override def add(in: UserBehavior, acc: Long): Long = acc+1
  //
  override def getResult(acc: Long): Long = acc

  override def merge(acc: Long, acc1: Long): Long = acc + acc1
}

//自定义窗口函数，输出ItemViewCount
class WindowResult() extends WindowFunction[Long,ItemViewCount, Long, TimeWindow]{
  override def apply(key: Long, window: TimeWindow, input: Iterable[Long], out: Collector[ItemViewCount]): Unit =  {
    out.collect(ItemViewCount(key,window.getEnd,input.iterator.next()))
  }
}

//自定义处理函数
class TopNHotItems(topsize: Int) extends KeyedProcessFunction[Long, ItemViewCount, String] {
  private var itemState: ListState[ItemViewCount] = _

  override def open(parameters: Configuration): Unit = {
    itemState = getRuntimeContext.getListState(new ListStateDescriptor[ItemViewCount]("item-state", classOf[ItemViewCount]))

  }
  override def processElement(value: ItemViewCount, ctx: KeyedProcessFunction[Long, ItemViewCount, String]#Context, out: Collector[String]): Unit = {
    //把每条数据存入状态列表
    itemState.add(value)
    //注册一个定时器
    ctx.timerService().registerEventTimeTimer(value.windowEnd + 1)
  }
  //定时器触发时，对所有的数据排序，并输出结果
  override def onTimer(timestamp: Long, ctx: _root_.org.apache.flink.streaming.api.functions.KeyedProcessFunction[Long, _root_.com.ongbo.hotAnalysis.ItemViewCount, _root_.scala.Predef.String]#OnTimerContext, out: _root_.org.apache.flink.util.Collector[_root_.scala.Predef.String]): Unit = {
    //将所有state中的数据取出，放到一个list Buffer中
    val allItems: ListBuffer[ItemViewCount] = new ListBuffer()
    import scala.collection.JavaConversions._
    for(item <- itemState.get()){
      allItems += item
    }

    //按照点计量count大小排序,sortBy默认是升序,并且取前三个
    val sortedItems = allItems.sortBy(_.count)(Ordering.Long.reverse).take(topsize)

    //清空状态
    itemState.clear()

    //格式化输出排名结果
    val result : StringBuilder = new StringBuilder
    result.append("时间：").append(new Timestamp(timestamp - 1)).append("\n")
    //输出每一个商品信息
    for(i<- sortedItems.indices){
      val currentItem = sortedItems(i)
      result.append("No").append(i+1).append(":")
        .append("  商品ID：").append(currentItem.itemId)
        .append("  浏览量：").append(currentItem.count).append("\n")
    }
    result.append("============================\n")
    //控制输出频率
    Thread.sleep(1000)
    out.collect(result.toString())
  }
}




/*自定义预聚合函数计算平均数*/
class AverageAgg() extends AggregateFunction[UserBehavior, (Long,Int), Double]{
  override def createAccumulator(): (Long, Int) = (0L,0)

  override def add(in: UserBehavior, acc: (Long, Int)): (Long, Int) = (acc._1+in.timestamp, acc._2+1)

  override def getResult(acc: (Long, Int)): Double = acc._1 /acc._2

  override def merge(acc: (Long, Int), acc1: (Long, Int)): (Long, Int) = (acc._1+acc1._1, acc._2+acc1._2)
}

下面往日志目录里面写日志

543462,1715,1464116,pv,1511658000
662867,2244074,1575622,pv,1511658000
561558,3611281,965809,pv,1511658000
894923,3076029,1879194,pv,1511658000
834377,4541270,3738615,pv,1511658000
315321,942195,4339722,pv,1511658000
625915,1162383,570735,pv,1511658000
578814,176722,982926,pv,1511658000
873335,1256540,1451783,pv,1511658000
429984,4625350,2355072,pv,1511658000
866796,534083,4203730,pv,1511658000
937166,321683,2355072,pv,1511658000
156905,2901727,3001296,pv,1511658000
758810,5109495,1575622,pv,1511658000
107304,111477,4173315,pv,1511658000
452437,3255022,5099474,pv,1511658000
813974,1332724,2520771,buy,1511658000
524395,3887779,2366905,pv,1511658000
470572,3760258,1299190,pv,1511658001
543789,3110556,4558987,cart,1511658001
354759,2191348,4756105,pv,1511658001
382009,2123538,4801426,pv,1511658001
677046,1598945,4145813,pv,1511658001
946161,3021357,1506018,pv,1511658001
464646,2512167,2733371,pv,1511658001
1007641,5046581,2355072,pv,1511658001
723938,4719377,1464116,pv,1511658001
513008,3472922,401357,pv,1511658001
769215,22738,2355072,pv,1511658002
652863,4967749,1320293,pv,1511658002
801610,900305,634390,pv,1511658002
411478,3259235,2667323,pv,1511658002
431664,764155,2520377,pv,1511658002
487768,4125503,2465336,pv,1511658002
223813,4104826,2042400,pv,1511658002
672849,1822977,4801426,fav,1511658002
550127,4602135,65362,pv,1511658002
205752,1467139,171529,pv,1511658002
64419,2029769,2729260,pv,1511658002
756093,2881426,2520377,pv,1511658002
48353,4362292,583014,pv,1511658002
355509,4712764,4082778,pv,1511658003
826492,4016552,2735466,pv,1511658003
624915,2243311,2520377,pv,1511658003
682317,655740,982926,fav,1511658003
677621,1051389,4801426,pv,1511658003
422974,4649255,4818107,pv,1511658003
86512,563566,4756105,pv,1511658003
565218,2331370,3607361,pv,1511658003
232313,4182588,1730376,pv,1511658003
436966,1329977,3607361,cart,1511658003
561158,269170,2342116,fav,1511658003
344379,3318242,2920476,cart,1511658003
858204,2450718,235534,pv,1511658004
833924,3190176,1051370,pv,1511658004
992993,1900968,3794706,fav,1511658004
911930,1150136,2131531,pv,1511658004
736959,319911,4756105,pv,1511658004
82170,3588374,2465336,pv,1511658004
587599,2067643,4818107,cart,1511658004
367451,15775,4756105,pv,1511658004
428316,2478780,4284875,pv,1511658004
284910,3680091,3829657,pv,1511658004
345119,737662,4357323,pv,1511658004
551442,1762997,1879194,pv,1511658004
550384,3908776,1029459,pv,1511658004
677500,4534693,2640118,pv,1511658004
398626,2791489,1467750,pv,1511658004
118053,3545571,2433095,pv,1511658005
457401,4063698,4801426,pv,1511658005
45105,3234847,3141941,fav,1511658005
604760,2661651,3738615,pv,1511658005
905383,2064903,2939262,cart,1511658005
740788,3657484,4936889,pv,1511658005
456838,1242724,4756105,fav,1511658005
585217,215764,2640118,pv,1511658006
658185,4025021,4048584,fav,1511658006
210431,2035568,2328673,pv,1511658006
602619,1838725,2247787,pv,1511658006
860388,3797303,4357323,pv,1511658006
175334,2624960,801221,pv,1511658006
72403,4249007,1320293,pv,1511658006
307385,2551880,4050612,pv,1511658006
819283,2094785,2520377,pv,1511658006
801272,565658,1158475,pv,1511658006
344680,3224461,4789432,pv,1511658006
125206,1102775,622168,pv,1511658006
59131,1960832,154040,pv,1511658006
252339,2455388,3745824,pv,1511658006
794780,4465604,4242717,pv,1511658007
388283,4701157,1457367,pv,1511658007
416261,2101120,1299190,pv,1511658007
231758,3622677,4758477,pv,1511658007
92253,642337,4135185,pv,1511658007
297958,1762578,4801426,pv,1511658007
786771,1940649,1320293,pv,1511658007
789048,3144191,2355072,pv,1511658007
895384,1138468,1602288,pv,1511658007
578800,1324176,4135836,pv,1511658007
886777,4606952,996587,pv,1511658008
364036,2340632,2924150,pv,1511658008
783973,3935641,982926,pv,1511658008
483113,1158047,2030976,pv,1511658008
321087,3987903,149192,fav,1511658008
986762,2704613,4217906,pv,1511658008
1003495,3717164,535180,pv,1511658008
32983,1445878,2030976,pv,1511658008
580912,1837669,4263418,pv,1511658008
498371,980167,3607361,cart,1511658008
14830,3878488,3002561,pv,1511658008
457499,4796667,2355072,pv,1511658008
879381,881945,3004853,pv,1511658008
606403,1404468,781315,pv,1511658008
543832,3007018,2735466,pv,1511658009
824188,161847,4801426,pv,1511658009
750784,4727562,1286537,cart,1511658009
87479,5157294,2158079,pv,1511658009
147730,1278040,4801426,pv,1511658009
865088,3924907,411153,pv,1511658009
1006129,1993506,2939262,pv,1511658009
401750,3127850,2640118,pv,1511658009
501335,2005713,1092085,pv,1511658009
266486,1777124,4672807,cart,1511658009
23125,4680673,1756314,buy,1511658009
309473,3665543,2640118,pv,1511658009
547966,693058,2129666,pv,1511658009
599118,2771420,4159072,pv,1511658010
684544,2335709,1451814,pv,1511658010
130320,2469137,4756105,pv,1511658010
464365,1474058,3794706,pv,1511658010
522088,4170363,149192,pv,1511658010
402626,3244134,4801426,pv,1511658010
19445,2183531,4756105,pv,1511658010
44670,5140156,2131531,fav,1511658010
522815,2092266,2355072,pv,1511658010
105757,4175185,3191556,cart,1511658010
585344,1858802,267209,pv,1511658010
412009,3422566,2355072,pv,1511658010
872433,1984468,2520377,pv,1511658010
463780,3377720,745134,pv,1511658010
359932,5035793,3002561,pv,1511658011
742048,3851196,4051859,pv,1511658011
778656,559998,4874384,pv,1511658011
326402,2103118,846990,pv,1511658011
453328,2158281,2920476,pv,1511658011
362515,1892018,883960,pv,1511658011
763534,2402567,3496816,pv,1511658011
935169,55330,4339722,pv,1511658011
361299,1505783,982926,pv,1511658011
628949,4210587,903809,cart,1511658011
989894,3419383,1474371,fav,1511658012
200517,4670782,4181361,cart,1511658012
664416,2941436,1789614,pv,1511658012
348706,2605837,4163659,pv,1511658012
959846,5083345,479837,pv,1511658012
848070,988062,2342116,pv,1511658012
897594,4298799,4173315,pv,1511658012
951961,1221785,4756105,pv,1511658012
549488,2423996,4022701,pv,1511658012
524655,1925334,2033408,pv,1511658013
738256,496639,3247211,pv,1511658013
778173,1032031,784897,cart,1511658013
990673,1476503,3102419,pv,1511658013
700498,1993845,4173315,pv,1511658013
905860,2369381,1158475,pv,1511658013
893832,1153119,982926,pv,1511658013
602895,3538101,4756105,pv,1511658013
542932,1360737,903809,pv,1511658013
283950,3179868,3171095,pv,1511658013
789348,3841875,737184,pv,1511658013
756750,3534407,4129924,fav,1511658013
196825,4765909,2885642,pv,1511658013
20676,2875398,2885642,pv,1511658013
228104,2062543,2355072,pv,1511658013
773198,4200289,982926,pv,1511658014
677896,2165635,1029459,pv,1511658014
158600,3312738,1697889,fav,1511658014
287521,972793,1194737,pv,1511658014
549674,625283,3767409,pv,1511658014
631139,4364531,384755,pv,1511658014
375063,4069076,3607361,pv,1511658014
306072,1966569,285583,pv,1511658014
643526,282851,1575622,pv,1511658014
764411,4580986,982926,pv,1511658014
976200,3266415,2355072,pv,1511658014
918987,298223,1859277,pv,1511658014
18430,1882195,1080785,pv,1511658014
629115,3093045,1320293,pv,1511658014
332520,5026647,2355072,pv,1511658014
476980,2311313,4801426,pv,1511658014
157636,4858109,672001,pv,1511658015
798555,1295731,2800021,pv,1511658015
494300,4333209,3002561,cart,1511658015
122406,2059276,4801426,pv,1511658015
437344,4061918,4558987,pv,1511658015
220037,2939369,3720767,buy,1511658015
537712,2563909,3100759,pv,1511658015
520145,4465657,1320293,pv,1511658015
38574,1201477,3125533,pv,1511658015
1014253,2963034,1464116,pv,1511658015
915638,4800478,2629787,pv,1511658015
768844,2116634,4757812,pv,1511658015
74745,2231297,323851,fav,1511658015
326607,3693156,2066955,pv,1511658015
651189,4000666,886203,pv,1511658015
778396,3607696,5012555,pv,1511658016
493284,127403,359388,pv,1511658016
254349,4445002,2355072,pv,1511658016
415381,3520931,2419959,pv,1511658016
516789,18479,4145813,pv,1511658016
644777,987403,982926,pv,1511658016
317457,1546479,3776866,pv,1511658016
156905,4377552,1168596,pv,1511658016
740608,2405231,4173315,pv,1511658016
341260,5034946,4756105,pv,1511658016
806318,3525976,3299155,pv,1511658016
674989,4568242,4284875,fav,1511658016
889043,5015057,1281171,pv,1511658017
912390,2303733,2735466,pv,1511658017
393656,3569818,3738615,pv,1511658017
225763,480292,500212,pv,1511658017
334506,3722390,411153,pv,1511658017
273314,4973721,4611962,pv,1511658017
663193,2108019,2003083,pv,1511658017
1000034,800784,982926,pv,1511658018
391116,1962121,4145813,pv,1511658018
40793,4153899,1320293,pv,1511658018
280358,2654723,2520377,pv,1511658018
344240,3712088,2390349,pv,1511658018
691026,4022625,1126526,pv,1511658018
688090,1502018,1879194,pv,1511658018
947196,2259297,1406426,pv,1511658018
518479,576777,411153,pv,1511658018
789309,5136896,2981856,pv,1511658018
248568,3349768,1045172,buy,1511658018
392176,3742963,2066955,pv,1511658018
460076,79067,4145813,fav,1511658018
415381,98283,1104384,pv,1511658018
104698,2500376,4159072,pv,1511658018
854096,4858377,4145813,pv,1511658018
587129,4427761,3776866,cart,1511658019
58778,630815,2355072,pv,1511658019
250968,2335179,2355072,pv,1511658019
380976,3933325,982926,pv,1511658019
904253,1055751,471212,pv,1511658019
907231,3686874,2812445,buy,1511658019
377042,3549064,2355072,pv,1511658019
356507,3220081,578983,pv,1511658019
753856,2167638,4048584,fav,1511658019
929073,1324319,1879194,pv,1511658019
43863,918139,982926,pv,1511658019
41109,4133170,3835723,buy,1511658020
108015,4751201,411153,pv,1511658020
285636,4130380,2355072,pv,1511658020
395049,4346219,570735,cart,1511658020
949647,4427447,4756105,pv,1511658020
306072,1296281,3720767,pv,1511658020
69580,3928745,2671397,pv,1511658020
735110,40025,2974419,cart,1511658020
365717,1391888,2030976,pv,1511658020
587754,4762986,3002561,pv,1511658020
183966,1424015,4672807,pv,1511658020
874905,876411,2465336,pv,1511658020
789333,3383623,344833,pv,1511658020
323387,1699578,982926,pv,1511658021
530376,205888,5071267,pv,1511658021
770137,1031049,154040,pv,1511658021
732889,283410,3607361,pv,1511658021
170558,3144143,2465336,pv,1511658021
1004611,201201,3419760,cart,1511658021
121442,5073040,2355072,pv,1511658021
511424,1078256,440901,pv,1511658021
810879,4042749,3607361,pv,1511658021
889013,1023293,634390,pv,1511658021
641439,183845,4357323,pv,1511658021
607188,3590098,2885642,pv,1511658021
789271,5057422,245312,buy,1511658021
350999,3956368,2945933,pv,1511658021
974720,4838031,2072473,cart,1511658021
460076,2147027,1320293,fav,1511658022
956539,3937435,4801426,pv,1511658022
682283,102982,1080785,pv,1511658022
133319,2817626,4801426,pv,1511658022
82010,2788503,1216617,pv,1511658022
417933,4159217,4756105,pv,1511658022
719252,2161152,4145813,pv,1511658022
386729,4115033,3343095,pv,1511658022
460076,3268126,737184,fav,1511658022
764871,564499,2640118,pv,1511658022
599118,2335010,4159072,pv,1511658022
463780,341543,820364,pv,1511658022
377108,493618,3975787,cart,1511658022
266094,3871120,2465336,pv,1511658022
392064,4974340,4558987,pv,1511658022
858761,5042307,4357323,pv,1511658023
266992,3559268,4869428,pv,1511658023
208833,3716643,383530,pv,1511658023
244463,5059853,4145813,pv,1511658023
874809,1024295,4466876,pv,1511658023
87520,1318574,4606718,pv,1511658023
483594,1088111,2827444,pv,1511658023
29206,4448187,4756105,pv,1511658023
186556,4070367,74446,pv,1511658023
190940,833331,4604594,pv,1511658023
201971,962676,2355072,pv,1511658023
180695,1716295,2565857,pv,1511658023
670681,4511840,846990,pv,1511658024
843075,3665269,1080785,fav,1511658024
409463,4443612,1722855,pv,1511658024
145550,3900777,4993094,pv,1511658024
608599,1097682,4082778,cart,1511658024
997866,3002287,4082778,pv,1511658024
436457,1884224,4915680,pv,1511658024
517112,4526678,2520377,pv,1511658024
659269,1605934,982926,pv,1511658025
195725,5000029,4450414,pv,1511658025
563203,1849653,4298656,pv,1511658025
669210,4843772,570735,pv,1511658025
445656,2581505,3381583,pv,1511658025
16103,3494764,2640118,pv,1511658025
175769,776327,2465336,pv,1511658025
960997,2649870,4756105,pv,1511658025
995188,3017896,2355072,pv,1511658025
475656,361127,3673049,pv,1511658025
940124,176753,4411949,pv,1511658025
538101,1732433,1787510,pv,1511658025
416871,4981228,2858794,pv,1511658025
256758,3889295,3800818,pv,1511658026
511279,4601104,4868797,pv,1511658026
880462,2780737,1859277,cart,1511658026
163323,258231,1029459,pv,1511658026
425634,4018832,4159072,pv,1511658026
932384,2428269,982926,pv,1511658026
565980,2988776,3075521,pv,1511658026
551219,281111,350027,pv,1511658026
65944,4309207,3159480,pv,1511658026
118861,512107,4145813,pv,1511658026
15696,1015021,3454970,pv,1511658027
425200,2455388,3745824,pv,1511658027
55330,465280,2355072,pv,1511658027
664146,1030747,3194735,fav,1511658027
587248,1384198,634390,pv,1511658027
472221,2895550,982926,pv,1511658027
173290,416572,360294,pv,1511658027
735750,2456660,1080785,pv,1511658027
420860,1949459,1288487,pv,1511658027
248955,229653,2195789,pv,1511658027
745774,4756886,800581,pv,1511658027
179889,2041935,2520377,pv,1511658027
287035,3836534,625430,pv,1511658028
196980,3445269,3108882,pv,1511658028
550480,974291,869231,pv,1511658028
585870,291358,4174942,pv,1511658028
165889,4364728,2615163,pv,1511658028
193922,4800535,3607361,pv,1511658028
660013,1364114,4801426,pv,1511658028
813479,158169,2355072,pv,1511658028
261948,4698781,2520377,pv,1511658028
120608,131099,876318,pv,1511658029
268407,584693,3158249,pv,1511658029
411536,2563526,4663501,pv,1511658029
502196,936753,4145813,pv,1511658029
652954,1362503,4284875,pv,1511658029
70758,3011543,2926020,pv,1511658029
684074,2288156,377676,pv,1511658029
515661,680174,2355072,pv,1511658029
322508,2466347,2465336,pv,1511658030
643826,4545733,873792,pv,1511658030
447146,1436416,4331400,pv,1511658030
646259,418486,119741,pv,1511658030
288397,1926504,149192,pv,1511658030
337203,1253410,405755,pv,1511658030
219642,2704585,4148053,pv,1511658030
992351,1452966,2303546,pv,1511658030
860477,4319079,149192,pv,1511658030
680394,2772093,3720767,pv,1511658030
667912,1047955,1320293,pv,1511658030
938116,2134897,2735466,pv,1511658030
276252,3465909,1299190,pv,1511658030
1004505,4048232,145519,pv,1511658031
954492,4932087,1320293,pv,1511658031
271586,151399,578983,pv,1511658031
629913,1014006,3702593,pv,1511658031
879398,2473922,4756105,pv,1511658031
644883,4961929,1573465,pv,1511658032
62574,169041,1807416,pv,1511658032
643095,5119949,4357323,pv,1511658032
14391,2367378,2920476,pv,1511658032
840495,3013895,2926020,pv,1511658032
846863,1607283,1132429,pv,1511658032
463414,4937993,3607361,pv,1511658032
743967,491563,4244487,pv,1511658032
571930,2104387,1646753,pv,1511658032
337203,2407315,4715650,pv,1511658032
87004,3581433,690811,pv,1511658032
808466,3209415,2355072,pv,1511658032
834242,2646442,4357323,pv,1511658032
5855,1547334,1120543,pv,1511658032
861315,4512645,3361496,fav,1511658032
128765,1897815,4756105,pv,1511658033
677500,4690832,3504022,pv,1511658033
594637,3443787,3747017,pv,1511658033
958078,952967,3177877,pv,1511658033
455903,1725977,3904106,pv,1511658033
441977,177176,50601,pv,1511658033
391280,4307461,982926,pv,1511658033
873094,2044886,2355072,pv,1511658033
664177,1293088,5053508,pv,1511658034
302778,3781391,1528133,pv,1511658034
475563,4841577,1275696,cart,1511658034
208142,3139742,4756105,pv,1511658034
138026,3709805,3672452,pv,1511658034
534394,3020956,2355072,pv,1511658034
387963,1177970,2885642,pv,1511658034
334506,2951108,411153,pv,1511658034
274331,2394590,1464116,pv,1511658034
356279,829406,1102540,pv,1511658034
775853,3788823,4217906,pv,1511658034
156203,4938048,2383838,pv,1511658034
572844,1438423,2520377,pv,1511658035
987111,2263608,245312,pv,1511658035
231038,708261,1964754,fav,1511658035
81884,2437934,753984,pv,1511658035
942856,4215633,4756105,pv,1511658035
208061,2916548,4692440,pv,1511658035
495163,5100809,1879194,pv,1511658035
43237,2393443,2355072,pv,1511658036
750878,4729766,4672807,pv,1511658036
882734,4273237,1540741,pv,1511658036
400141,1714043,237446,pv,1511658036
585344,646781,267209,pv,1511658036
227633,3498188,4476428,pv,1511658036
284584,2019640,3607361,cart,1511658036
988694,3542016,982926,pv,1511658036
164332,3158500,235534,pv,1511658036
945934,2793460,3720767,pv,1511658036
691649,437087,223690,pv,1511658036
128816,2803125,3637084,pv,1511658036
715158,1448525,1842993,pv,1511658036
606841,4290041,3299155,cart,1511658037
655163,3409614,4357323,pv,1511658037
337203,1253410,405755,pv,1511658037
169609,4290998,4731191,pv,1511658037
515001,4142409,2735466,pv,1511658037
525614,4397050,3738615,pv,1511658037
48008,3043421,4239038,pv,1511658037
1015928,2849670,2735466,pv,1511658037
408783,703194,2885642,pv,1511658037
42364,3978662,611849,pv,1511658037
790128,4946853,2096639,pv,1511658038
226081,689681,4756105,pv,1511658038
774686,96790,2188684,pv,1511658038
317012,1665474,2355072,pv,1511658038
211077,990967,570735,pv,1511658038
879481,3523697,2154867,pv,1511658038
24625,5074681,2465336,pv,1511658038
638709,3848680,1575622,pv,1511658038
178232,1413610,149192,pv,1511658038
19742,4388000,200278,pv,1511658038
216011,2678565,1548278,pv,1511658038
709957,1969563,4606718,pv,1511658038
106526,856738,1080785,pv,1511658038
325562,363858,2735466,pv,1511658038
979063,2452027,4022701,pv,1511658038
451072,1035375,1320293,pv,1511658038
374336,3615843,4801426,pv,1511658038
466049,140210,1132429,pv,1511658038
38369,1780938,4357323,pv,1511658038
27948,1190005,1320293,pv,1511658038
212574,321621,4756105,pv,1511658039
793295,2941339,149192,pv,1511658039
971953,1503702,4756105,pv,1511658039
270332,1751547,4718907,pv,1511658039
261666,5141180,2355072,pv,1511658039
79249,1336042,2465336,pv,1511658039
743689,3161594,4181361,pv,1511658039
187936,1209195,4145813,pv,1511658039
954462,2653229,2448593,pv,1511658039
505372,3578525,702060,pv,1511658039
492876,2648336,35290,pv,1511658039
572164,4818203,1464116,fav,1511658039
204115,3552080,2477357,pv,1511658039
342902,679860,4756105,pv,1511658039

上面就是模拟Web日志都一条条写入了Flume，然后flume就将其导入了Kafka，然后Flink作为消费者就会去消费数据，然后利用各种窗口操作就可以了OK。

上面就是实验结果，然后我们可以将这些数据提供给前台，或者可以给后台开发提供数据，可以作为热门商品推荐之类的。