(base) [root@lijia1 install]# tar -zxf flume-ng-1.6.0-cdh5.14.0.tar.gz -C …/bigdata/ (base) [root@lijia1 bigdata]# mv apache-flume-1.6.0-cdh5.14.0-bin/ flume160514 (base) [root@lijia1 bigdata]# cd ./flume160514/ (base) [root@lijia1 flume160514]# cd ./conf/ (base) [root@lijia1 conf]# cp flume-env.sh.template flume-env.sh (base) [root@lijia1 conf]# vi ./flume-env.sh export JAVA_HOME=/opt/bigdata/jdk180/ export JAVA_OPTS="-Xms2000m -Xmx2000m -Dcom.sun.management.jmxremote"
(base) [root@lijia1 conf]# mkdir job
(base) [root@lijia1 job]# yum install -y nc (base) [root@lijia1 job]# nc -lk 44444
[root@lijia1 ~]# yum list telnet* 列出telnet相关的安装包 [root@lijia1 ~]# yum install telnet-server 安装telnet服务 [root@lijia1 ~]# yum install telnet.* 安装telnet客户端
[root@lijia1 ~]# telnet localhost 44444
// 读取电脑指定端口44444内容 [root@lijia1 job]# vi netcat-flume-logger.conf a1.sources = r1 a1.sinks = k1 a1.channels = c1
a1.sources.r1.type = netcat a1.sources.r1.bind = localhost a1.sources.r1.port = 44444
a1.sinks.k1.type = logger
a1.channels.c1.type = memory a1.channels.c1.capacity = 1000 a1.channels.c1.transactionCapacity = 100
a1.sources.r1.channels = c1 a1.sinks.k1.channel = c1
[root@lijia1 flume160514]# ./bin/flume-ng agent --name a1 --conf conf/ --conf-file conf/job/netcat-flume-logger.conf -Dflume.root.logger=INFO,console
// 读取指定文件内容 [root@lijia1 job]# vi file-flume-logger.conf a2.sources = r1 a2.sinks = k1 a2.channels = c1
a2.sources.r1.type = exec a2.sources.r1.command = tail -f /opt/bigdata/flume160514/conf/job/tmp.txt
a2.sinks.k1.type = logger
a2.channels.c1.type = memory a2.channels.c1.capacity = 1000 a2.channels.c1.transactionCapacity = 100
a2.sources.r1.channels = c1 a2.sinks.k1.channel = c1
[root@lijia1 flume160514]# ./bin/flume-ng agent --name a2 --conf conf/ --conf-file conf/job/file-flume-logger.conf -Dflume.root.logger=INFO,console
[root@lijia1 prodata]# wc -l events.csv 查看文件总行数
// 读取指定文件夹中复合正则表达式的文件内容 [root@lijia1 job]# vi ./events-flume-logger.conf events.sources = eventsSource events.channels = eventsChannel events.sinks = eventsSink
events.sinks.eventsSink.type = logger
events.sources.eventsSource.type = spooldir events.sources.eventsSource.spoolDir = /opt/kb07file/flumeFile/events events.sources.eventsSource.deserializer = LINE events.sources.eventsSource.deserializer.maxLineLength = 32000 events.sources.eventsSource.includePattern = events_[0-9]{4}-[0-9]{2}-[0-9]{2}.csv
events.channels.eventsChannel.type = file events.channels.eventsChannel.checkpointDir = /opt/kb07file/flumeFile/checkpoint/events events.channels.eventsChannel.dataDirs = /opt/kb07file/flumeFile/data/events
events.sources.eventsSource.channels = eventsChannel events.sinks.eventsSink.channel = eventsChannel
[root@lijia1 flume160514]# ./bin/flume-ng agent --name events --conf conf/ --conf-file conf/job/events-flume-logger.conf -Dflume.root.logger=INFO,console
