Flume组件安装配置


目录:

Flume 组件安装配置

安装

[root@master software]# tar -xvf apache-flume-1.7.0-bin.tar.gz  -C /usr/local/
[root@master software]# ln -s /usr/local/apache-flume-1.7.0-bin/ /usr/local/flume

配置Flume

[root@master software]# vim /etc/profile
export JAVA_HOME=/usr/local/jdk
export HADOOP_HOME=/usr/local/hadoop
export HIVE_HOME=/usr/local/hive
export ZOOKEEPER_HOME=/usr/local/zookeeper
export HBASE_HOME=/usr/local/hbase

export SQOOP_HOME=/usr/local/sqoop
export CLASSPATH=$CLASSPATH:$SQOOP_HOME/lib
export FLUME_HOME=/usr/local/flume 

export PATH=$PATH:$JAVA_HOME/bin:$HADOOP_HOME/bin:$HADOOP_HOME/sbin:$HIVE_HOME/bin:$ZOOKEEPER_HOME/bin:$HBASE_HOME/bin:$SQOOP_HOME/bin:$FLUME_HOME/bin

[root@master software]# source /etc/profile

# 修改并配置 flume-env.sh 文件
[root@master software]# cd /usr/local/flume/conf/
[root@master conf]# cp flume-env.sh.template  flume-env.sh

[root@master conf]# vim flume-env.sh
export JAVA_HOME=/usr/local/jdk

[root@master conf]# chown hadoop.hadoop -R /usr/local/flume/

[root@master ~]#su - hadoop
[hadoop@master ~]$source /etc/profile
[hadoop@master ~]$ flume-ng version
Flume 1.7.0
Source code repository: https://git-wip-us.apache.org/repos/asf/flume.git
Revision: 511d868555dd4d16e6ce4fedc72c2d1454546707
Compiled by bessbd on Wed Oct 12 20:51:10 CEST 2016
From source with checksum 0d21b3ffdc55a07e1d08875872c00523

使用 Flume 发送和接受信息

[hadoop@master flume]$ vim /usr/local/src/flume/simple-hdfs-flume.conf

a1.sources=r1
a1.sinks=k1
a1.channels=c1
a1.sources.r1.type=spooldir
a1.sources.r1.spoolDir=/usr/local/hadoop/logs/
a1.sources.r1.fileHeader=true
a1.sinks.k1.type=hdfs
a1.sinks.k1.hdfs.path=hdfs://master:9000/tmp/flume
a1.sinks.k1.hdfs.rollsize=1048760
a1.sinks.k1.hdfs.rollCount=0
a1.sinks.k1.hdfs.rollInterval=900
a1.sinks.k1.hdfs.useLocalTimeStamp=true
a1.channels.c1.type=file
a1.channels.c1.capacity=1000
a1.channels.c1.transactionCapacity=100
a1.sources.r1.channels = c1
a1.sinks.k1.channel = c1

# 使用 flume-ng agent 命令加载 simple-hdfs-flume.conf 配置信息,启动 flume 传输数据。
[hadoop@master flume] $ flume-ng agent --conf-file simple-hdfs-flume.conf --name a1

# ctrl+c 退出 flume 传输
# 查看 Flume 传输到 HDFS 的文件,若能查看到 HDFS 上/tmp/flume 目录有传输的数据文件,则表示数据传输成功。
[hadoop@master flume] $ hdfs dfs -ls /tmp/flume