压缩--HBase配置snappy

发表于 | 分类于

  1. 编译安装snappy

    1
    2
    3
    4
    5
    git clone https://github.com/google/snappy.git
    yum install -y automake autoconf gcc-c++ cmake libedit
    ./configure
    make
    make install

  2. 编译安装hadoop-snappy

    1
    2
    3
    4
    5
    6
    git clone https://github.com/louishust/hadoop-snappy.git
    ln -s /home/q/java/default/jre/lib/amd64/server/libjvm.so /usr/local/lib/
    mvn package
    cp target/hadoop-snappy-0.0.1-SNAPSHOT.jar /home/q/hadoop/hadoop-2.5.0-cdh5.2.0/lib/
    cp target/hadoop-snappy-0.0.1-SNAPSHOT-tar/hadoop-snappy-0.0.1-SNAPSHOT/lib/native/Linux-amd64-64/*  /home/q/hadoop/hadoop-2.5.0-cdh5.2.0/lib/native
    cp  target/hadoop-snappy-0.0.1-SNAPSHOT-tar/hadoop-snappy-0.0.1-SNAPSHOT/lib/native/Linux-amd64-64/* /home/q/hbase/hbase-0.98.6-cdh5.2.0/lib/native/Linux-amd64-64/

  3. 重新编译libhadoop以支持snappy
    首先需要protocbuf 2.5.0以上

    1
    2
    mvn clean package -Pdist,native -DskipTests -Drequire.snappy
    cp  /home/q/hadoop/hadoop-2.5.0-cdh5.2.0/src/hadoop-dist/target/hadoop-2.5.0-cdh5.2.0/lib/native/* /home/q/hadoop/hadoop-2.5.0-cdh5.2.0/lib/native

  4. 修改hadoop配置文件
    core-site.xml

    1
    2
    3
    <property>     
    <name>io.compression.codecs</name>     <value>org.apache.hadoop.io.compress.SnappyCodec</value>
    </property>

hadoop-env.sh

1
2
export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/home/q/hadoop/hadoop-2.5.0-cdh5.2.0/lib/native:/usr/local/lib/
export JAVA_LIBRARY_PATH=$JAVA_LIBRARY_PATH:/home/q/hadoop/hadoop-2.5.0-cdh5.2.0/lib/native:/home/q/hadoop/hadoop-2.5.0-cdh5.2.0/lib

yarn-site.xml

1
2
3
4
5
6
7
8
<property> 
<name>mapreduce.map.output.compress</name>
     <value>true</value>
</property>
<property>
     <name>mapred.map.output.compress.codec</name>
     <value>org.apache.hadoop.io.compress.SnappyCodec</value>
</property>

  1. 修改hbase配置文件hbase-env.sh

    1
    2
    3
    4
    ### for snappyexport HBASE_HOME=/home/q/hbase/hbase-0.98.6-cdh5.2.0/
    export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$HADOOP_HOME/lib/native/:/usr/local/lib/
    export HBASE_LIBRARY_PATH=$HBASE_LIBRARY_PATH:$HBASE_HOME/lib/native/Linux-amd64-64/:/usr/local/lib/
    export CLASSPATH=$CLASSPATH:$HBASE_LIBRARY_PATH

  2. 配置libjvm

    1
    sudo ln -s /home/q/java/default/jre/lib/amd64/server/libjvm.so /usr/local/lib/

  3. 重启集群

    1
    2
    3
    4
    bin/stop-hbase.sh
    sbin/stop-dfs.sh
    sbin/start-dfs.sh
    bin/start-hbase.sh

h2. 8. 验证snappy

1
2
3
4
touch /tmp/a
hbase org.apache.hadoop.hbase.util.CompressionTest /tmp/a snappy
bin/hbase shell
create 'tsnappy',{ NAME => 'f', COMPRESSION => 'snappy'}