hadoop配置参考

发表于 | 分类于

hadoop内存配置

修改/home/q/hadoop/hadoop-2.3.0-cdh5.1.0/etc/hadoop/hadoop-env.sh

hadoop守护进程和namenode堆进程

1
2
export HADOOP_HEAPSIZE=8192
export HADOOP_NAMENODE_INIT_HEAPSIZE=8192

NameNode 64G

1
export HADOOP_NAMENODE_OPTS="-Xms64g -Xmx64g -XX:NewSize=50g -XX:PermSize=256m -server -XX:SurvivorRatio=3 -XX:+DisableExplicitGC -XX:+UseConcMarkSweepGC -XX:+UseFastAccessorMethods -XX:+UseCMSInitiatingOccupancyOnly -XX:CMSInitiatingOccupancyFraction=75 -XX:SoftRefLRUPolicyMSPerMB=0 -XX:+PrintClassHistogram -XX:+PrintGCDetails -XX:+PrintGCTimeStamps -XX:+PrintHeapAtGC -Xloggc:/home/q/hadoop/hadoop-2.3.0-cdh5.1.0/logs/gc.log -Dhadoop.security.logger=ERROR,RFAS "

DataNode 32G

1
export HADOOP_DATANODE_OPTS="-Xms32g -Xmx32g -XX:NewSize=20g -XX:PermSize=256m -server -XX:SurvivorRatio=3 -XX:+DisableExplicitGC -XX:+UseConcMarkSweepGC -XX:+UseFastAccessorMethods -XX:+UseCMSInitiatingOccupancyOnly -XX:CMSInitiatingOccupancyFraction=75 -XX:SoftRefLRUPolicyMSPerMB=0 -XX:+PrintClassHistogram -XX:+PrintGCDetails -XX:+PrintGCTimeStamps -XX:+PrintHeapAtGC -Xloggc:/home/q/hadoop/hadoop-2.3.0-cdh5.1.0/logs/gc.log -Dhadoop.security.logger=${HADOOP_SECURITY_LOGGER:-INFO,RFAS} -Dhdfs.audit.logger=${HDFS_AUDIT_LOGGER:-INFO,NullAppender}"

core-site.xml

修改/home/q/hadoop/hadoop-2.3.0-cdh5.1.0/etc/hadoop/core-site.xml

1
2
io.file.buffer.size=4194304
fs.trash.interval=1440

hdfs-site.xml

修改/home/q/hadoop/hadoop-2.3.0-cdh5.1.0/etc/hadoop/hdfs-site.xml

1
2
3
4
5
6
7
8
dfs.ha.fencing.ssh.connect-timeout=60000
ipc.client.connect.timeout=60000
dfs.image.transfer.bandwidthPerSec=1048576
dfs.namenode.handler.count=60
dfs.datanode.handler.count=40
dfs.datanode.balance.bandwidthPerSec=5242880
dfs.datanode.du.reserved=400000000000
dfs.datanode.max.xcievers=8192