Hadoop 环境搭建

使用Hadoop 1.2版本

#下载
dockerlee@lee-PC:source$ wget http://mirror.bit.edu.cn/apache/hadoop/common/hadoop-1.2.1/hadoop-1.2.1.tar.gz

#解压
dockerlee@lee-PC:source$ tar -zxvf hadoop-1.2.1.tar.gz

#移动到/opt目录下
dockerlee@lee-PC:source$ sudo mv hadoop-1.2.1 /opt/

#更改hadoop配置文件，该目录下
dockerlee@lee-PC:conf$ pwd
/opt/hadoop-1.2.1/conf

修改hadoop-env.sh文件

#查看JAVA_HOME
dockerlee@lee-PC:conf$ echo $JAVA_HOME
/usr/lib/jvm/java-7-openjdk-amd64

#修改hadoop-env.sh文件
#将其中# export JAVA_HOME=/usr/lib/j2sdk1.5-sun 修改为
dockerlee@lee-PC:conf$ vim hadoop-env.sh 
export JAVA_HOME=/usr/lib/jvm/java-7-openjdk-amd64

修改core-site.xml文件，需要修改lee-PC为主机名

dockerlee@lee-PC:conf$ vim core-site.xml

<?xml version="1.0"?>
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>

<!-- Put site-specific property overrides in this file. -->

<configuration>
<property>
<name>hadoop.tmp.dir</name>
<value>/hadoop</value>
</property>

<property>
<name>dfs.name.dir</name>
<value>/hadoop/name</value>
</property>

<property>
<name>fs.default.name</name>
<value>hdfs://localhost:9000</value>
<!-- <value>hdfs://lee-PC:9000</value> -->
<!-- 主机名:端口号 -->
</property>
</configuration>

第一个是配置hadoop的工作目录

修改hdfs-site.xml文件

dockerlee@lee-PC:conf$ vim hdfs-site.xml

<?xml version="1.0"?>
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>

<!-- Put site-specific property overrides in this file. -->

<configuration>
<property>
<name>dfs.data.dir</name>
<value>/hadoop/data</value>
</property>
</configuration>

修改mapred-site.xml文件，需要修改lee-PC为主机名

dockerlee@lee-PC:conf$ vim mapred-site.xml

<?xml version="1.0"?>
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>

<!-- Put site-specific property overrides in this file. -->

<configuration>
<property>
<name>mapred.job.tracker</name>
<value>lee-PC:9001</value>
</property>
</configuration>

修改环境变量

dockerlee@lee-PC:conf$ sudo vim /etc/profile

export JAVA_HOME=/usr/lib/jvm/java-7-openjdk-amd64
export JRE_HOME=$JAVA_HOME/jre
export CLASSPATH=$JAVA_HOME/lib:$JRE_HOME/lib:$CLASSPATH
export HADOOP_HOME=/opt/hadoop-1.2.1
export PATH=$JAVA_HOME/bin:$JRE_HOME/bin:$HADOOP_HOME/bin:$PATH

#使环境变量生效
dockerlee@lee-PC:conf$ source /etc/profile

hadoop初始操作

#切换目录
dockerlee@lee-PC:bin$ pwd
/opt/hadoop-1.2.1/bin

#格式化操作
dockerlee@lee-PC:bin$ hadoop namenode -format

dockerlee@lee-PC:bin$ ./start-all.sh
#会要求输入密码

错误处理

#错误情况
namenode running as process 15997. Stop it first.
root@localhost's password: 
localhost: Permission denied, please try again.
root@localhost's password:

#解决办法，添加key
ssh-keygen -t rsa -C "youremail@example.com"
cat id_rsa.pub >> authorized_keys

#错误情况
localhost: Error: JAVA_HOME is not set.

#解决办法
#修改conf/hadoop-env.sh文件

export HADOOP_NAMENODE_OPTS="-Dcom.sun.management.jmxremote $HADOOP_NAMENODE_OPTS"
export HADOOP_SECONDARYNAMENODE_OPTS="-Dcom.sun.management.jmxremote $HADOOP_SECONDARYNAMENODE_OPTS"
export HADOOP_DATANODE_OPTS="-Dcom.sun.management.jmxremote $HADOOP_DATANODE_OPTS"
export HADOOP_BALANCER_OPTS="-Dcom.sun.management.jmxremote $HADOOP_BALANCER_OPTS"
export HADOOP_JOBTRACKER_OPTS="-Dcom.sun.management.jmxremote $HADOOP_JOBTRACKER_OPTS"
export JAVA_HOME=/usr/lib/jvm/java-7-openjdk-amd64

#错误情况
2016-08-31 16:22:01,909 ERROR org.apache.hadoop.hdfs.server.datanode.DataNode: java.io.IOException: Incompatible namespac
eIDs in /hadoop/data: namenode namespaceID = 2123037291; datanode namespaceID = 649473181

#解决办法
删除dfs.data.dir所有文件