Jusene's Blog

Hadoop 2.6.5 安装

字数统计: 1k阅读时长: 5 min
2020/06/08 Share

集群准备

主机名 ip 用途 |
master 10.211.55.16 resourcemanager,namenode,secondarynamenode
slave1 10.211.55.17 nodemanager,datanode
slave2 10.211.55.18 nodemanager,datanode
  • 双机互信
  • hosts文件
  • 安装jdk

下载地址:http://archive.apache.org/dist/hadoop/common/hadoop-2.6.5

配置

1
tar xf hadoop-2.6.5.tar.gz -C /usr/local/
2
ln -s /usr/local/hadoop-2.6.5 /usr/local/hadoop
3
4
yum install -y java-1.8.0-openjdk-devel

修改配置文件:

  1. 添加JAVA_HOME配置
    1
    cat /usr/local/hadoop/etc/hadoop/hadoop-env.sh
    2
    ....
    3
    export JAVA_HOME=/usr/lib/jvm/java-1.8.0-openjdk-1.8.0.252.b09-2.el7_8.x86_64/jre
1
cat /usr/local/hadoop/etc/hadoop/yarn-env.sh
2
....
3
export JAVA_HOME=/usr/lib/jvm/java-1.8.0-openjdk-1.8.0.252.b09-2.el7_8.x86_64/jre
  1. 修改配置文件,添加从节点主机名

    1
    cat /usr/local/hadoop/etc/hadoop/slaves
    2
    slave1
    3
    slave2
  2. 修改配置文件,添加RPC配置

    1
    cat /usr/local/hadoop/etc/hadoop/core-site.xml
    2
    <configuration>
    3
        <property>
    4
            <name>fs.defaultFS</name>
    5
            <value>hdfs://master:9000</value>
    6
        </property>
    7
        <property>
    8
            <name>hadoop.tmp.dir</name>
    9
            <value>file:/usr/local/hadoop/tmp</value>
    10
        </property>
    11
    </configuration>
  3. 修改配置文件,添加DFS配置

    1
    cat /usr/local/hadoop/etc/hadoop/hdfs-site.xml
    2
    <configuration>
    3
        <!-- datanode 的副本数-->
    4
        <property>
    5
            <name>dfs.replication</name>
    6
            <value>3</value>
    7
        </property>
    8
        <property>
    9
            <name>dfs.namenode.secondary.http-address</name>
    10
            <value>master:9001</value>
    11
        </property>
    12
        <property>
    13
            <name>dfs.namenode.name.dir</name>
    14
            <value>file:/usr/local/hadoop/dfs/name</value>
    15
        </property>
    16
        <property>
    17
            <name>dfs.datanode.data.dir</name>
    18
            <value>file:/usr/local/hadoop/dfs/data</value>
    19
        </property>
    20
    </configuration>
  4. 添加配置文件 MR配置

    1
    cp /usr/local/hadoop/etc/hadoop/mapred-site.xml.template /usr/local/hadoop/etc/hadoop/mapred-site.xml
    2
    cat /usr/local/hadoop/etc/hadoop/mapred-site.xml
    3
    <configuration>
    4
        <property>
    5
            <name>mapreduce.framework.name</name>
    6
            <value>yarn</value>
    7
        </property>
    8
        <property>
    9
            <name>mapreduce.jobhistory.address</name>
    10
            <value>http://master:10020</value>
    11
        </property>
    12
        <property>
    13
            <name>mapreduce.jobhistory.webapp.address</name>
    14
            <value>http://master:19888</value>
    15
        </property>
    16
    </configuration>
  5. 修改配置文件 添加管理配置

    1
    cat /usr/local/hadoop/etc/hadoop/yarn-site.xml
    2
    <configuration>
    3
        <property>
    4
            <name>yarn.nodemanager.aux-services</name>
    5
            <value>mapreduce_shuffle</value>
    6
        </property>
    7
        <property>
    8
            <name>yarn.nodemanager.aux-services.mapreduce.shuffle.class</name>
    9
            <value>org.apache.hadoop.mapred.ShuffleHandler</value>
    10
        </property>
    11
        <property>
    12
            <name>yarn.resourcemanager.address</name>
    13
            <value>master:8032</value>
    14
        </property>
    15
        <property>
    16
            <name>yarn.resourcemanager.scheduler.address</name>
    17
            <value>master:8030</value>
    18
        </property>
    19
        <property>
    20
            <name>yarn.resourcemanager.resource-tracker.address</name>
    21
            <value>master:8035</value>
    22
        </property>
    23
        <property>
    24
            <name>yarn.resourcemanager.admin.address</name>
    25
            <value>master:8033</value>
    26
        </property>
    27
        <property>
    28
            <name>yarn.resourcemanager.webapp.address</name>
    29
            <value>master:8088</value>    
    30
        </property>
    31
    32
        <!--  关闭虚拟内存检查 -->
    33
        <property>
    34
            <name>yarn.nodemanager.vmem-check-enabled</name>
    35
            <value>false</value>
    36
        </property>
    37
    </configuration>
  6. 创建临时目录和文件目录

    1
    mkdir -p /usr/local/hadoop/tmp
    2
    mkdir -p /usr/local/hadoop/dfs/name
    3
    mkdir -p /usr/local/hadoop/dfs/data
  7. 设置安装环境

    1
    vim ~/.bashrc
    2
    export HADOOP_HOME=/usr/local/hadoop
    3
    export PATH=$PATH:$HADOOP_HOME/bin
    4
    5
    source ~/.bashrc
  8. 拷贝安装包

    1
    scp -r /usr/local/hadoop root@slave1:/usr/local/
    2
    scp -r /usr/local/hadoop root@slave2:/usr/local/
  9. 启动集群

  • master
  • 初始化namenode
    1
    hadoop namenode -format
  • 启动集群
    1
    /usr/local/hadoop/sbin/start-all.sh
  1. 启动历史服务器

    1
    /usr/local/hadoop/sbin/mr-jobhistory-daemon.sh start historyserver
  2. resourcemanager web

  1. namenode web
CATALOG
  1. 1. 集群准备
  2. 2. 配置