Hadoop集群及组件

it2024-12-29 8

Hadoop集群及组件

zookeeper集群图例

#mermaid-svg-BBpzjYrCFFDb8QTp .label{font-family:'trebuchet ms', verdana, arial;font-family:var(--mermaid-font-family);fill:#333;color:#333}#mermaid-svg-BBpzjYrCFFDb8QTp .label text{fill:#333}#mermaid-svg-BBpzjYrCFFDb8QTp .node rect,#mermaid-svg-BBpzjYrCFFDb8QTp .node circle,#mermaid-svg-BBpzjYrCFFDb8QTp .node ellipse,#mermaid-svg-BBpzjYrCFFDb8QTp .node polygon,#mermaid-svg-BBpzjYrCFFDb8QTp .node path{fill:#ECECFF;stroke:#9370db;stroke-width:1px}#mermaid-svg-BBpzjYrCFFDb8QTp .node .label{text-align:center;fill:#333}#mermaid-svg-BBpzjYrCFFDb8QTp .node.clickable{cursor:pointer}#mermaid-svg-BBpzjYrCFFDb8QTp .arrowheadPath{fill:#333}#mermaid-svg-BBpzjYrCFFDb8QTp .edgePath .path{stroke:#333;stroke-width:1.5px}#mermaid-svg-BBpzjYrCFFDb8QTp .flowchart-link{stroke:#333;fill:none}#mermaid-svg-BBpzjYrCFFDb8QTp .edgeLabel{background-color:#e8e8e8;text-align:center}#mermaid-svg-BBpzjYrCFFDb8QTp .edgeLabel rect{opacity:0.9}#mermaid-svg-BBpzjYrCFFDb8QTp .edgeLabel span{color:#333}#mermaid-svg-BBpzjYrCFFDb8QTp .cluster rect{fill:#ffffde;stroke:#aa3;stroke-width:1px}#mermaid-svg-BBpzjYrCFFDb8QTp .cluster text{fill:#333}#mermaid-svg-BBpzjYrCFFDb8QTp div.mermaidTooltip{position:absolute;text-align:center;max-width:200px;padding:2px;font-family:'trebuchet ms', verdana, arial;font-family:var(--mermaid-font-family);font-size:12px;background:#ffffde;border:1px solid #aa3;border-radius:2px;pointer-events:none;z-index:100}#mermaid-svg-BBpzjYrCFFDb8QTp .actor{stroke:#ccf;fill:#ECECFF}#mermaid-svg-BBpzjYrCFFDb8QTp text.actor>tspan{fill:#000;stroke:none}#mermaid-svg-BBpzjYrCFFDb8QTp .actor-line{stroke:grey}#mermaid-svg-BBpzjYrCFFDb8QTp .messageLine0{stroke-width:1.5;stroke-dasharray:none;stroke:#333}#mermaid-svg-BBpzjYrCFFDb8QTp .messageLine1{stroke-width:1.5;stroke-dasharray:2, 2;stroke:#333}#mermaid-svg-BBpzjYrCFFDb8QTp #arrowhead path{fill:#333;stroke:#333}#mermaid-svg-BBpzjYrCFFDb8QTp .sequenceNumber{fill:#fff}#mermaid-svg-BBpzjYrCFFDb8QTp #sequencenumber{fill:#333}#mermaid-svg-BBpzjYrCFFDb8QTp #crosshead path{fill:#333;stroke:#333}#mermaid-svg-BBpzjYrCFFDb8QTp .messageText{fill:#333;stroke:#333}#mermaid-svg-BBpzjYrCFFDb8QTp .labelBox{stroke:#ccf;fill:#ECECFF}#mermaid-svg-BBpzjYrCFFDb8QTp .labelText,#mermaid-svg-BBpzjYrCFFDb8QTp .labelText>tspan{fill:#000;stroke:none}#mermaid-svg-BBpzjYrCFFDb8QTp .loopText,#mermaid-svg-BBpzjYrCFFDb8QTp .loopText>tspan{fill:#000;stroke:none}#mermaid-svg-BBpzjYrCFFDb8QTp .loopLine{stroke-width:2px;stroke-dasharray:2, 2;stroke:#ccf;fill:#ccf}#mermaid-svg-BBpzjYrCFFDb8QTp .note{stroke:#aa3;fill:#fff5ad}#mermaid-svg-BBpzjYrCFFDb8QTp .noteText,#mermaid-svg-BBpzjYrCFFDb8QTp .noteText>tspan{fill:#000;stroke:none}#mermaid-svg-BBpzjYrCFFDb8QTp .activation0{fill:#f4f4f4;stroke:#666}#mermaid-svg-BBpzjYrCFFDb8QTp .activation1{fill:#f4f4f4;stroke:#666}#mermaid-svg-BBpzjYrCFFDb8QTp .activation2{fill:#f4f4f4;stroke:#666}#mermaid-svg-BBpzjYrCFFDb8QTp .mermaid-main-font{font-family:"trebuchet ms", verdana, arial;font-family:var(--mermaid-font-family)}#mermaid-svg-BBpzjYrCFFDb8QTp .section{stroke:none;opacity:0.2}#mermaid-svg-BBpzjYrCFFDb8QTp .section0{fill:rgba(102,102,255,0.49)}#mermaid-svg-BBpzjYrCFFDb8QTp .section2{fill:#fff400}#mermaid-svg-BBpzjYrCFFDb8QTp .section1,#mermaid-svg-BBpzjYrCFFDb8QTp .section3{fill:#fff;opacity:0.2}#mermaid-svg-BBpzjYrCFFDb8QTp .sectionTitle0{fill:#333}#mermaid-svg-BBpzjYrCFFDb8QTp .sectionTitle1{fill:#333}#mermaid-svg-BBpzjYrCFFDb8QTp .sectionTitle2{fill:#333}#mermaid-svg-BBpzjYrCFFDb8QTp .sectionTitle3{fill:#333}#mermaid-svg-BBpzjYrCFFDb8QTp .sectionTitle{text-anchor:start;font-size:11px;text-height:14px;font-family:'trebuchet ms', verdana, arial;font-family:var(--mermaid-font-family)}#mermaid-svg-BBpzjYrCFFDb8QTp .grid .tick{stroke:#d3d3d3;opacity:0.8;shape-rendering:crispEdges}#mermaid-svg-BBpzjYrCFFDb8QTp .grid .tick text{font-family:'trebuchet ms', verdana, arial;font-family:var(--mermaid-font-family)}#mermaid-svg-BBpzjYrCFFDb8QTp .grid path{stroke-width:0}#mermaid-svg-BBpzjYrCFFDb8QTp .today{fill:none;stroke:red;stroke-width:2px}#mermaid-svg-BBpzjYrCFFDb8QTp .task{stroke-width:2}#mermaid-svg-BBpzjYrCFFDb8QTp .taskText{text-anchor:middle;font-family:'trebuchet ms', verdana, arial;font-family:var(--mermaid-font-family)}#mermaid-svg-BBpzjYrCFFDb8QTp .taskText:not([font-size]){font-size:11px}#mermaid-svg-BBpzjYrCFFDb8QTp .taskTextOutsideRight{fill:#000;text-anchor:start;font-size:11px;font-family:'trebuchet ms', verdana, arial;font-family:var(--mermaid-font-family)}#mermaid-svg-BBpzjYrCFFDb8QTp .taskTextOutsideLeft{fill:#000;text-anchor:end;font-size:11px}#mermaid-svg-BBpzjYrCFFDb8QTp .task.clickable{cursor:pointer}#mermaid-svg-BBpzjYrCFFDb8QTp .taskText.clickable{cursor:pointer;fill:#003163 !important;font-weight:bold}#mermaid-svg-BBpzjYrCFFDb8QTp .taskTextOutsideLeft.clickable{cursor:pointer;fill:#003163 !important;font-weight:bold}#mermaid-svg-BBpzjYrCFFDb8QTp .taskTextOutsideRight.clickable{cursor:pointer;fill:#003163 !important;font-weight:bold}#mermaid-svg-BBpzjYrCFFDb8QTp .taskText0,#mermaid-svg-BBpzjYrCFFDb8QTp .taskText1,#mermaid-svg-BBpzjYrCFFDb8QTp .taskText2,#mermaid-svg-BBpzjYrCFFDb8QTp .taskText3{fill:#fff}#mermaid-svg-BBpzjYrCFFDb8QTp .task0,#mermaid-svg-BBpzjYrCFFDb8QTp .task1,#mermaid-svg-BBpzjYrCFFDb8QTp .task2,#mermaid-svg-BBpzjYrCFFDb8QTp .task3{fill:#8a90dd;stroke:#534fbc}#mermaid-svg-BBpzjYrCFFDb8QTp .taskTextOutside0,#mermaid-svg-BBpzjYrCFFDb8QTp .taskTextOutside2{fill:#000}#mermaid-svg-BBpzjYrCFFDb8QTp .taskTextOutside1,#mermaid-svg-BBpzjYrCFFDb8QTp .taskTextOutside3{fill:#000}#mermaid-svg-BBpzjYrCFFDb8QTp .active0,#mermaid-svg-BBpzjYrCFFDb8QTp .active1,#mermaid-svg-BBpzjYrCFFDb8QTp .active2,#mermaid-svg-BBpzjYrCFFDb8QTp .active3{fill:#bfc7ff;stroke:#534fbc}#mermaid-svg-BBpzjYrCFFDb8QTp .activeText0,#mermaid-svg-BBpzjYrCFFDb8QTp .activeText1,#mermaid-svg-BBpzjYrCFFDb8QTp .activeText2,#mermaid-svg-BBpzjYrCFFDb8QTp .activeText3{fill:#000 !important}#mermaid-svg-BBpzjYrCFFDb8QTp .done0,#mermaid-svg-BBpzjYrCFFDb8QTp .done1,#mermaid-svg-BBpzjYrCFFDb8QTp .done2,#mermaid-svg-BBpzjYrCFFDb8QTp .done3{stroke:grey;fill:#d3d3d3;stroke-width:2}#mermaid-svg-BBpzjYrCFFDb8QTp .doneText0,#mermaid-svg-BBpzjYrCFFDb8QTp .doneText1,#mermaid-svg-BBpzjYrCFFDb8QTp .doneText2,#mermaid-svg-BBpzjYrCFFDb8QTp .doneText3{fill:#000 !important}#mermaid-svg-BBpzjYrCFFDb8QTp .crit0,#mermaid-svg-BBpzjYrCFFDb8QTp .crit1,#mermaid-svg-BBpzjYrCFFDb8QTp .crit2,#mermaid-svg-BBpzjYrCFFDb8QTp .crit3{stroke:#f88;fill:red;stroke-width:2}#mermaid-svg-BBpzjYrCFFDb8QTp .activeCrit0,#mermaid-svg-BBpzjYrCFFDb8QTp .activeCrit1,#mermaid-svg-BBpzjYrCFFDb8QTp .activeCrit2,#mermaid-svg-BBpzjYrCFFDb8QTp .activeCrit3{stroke:#f88;fill:#bfc7ff;stroke-width:2}#mermaid-svg-BBpzjYrCFFDb8QTp .doneCrit0,#mermaid-svg-BBpzjYrCFFDb8QTp .doneCrit1,#mermaid-svg-BBpzjYrCFFDb8QTp .doneCrit2,#mermaid-svg-BBpzjYrCFFDb8QTp .doneCrit3{stroke:#f88;fill:#d3d3d3;stroke-width:2;cursor:pointer;shape-rendering:crispEdges}#mermaid-svg-BBpzjYrCFFDb8QTp .milestone{transform:rotate(45deg) scale(0.8, 0.8)}#mermaid-svg-BBpzjYrCFFDb8QTp .milestoneText{font-style:italic}#mermaid-svg-BBpzjYrCFFDb8QTp .doneCritText0,#mermaid-svg-BBpzjYrCFFDb8QTp .doneCritText1,#mermaid-svg-BBpzjYrCFFDb8QTp .doneCritText2,#mermaid-svg-BBpzjYrCFFDb8QTp .doneCritText3{fill:#000 !important}#mermaid-svg-BBpzjYrCFFDb8QTp .activeCritText0,#mermaid-svg-BBpzjYrCFFDb8QTp .activeCritText1,#mermaid-svg-BBpzjYrCFFDb8QTp .activeCritText2,#mermaid-svg-BBpzjYrCFFDb8QTp .activeCritText3{fill:#000 !important}#mermaid-svg-BBpzjYrCFFDb8QTp .titleText{text-anchor:middle;font-size:18px;fill:#000;font-family:'trebuchet ms', verdana, arial;font-family:var(--mermaid-font-family)}#mermaid-svg-BBpzjYrCFFDb8QTp g.classGroup text{fill:#9370db;stroke:none;font-family:'trebuchet ms', verdana, arial;font-family:var(--mermaid-font-family);font-size:10px}#mermaid-svg-BBpzjYrCFFDb8QTp g.classGroup text .title{font-weight:bolder}#mermaid-svg-BBpzjYrCFFDb8QTp g.clickable{cursor:pointer}#mermaid-svg-BBpzjYrCFFDb8QTp g.classGroup rect{fill:#ECECFF;stroke:#9370db}#mermaid-svg-BBpzjYrCFFDb8QTp g.classGroup line{stroke:#9370db;stroke-width:1}#mermaid-svg-BBpzjYrCFFDb8QTp .classLabel .box{stroke:none;stroke-width:0;fill:#ECECFF;opacity:0.5}#mermaid-svg-BBpzjYrCFFDb8QTp .classLabel .label{fill:#9370db;font-size:10px}#mermaid-svg-BBpzjYrCFFDb8QTp .relation{stroke:#9370db;stroke-width:1;fill:none}#mermaid-svg-BBpzjYrCFFDb8QTp .dashed-line{stroke-dasharray:3}#mermaid-svg-BBpzjYrCFFDb8QTp #compositionStart{fill:#9370db;stroke:#9370db;stroke-width:1}#mermaid-svg-BBpzjYrCFFDb8QTp #compositionEnd{fill:#9370db;stroke:#9370db;stroke-width:1}#mermaid-svg-BBpzjYrCFFDb8QTp #aggregationStart{fill:#ECECFF;stroke:#9370db;stroke-width:1}#mermaid-svg-BBpzjYrCFFDb8QTp #aggregationEnd{fill:#ECECFF;stroke:#9370db;stroke-width:1}#mermaid-svg-BBpzjYrCFFDb8QTp #dependencyStart{fill:#9370db;stroke:#9370db;stroke-width:1}#mermaid-svg-BBpzjYrCFFDb8QTp #dependencyEnd{fill:#9370db;stroke:#9370db;stroke-width:1}#mermaid-svg-BBpzjYrCFFDb8QTp #extensionStart{fill:#9370db;stroke:#9370db;stroke-width:1}#mermaid-svg-BBpzjYrCFFDb8QTp #extensionEnd{fill:#9370db;stroke:#9370db;stroke-width:1}#mermaid-svg-BBpzjYrCFFDb8QTp .commit-id,#mermaid-svg-BBpzjYrCFFDb8QTp .commit-msg,#mermaid-svg-BBpzjYrCFFDb8QTp .branch-label{fill:lightgrey;color:lightgrey;font-family:'trebuchet ms', verdana, arial;font-family:var(--mermaid-font-family)}#mermaid-svg-BBpzjYrCFFDb8QTp .pieTitleText{text-anchor:middle;font-size:25px;fill:#000;font-family:'trebuchet ms', verdana, arial;font-family:var(--mermaid-font-family)}#mermaid-svg-BBpzjYrCFFDb8QTp .slice{font-family:'trebuchet ms', verdana, arial;font-family:var(--mermaid-font-family)}#mermaid-svg-BBpzjYrCFFDb8QTp g.stateGroup text{fill:#9370db;stroke:none;font-size:10px;font-family:'trebuchet ms', verdana, arial;font-family:var(--mermaid-font-family)}#mermaid-svg-BBpzjYrCFFDb8QTp g.stateGroup text{fill:#9370db;fill:#333;stroke:none;font-size:10px}#mermaid-svg-BBpzjYrCFFDb8QTp g.statediagram-cluster .cluster-label text{fill:#333}#mermaid-svg-BBpzjYrCFFDb8QTp g.stateGroup .state-title{font-weight:bolder;fill:#000}#mermaid-svg-BBpzjYrCFFDb8QTp g.stateGroup rect{fill:#ECECFF;stroke:#9370db}#mermaid-svg-BBpzjYrCFFDb8QTp g.stateGroup line{stroke:#9370db;stroke-width:1}#mermaid-svg-BBpzjYrCFFDb8QTp .transition{stroke:#9370db;stroke-width:1;fill:none}#mermaid-svg-BBpzjYrCFFDb8QTp .stateGroup .composit{fill:white;border-bottom:1px}#mermaid-svg-BBpzjYrCFFDb8QTp .stateGroup .alt-composit{fill:#e0e0e0;border-bottom:1px}#mermaid-svg-BBpzjYrCFFDb8QTp .state-note{stroke:#aa3;fill:#fff5ad}#mermaid-svg-BBpzjYrCFFDb8QTp .state-note text{fill:black;stroke:none;font-size:10px}#mermaid-svg-BBpzjYrCFFDb8QTp .stateLabel .box{stroke:none;stroke-width:0;fill:#ECECFF;opacity:0.7}#mermaid-svg-BBpzjYrCFFDb8QTp .edgeLabel text{fill:#333}#mermaid-svg-BBpzjYrCFFDb8QTp .stateLabel text{fill:#000;font-size:10px;font-weight:bold;font-family:'trebuchet ms', verdana, arial;font-family:var(--mermaid-font-family)}#mermaid-svg-BBpzjYrCFFDb8QTp .node circle.state-start{fill:black;stroke:black}#mermaid-svg-BBpzjYrCFFDb8QTp .node circle.state-end{fill:black;stroke:white;stroke-width:1.5}#mermaid-svg-BBpzjYrCFFDb8QTp #statediagram-barbEnd{fill:#9370db}#mermaid-svg-BBpzjYrCFFDb8QTp .statediagram-cluster rect{fill:#ECECFF;stroke:#9370db;stroke-width:1px}#mermaid-svg-BBpzjYrCFFDb8QTp .statediagram-cluster rect.outer{rx:5px;ry:5px}#mermaid-svg-BBpzjYrCFFDb8QTp .statediagram-state .divider{stroke:#9370db}#mermaid-svg-BBpzjYrCFFDb8QTp .statediagram-state .title-state{rx:5px;ry:5px}#mermaid-svg-BBpzjYrCFFDb8QTp .statediagram-cluster.statediagram-cluster .inner{fill:white}#mermaid-svg-BBpzjYrCFFDb8QTp .statediagram-cluster.statediagram-cluster-alt .inner{fill:#e0e0e0}#mermaid-svg-BBpzjYrCFFDb8QTp .statediagram-cluster .inner{rx:0;ry:0}#mermaid-svg-BBpzjYrCFFDb8QTp .statediagram-state rect.basic{rx:5px;ry:5px}#mermaid-svg-BBpzjYrCFFDb8QTp .statediagram-state rect.divider{stroke-dasharray:10,10;fill:#efefef}#mermaid-svg-BBpzjYrCFFDb8QTp .note-edge{stroke-dasharray:5}#mermaid-svg-BBpzjYrCFFDb8QTp .statediagram-note rect{fill:#fff5ad;stroke:#aa3;stroke-width:1px;rx:0;ry:0}:root{--mermaid-font-family: '"trebuchet ms", verdana, arial';--mermaid-font-family: "Comic Sans MS", "Comic Sans", cursive}#mermaid-svg-BBpzjYrCFFDb8QTp .error-icon{fill:#522}#mermaid-svg-BBpzjYrCFFDb8QTp .error-text{fill:#522;stroke:#522}#mermaid-svg-BBpzjYrCFFDb8QTp .edge-thickness-normal{stroke-width:2px}#mermaid-svg-BBpzjYrCFFDb8QTp .edge-thickness-thick{stroke-width:3.5px}#mermaid-svg-BBpzjYrCFFDb8QTp .edge-pattern-solid{stroke-dasharray:0}#mermaid-svg-BBpzjYrCFFDb8QTp .edge-pattern-dashed{stroke-dasharray:3}#mermaid-svg-BBpzjYrCFFDb8QTp .edge-pattern-dotted{stroke-dasharray:2}#mermaid-svg-BBpzjYrCFFDb8QTp .marker{fill:#333}#mermaid-svg-BBpzjYrCFFDb8QTp .marker.cross{stroke:#333} :root { --mermaid-font-family: "trebuchet ms", verdana, arial;} #mermaid-svg-BBpzjYrCFFDb8QTp { color: rgba(0, 0, 0, 0.75); font: ; } leader follower follower observer

zookeeper集群

集群安装

1、重启云主机 hadoop1,node-0001,node-0002,node-0003

2、在 hadoop1 上安装配置 zookeeper，并同步给其他主机

拷贝云盘 public/hadoop/zookeeper-3.4.13.tar.gz 到hadoop1

[root@hadoop1 ~]# yum install -y java-1.8.0-openjdk-devel [root@hadoop1 ~]# tar zxf zookeeper-3.4.13.tar.gz [root@hadoop1 ~]# mv zookeeper-3.4.13 /usr/local/zookeeper [root@hadoop1 ~]# cd /usr/local/zookeeper/conf/ [root@hadoop1 conf]# cp zoo_sample.cfg zoo.cfg [root@hadoop1 conf]# vim zoo.cfg # 配置文件最后添加 server.1=node-0001:2888:3888 server.2=node-0002:2888:3888 server.3=node-0003:2888:3888 server.4=hadoop1:2888:3888:observer [root@hadoop1 ~]# for i in node-{0001..0003};do rsync -aXSH --delete /usr/local/zookeeper ${i}:/usr/local/ done

所有节点手工启动服务

[root@hadoop1 ~]# mkdir /tmp/zookeeper [root@hadoop1 ~]# grep -Po "\d+(?==${HOSTNAME})" /usr/local/zookeeper/conf/zoo.cfg >/tmp/zookeeper/myid [root@hadoop1 ~]# /usr/local/zookeeper/bin/zkServer.sh start [root@hadoop1 ~]# jps 1001 QuorumPeerMain

当所有节点启动完成以后使用命令验证:

``/usr/local/zookeeper/bin/zkServer.sh status`

zookeeper集群管理

手册地址

[root@hadoop1 ~]# yum install -y socat [root@hadoop1 ~]# socat - TCP:node-0001:2181 ruok imok zkstats在当日笔记目录 [root@hadoop1 bin]#chmod 755 zkstats [root@hadoop1 bin]# ./zkstats hadoop1 node-{0001..0003} hadoop1 Mode: observer node-0001 Mode: follower node-0002 Mode: leader node-0003 Mode: follower

kafka集群

kafka安装

1、安装配置 kafka，并同步给其他主机

拷贝云盘 public/hadoop/kafka_2.12-2.1.0.tgz 到 hadoop1

[root@hadoop1 ~]# yum install -y java-1.8.0-openjdk-devel [root@hadoop1 ~]# tar zxf kafka_2.12-2.1.0.tgz [root@hadoop1 ~]# mv kafka_2.12-2.1.0 /usr/local/kafka [root@hadoop1 ~]# for i in node-{0001..0003};do rsync -aXSH --delete /usr/local/kafka ${i}:/usr/local/ done

2、修改 node-0001,node-0002,node-0003 配置文件并启动服务

[root@node-0001 ~]# vim /usr/local/kafka/config/server.properties 21 broker.id=1 123 zookeeper.connect=node-0001:2181,node-0002:2181,node-0003:2181 [root@node-0001 ~]# /usr/local/kafka/bin/kafka-server-start.sh -daemon /usr/local/kafka/config/server.properties [root@node-0001 ~]# jps 1400 Kafka

3、验证（在不同机器上执行）

[root@node-0001 ~]# /usr/local/kafka/bin/kafka-topics.sh --create --partitions 1 --replication-factor 1 --zookeeper localhost:2181 --topic mymsg #---------------------------------------------------------------------------------------- [root@node-0002 ~]# /usr/local/kafka/bin/kafka-console-producer.sh --broker-list localhost:9092 --topic mymsg #---------------------------------------------------------------------------------------- [root@node-0003 ~]# /usr/local/kafka/bin/kafka-console-consumer.sh --bootstrap-server localhost:9092 --topic mymsg

Hadoop高可用集群

高可用架构图例

#mermaid-svg-S4dq3ucdpIsmkLxb .label{font-family:'trebuchet ms', verdana, arial;font-family:var(--mermaid-font-family);fill:#333;color:#333}#mermaid-svg-S4dq3ucdpIsmkLxb .label text{fill:#333}#mermaid-svg-S4dq3ucdpIsmkLxb .node rect,#mermaid-svg-S4dq3ucdpIsmkLxb .node circle,#mermaid-svg-S4dq3ucdpIsmkLxb .node ellipse,#mermaid-svg-S4dq3ucdpIsmkLxb .node polygon,#mermaid-svg-S4dq3ucdpIsmkLxb .node path{fill:#ECECFF;stroke:#9370db;stroke-width:1px}#mermaid-svg-S4dq3ucdpIsmkLxb .node .label{text-align:center;fill:#333}#mermaid-svg-S4dq3ucdpIsmkLxb .node.clickable{cursor:pointer}#mermaid-svg-S4dq3ucdpIsmkLxb .arrowheadPath{fill:#333}#mermaid-svg-S4dq3ucdpIsmkLxb .edgePath .path{stroke:#333;stroke-width:1.5px}#mermaid-svg-S4dq3ucdpIsmkLxb .flowchart-link{stroke:#333;fill:none}#mermaid-svg-S4dq3ucdpIsmkLxb .edgeLabel{background-color:#e8e8e8;text-align:center}#mermaid-svg-S4dq3ucdpIsmkLxb .edgeLabel rect{opacity:0.9}#mermaid-svg-S4dq3ucdpIsmkLxb .edgeLabel span{color:#333}#mermaid-svg-S4dq3ucdpIsmkLxb .cluster rect{fill:#ffffde;stroke:#aa3;stroke-width:1px}#mermaid-svg-S4dq3ucdpIsmkLxb .cluster text{fill:#333}#mermaid-svg-S4dq3ucdpIsmkLxb div.mermaidTooltip{position:absolute;text-align:center;max-width:200px;padding:2px;font-family:'trebuchet ms', verdana, arial;font-family:var(--mermaid-font-family);font-size:12px;background:#ffffde;border:1px solid #aa3;border-radius:2px;pointer-events:none;z-index:100}#mermaid-svg-S4dq3ucdpIsmkLxb .actor{stroke:#ccf;fill:#ECECFF}#mermaid-svg-S4dq3ucdpIsmkLxb text.actor>tspan{fill:#000;stroke:none}#mermaid-svg-S4dq3ucdpIsmkLxb .actor-line{stroke:grey}#mermaid-svg-S4dq3ucdpIsmkLxb .messageLine0{stroke-width:1.5;stroke-dasharray:none;stroke:#333}#mermaid-svg-S4dq3ucdpIsmkLxb .messageLine1{stroke-width:1.5;stroke-dasharray:2, 2;stroke:#333}#mermaid-svg-S4dq3ucdpIsmkLxb #arrowhead path{fill:#333;stroke:#333}#mermaid-svg-S4dq3ucdpIsmkLxb .sequenceNumber{fill:#fff}#mermaid-svg-S4dq3ucdpIsmkLxb #sequencenumber{fill:#333}#mermaid-svg-S4dq3ucdpIsmkLxb #crosshead path{fill:#333;stroke:#333}#mermaid-svg-S4dq3ucdpIsmkLxb .messageText{fill:#333;stroke:#333}#mermaid-svg-S4dq3ucdpIsmkLxb .labelBox{stroke:#ccf;fill:#ECECFF}#mermaid-svg-S4dq3ucdpIsmkLxb .labelText,#mermaid-svg-S4dq3ucdpIsmkLxb .labelText>tspan{fill:#000;stroke:none}#mermaid-svg-S4dq3ucdpIsmkLxb .loopText,#mermaid-svg-S4dq3ucdpIsmkLxb .loopText>tspan{fill:#000;stroke:none}#mermaid-svg-S4dq3ucdpIsmkLxb .loopLine{stroke-width:2px;stroke-dasharray:2, 2;stroke:#ccf;fill:#ccf}#mermaid-svg-S4dq3ucdpIsmkLxb .note{stroke:#aa3;fill:#fff5ad}#mermaid-svg-S4dq3ucdpIsmkLxb .noteText,#mermaid-svg-S4dq3ucdpIsmkLxb .noteText>tspan{fill:#000;stroke:none}#mermaid-svg-S4dq3ucdpIsmkLxb .activation0{fill:#f4f4f4;stroke:#666}#mermaid-svg-S4dq3ucdpIsmkLxb .activation1{fill:#f4f4f4;stroke:#666}#mermaid-svg-S4dq3ucdpIsmkLxb .activation2{fill:#f4f4f4;stroke:#666}#mermaid-svg-S4dq3ucdpIsmkLxb .mermaid-main-font{font-family:"trebuchet ms", verdana, arial;font-family:var(--mermaid-font-family)}#mermaid-svg-S4dq3ucdpIsmkLxb .section{stroke:none;opacity:0.2}#mermaid-svg-S4dq3ucdpIsmkLxb .section0{fill:rgba(102,102,255,0.49)}#mermaid-svg-S4dq3ucdpIsmkLxb .section2{fill:#fff400}#mermaid-svg-S4dq3ucdpIsmkLxb .section1,#mermaid-svg-S4dq3ucdpIsmkLxb .section3{fill:#fff;opacity:0.2}#mermaid-svg-S4dq3ucdpIsmkLxb .sectionTitle0{fill:#333}#mermaid-svg-S4dq3ucdpIsmkLxb .sectionTitle1{fill:#333}#mermaid-svg-S4dq3ucdpIsmkLxb .sectionTitle2{fill:#333}#mermaid-svg-S4dq3ucdpIsmkLxb .sectionTitle3{fill:#333}#mermaid-svg-S4dq3ucdpIsmkLxb .sectionTitle{text-anchor:start;font-size:11px;text-height:14px;font-family:'trebuchet ms', verdana, arial;font-family:var(--mermaid-font-family)}#mermaid-svg-S4dq3ucdpIsmkLxb .grid .tick{stroke:#d3d3d3;opacity:0.8;shape-rendering:crispEdges}#mermaid-svg-S4dq3ucdpIsmkLxb .grid .tick text{font-family:'trebuchet ms', verdana, arial;font-family:var(--mermaid-font-family)}#mermaid-svg-S4dq3ucdpIsmkLxb .grid path{stroke-width:0}#mermaid-svg-S4dq3ucdpIsmkLxb .today{fill:none;stroke:red;stroke-width:2px}#mermaid-svg-S4dq3ucdpIsmkLxb .task{stroke-width:2}#mermaid-svg-S4dq3ucdpIsmkLxb .taskText{text-anchor:middle;font-family:'trebuchet ms', verdana, arial;font-family:var(--mermaid-font-family)}#mermaid-svg-S4dq3ucdpIsmkLxb .taskText:not([font-size]){font-size:11px}#mermaid-svg-S4dq3ucdpIsmkLxb .taskTextOutsideRight{fill:#000;text-anchor:start;font-size:11px;font-family:'trebuchet ms', verdana, arial;font-family:var(--mermaid-font-family)}#mermaid-svg-S4dq3ucdpIsmkLxb .taskTextOutsideLeft{fill:#000;text-anchor:end;font-size:11px}#mermaid-svg-S4dq3ucdpIsmkLxb .task.clickable{cursor:pointer}#mermaid-svg-S4dq3ucdpIsmkLxb .taskText.clickable{cursor:pointer;fill:#003163 !important;font-weight:bold}#mermaid-svg-S4dq3ucdpIsmkLxb .taskTextOutsideLeft.clickable{cursor:pointer;fill:#003163 !important;font-weight:bold}#mermaid-svg-S4dq3ucdpIsmkLxb .taskTextOutsideRight.clickable{cursor:pointer;fill:#003163 !important;font-weight:bold}#mermaid-svg-S4dq3ucdpIsmkLxb .taskText0,#mermaid-svg-S4dq3ucdpIsmkLxb .taskText1,#mermaid-svg-S4dq3ucdpIsmkLxb .taskText2,#mermaid-svg-S4dq3ucdpIsmkLxb .taskText3{fill:#fff}#mermaid-svg-S4dq3ucdpIsmkLxb .task0,#mermaid-svg-S4dq3ucdpIsmkLxb .task1,#mermaid-svg-S4dq3ucdpIsmkLxb .task2,#mermaid-svg-S4dq3ucdpIsmkLxb .task3{fill:#8a90dd;stroke:#534fbc}#mermaid-svg-S4dq3ucdpIsmkLxb .taskTextOutside0,#mermaid-svg-S4dq3ucdpIsmkLxb .taskTextOutside2{fill:#000}#mermaid-svg-S4dq3ucdpIsmkLxb .taskTextOutside1,#mermaid-svg-S4dq3ucdpIsmkLxb .taskTextOutside3{fill:#000}#mermaid-svg-S4dq3ucdpIsmkLxb .active0,#mermaid-svg-S4dq3ucdpIsmkLxb .active1,#mermaid-svg-S4dq3ucdpIsmkLxb .active2,#mermaid-svg-S4dq3ucdpIsmkLxb .active3{fill:#bfc7ff;stroke:#534fbc}#mermaid-svg-S4dq3ucdpIsmkLxb .activeText0,#mermaid-svg-S4dq3ucdpIsmkLxb .activeText1,#mermaid-svg-S4dq3ucdpIsmkLxb .activeText2,#mermaid-svg-S4dq3ucdpIsmkLxb .activeText3{fill:#000 !important}#mermaid-svg-S4dq3ucdpIsmkLxb .done0,#mermaid-svg-S4dq3ucdpIsmkLxb .done1,#mermaid-svg-S4dq3ucdpIsmkLxb .done2,#mermaid-svg-S4dq3ucdpIsmkLxb .done3{stroke:grey;fill:#d3d3d3;stroke-width:2}#mermaid-svg-S4dq3ucdpIsmkLxb .doneText0,#mermaid-svg-S4dq3ucdpIsmkLxb .doneText1,#mermaid-svg-S4dq3ucdpIsmkLxb .doneText2,#mermaid-svg-S4dq3ucdpIsmkLxb .doneText3{fill:#000 !important}#mermaid-svg-S4dq3ucdpIsmkLxb .crit0,#mermaid-svg-S4dq3ucdpIsmkLxb .crit1,#mermaid-svg-S4dq3ucdpIsmkLxb .crit2,#mermaid-svg-S4dq3ucdpIsmkLxb .crit3{stroke:#f88;fill:red;stroke-width:2}#mermaid-svg-S4dq3ucdpIsmkLxb .activeCrit0,#mermaid-svg-S4dq3ucdpIsmkLxb .activeCrit1,#mermaid-svg-S4dq3ucdpIsmkLxb .activeCrit2,#mermaid-svg-S4dq3ucdpIsmkLxb .activeCrit3{stroke:#f88;fill:#bfc7ff;stroke-width:2}#mermaid-svg-S4dq3ucdpIsmkLxb .doneCrit0,#mermaid-svg-S4dq3ucdpIsmkLxb .doneCrit1,#mermaid-svg-S4dq3ucdpIsmkLxb .doneCrit2,#mermaid-svg-S4dq3ucdpIsmkLxb .doneCrit3{stroke:#f88;fill:#d3d3d3;stroke-width:2;cursor:pointer;shape-rendering:crispEdges}#mermaid-svg-S4dq3ucdpIsmkLxb .milestone{transform:rotate(45deg) scale(0.8, 0.8)}#mermaid-svg-S4dq3ucdpIsmkLxb .milestoneText{font-style:italic}#mermaid-svg-S4dq3ucdpIsmkLxb .doneCritText0,#mermaid-svg-S4dq3ucdpIsmkLxb .doneCritText1,#mermaid-svg-S4dq3ucdpIsmkLxb .doneCritText2,#mermaid-svg-S4dq3ucdpIsmkLxb .doneCritText3{fill:#000 !important}#mermaid-svg-S4dq3ucdpIsmkLxb .activeCritText0,#mermaid-svg-S4dq3ucdpIsmkLxb .activeCritText1,#mermaid-svg-S4dq3ucdpIsmkLxb .activeCritText2,#mermaid-svg-S4dq3ucdpIsmkLxb .activeCritText3{fill:#000 !important}#mermaid-svg-S4dq3ucdpIsmkLxb .titleText{text-anchor:middle;font-size:18px;fill:#000;font-family:'trebuchet ms', verdana, arial;font-family:var(--mermaid-font-family)}#mermaid-svg-S4dq3ucdpIsmkLxb g.classGroup text{fill:#9370db;stroke:none;font-family:'trebuchet ms', verdana, arial;font-family:var(--mermaid-font-family);font-size:10px}#mermaid-svg-S4dq3ucdpIsmkLxb g.classGroup text .title{font-weight:bolder}#mermaid-svg-S4dq3ucdpIsmkLxb g.clickable{cursor:pointer}#mermaid-svg-S4dq3ucdpIsmkLxb g.classGroup rect{fill:#ECECFF;stroke:#9370db}#mermaid-svg-S4dq3ucdpIsmkLxb g.classGroup line{stroke:#9370db;stroke-width:1}#mermaid-svg-S4dq3ucdpIsmkLxb .classLabel .box{stroke:none;stroke-width:0;fill:#ECECFF;opacity:0.5}#mermaid-svg-S4dq3ucdpIsmkLxb .classLabel .label{fill:#9370db;font-size:10px}#mermaid-svg-S4dq3ucdpIsmkLxb .relation{stroke:#9370db;stroke-width:1;fill:none}#mermaid-svg-S4dq3ucdpIsmkLxb .dashed-line{stroke-dasharray:3}#mermaid-svg-S4dq3ucdpIsmkLxb #compositionStart{fill:#9370db;stroke:#9370db;stroke-width:1}#mermaid-svg-S4dq3ucdpIsmkLxb #compositionEnd{fill:#9370db;stroke:#9370db;stroke-width:1}#mermaid-svg-S4dq3ucdpIsmkLxb #aggregationStart{fill:#ECECFF;stroke:#9370db;stroke-width:1}#mermaid-svg-S4dq3ucdpIsmkLxb #aggregationEnd{fill:#ECECFF;stroke:#9370db;stroke-width:1}#mermaid-svg-S4dq3ucdpIsmkLxb #dependencyStart{fill:#9370db;stroke:#9370db;stroke-width:1}#mermaid-svg-S4dq3ucdpIsmkLxb #dependencyEnd{fill:#9370db;stroke:#9370db;stroke-width:1}#mermaid-svg-S4dq3ucdpIsmkLxb #extensionStart{fill:#9370db;stroke:#9370db;stroke-width:1}#mermaid-svg-S4dq3ucdpIsmkLxb #extensionEnd{fill:#9370db;stroke:#9370db;stroke-width:1}#mermaid-svg-S4dq3ucdpIsmkLxb .commit-id,#mermaid-svg-S4dq3ucdpIsmkLxb .commit-msg,#mermaid-svg-S4dq3ucdpIsmkLxb .branch-label{fill:lightgrey;color:lightgrey;font-family:'trebuchet ms', verdana, arial;font-family:var(--mermaid-font-family)}#mermaid-svg-S4dq3ucdpIsmkLxb .pieTitleText{text-anchor:middle;font-size:25px;fill:#000;font-family:'trebuchet ms', verdana, arial;font-family:var(--mermaid-font-family)}#mermaid-svg-S4dq3ucdpIsmkLxb .slice{font-family:'trebuchet ms', verdana, arial;font-family:var(--mermaid-font-family)}#mermaid-svg-S4dq3ucdpIsmkLxb g.stateGroup text{fill:#9370db;stroke:none;font-size:10px;font-family:'trebuchet ms', verdana, arial;font-family:var(--mermaid-font-family)}#mermaid-svg-S4dq3ucdpIsmkLxb g.stateGroup text{fill:#9370db;fill:#333;stroke:none;font-size:10px}#mermaid-svg-S4dq3ucdpIsmkLxb g.statediagram-cluster .cluster-label text{fill:#333}#mermaid-svg-S4dq3ucdpIsmkLxb g.stateGroup .state-title{font-weight:bolder;fill:#000}#mermaid-svg-S4dq3ucdpIsmkLxb g.stateGroup rect{fill:#ECECFF;stroke:#9370db}#mermaid-svg-S4dq3ucdpIsmkLxb g.stateGroup line{stroke:#9370db;stroke-width:1}#mermaid-svg-S4dq3ucdpIsmkLxb .transition{stroke:#9370db;stroke-width:1;fill:none}#mermaid-svg-S4dq3ucdpIsmkLxb .stateGroup .composit{fill:white;border-bottom:1px}#mermaid-svg-S4dq3ucdpIsmkLxb .stateGroup .alt-composit{fill:#e0e0e0;border-bottom:1px}#mermaid-svg-S4dq3ucdpIsmkLxb .state-note{stroke:#aa3;fill:#fff5ad}#mermaid-svg-S4dq3ucdpIsmkLxb .state-note text{fill:black;stroke:none;font-size:10px}#mermaid-svg-S4dq3ucdpIsmkLxb .stateLabel .box{stroke:none;stroke-width:0;fill:#ECECFF;opacity:0.7}#mermaid-svg-S4dq3ucdpIsmkLxb .edgeLabel text{fill:#333}#mermaid-svg-S4dq3ucdpIsmkLxb .stateLabel text{fill:#000;font-size:10px;font-weight:bold;font-family:'trebuchet ms', verdana, arial;font-family:var(--mermaid-font-family)}#mermaid-svg-S4dq3ucdpIsmkLxb .node circle.state-start{fill:black;stroke:black}#mermaid-svg-S4dq3ucdpIsmkLxb .node circle.state-end{fill:black;stroke:white;stroke-width:1.5}#mermaid-svg-S4dq3ucdpIsmkLxb #statediagram-barbEnd{fill:#9370db}#mermaid-svg-S4dq3ucdpIsmkLxb .statediagram-cluster rect{fill:#ECECFF;stroke:#9370db;stroke-width:1px}#mermaid-svg-S4dq3ucdpIsmkLxb .statediagram-cluster rect.outer{rx:5px;ry:5px}#mermaid-svg-S4dq3ucdpIsmkLxb .statediagram-state .divider{stroke:#9370db}#mermaid-svg-S4dq3ucdpIsmkLxb .statediagram-state .title-state{rx:5px;ry:5px}#mermaid-svg-S4dq3ucdpIsmkLxb .statediagram-cluster.statediagram-cluster .inner{fill:white}#mermaid-svg-S4dq3ucdpIsmkLxb .statediagram-cluster.statediagram-cluster-alt .inner{fill:#e0e0e0}#mermaid-svg-S4dq3ucdpIsmkLxb .statediagram-cluster .inner{rx:0;ry:0}#mermaid-svg-S4dq3ucdpIsmkLxb .statediagram-state rect.basic{rx:5px;ry:5px}#mermaid-svg-S4dq3ucdpIsmkLxb .statediagram-state rect.divider{stroke-dasharray:10,10;fill:#efefef}#mermaid-svg-S4dq3ucdpIsmkLxb .note-edge{stroke-dasharray:5}#mermaid-svg-S4dq3ucdpIsmkLxb .statediagram-note rect{fill:#fff5ad;stroke:#aa3;stroke-width:1px;rx:0;ry:0}:root{--mermaid-font-family: '"trebuchet ms", verdana, arial';--mermaid-font-family: "Comic Sans MS", "Comic Sans", cursive}#mermaid-svg-S4dq3ucdpIsmkLxb .error-icon{fill:#522}#mermaid-svg-S4dq3ucdpIsmkLxb .error-text{fill:#522;stroke:#522}#mermaid-svg-S4dq3ucdpIsmkLxb .edge-thickness-normal{stroke-width:2px}#mermaid-svg-S4dq3ucdpIsmkLxb .edge-thickness-thick{stroke-width:3.5px}#mermaid-svg-S4dq3ucdpIsmkLxb .edge-pattern-solid{stroke-dasharray:0}#mermaid-svg-S4dq3ucdpIsmkLxb .edge-pattern-dashed{stroke-dasharray:3}#mermaid-svg-S4dq3ucdpIsmkLxb .edge-pattern-dotted{stroke-dasharray:2}#mermaid-svg-S4dq3ucdpIsmkLxb .marker{fill:#333}#mermaid-svg-S4dq3ucdpIsmkLxb .marker.cross{stroke:#333} :root { --mermaid-font-family: "trebuchet ms", verdana, arial;} #mermaid-svg-S4dq3ucdpIsmkLxb { color: rgba(0, 0, 0, 0.75); font: ; } hadoop1 NameNode ResourceManager DFSZKFailoverController hadoop2 NameNode ResourceManager DFSZKFailoverController node-0001 DataNode NodeManager JournalNode QuorumPeerMain node-0002 DataNode NodeManager JournalNode QuorumPeerMain node-0003 DataNode NodeManager JournalNode QuorumPeerMain

购买云主机

主机名IP地址基础配置hadoop2192.168.1.562cpu, 2G内存

角色与配置

主机名IP地址角色服务hadoop1192.168.1.50namenode, resourcemanager, ZKFChadoop2192.168.1.56namenode, resourcemanager, ZKFCnode-0001192.168.1.51datanode, nodemanager, zookeeper, journalnodenode-0002192.168.1.52datanode, nodemanager, zookeeper, journalnodenode-0003192.168.1.53datanode, nodemanager, zookeeper, journalnode

环境初始化

hadoop1 上执行

[root@hadoop1 ~]# vim /etc/hosts 192.168.1.50 hadoop1 192.168.1.56 hadoop2 192.168.1.51 node-0001 192.168.1.52 node-0002 192.168.1.53 node-0003 [root@hadoop1 ~]# rsync -aXSH --delete /root/.ssh hadoop2:/root/ [root@hadoop1 ~]# for i in hadoop2 node-{0001..0003};do rsync -av /etc/hosts ${i}:/etc/ done

hadoop2 上执行

[root@hadoop2 ~]# yum install -y java-1.8.0-openjdk-devel [root@hadoop2 ~]# vim /etc/ssh/ssh_config # 60行新添加 StrictHostKeyChecking no

集群配置文件

在 hadoop1 上完成以下文件的配置

1、配置 hadoop-env.sh

[root@hadoop1 ~]# vim /usr/local/hadoop/etc/hadoop/hadoop-env.sh 25: export JAVA_HOME="java-1.8.0-openjdk安装路径" 33: export HADOOP_CONF_DIR="/usr/local/hadoop/etc/hadoop"

2、配置 slaves

[root@hadoop1 ~]# vim /usr/local/hadoop/etc/hadoop/slaves node-0001 node-0002 node-0003

3、配置 core-site.xml

[root@hadoop1 ~]# vim /usr/local/hadoop/etc/hadoop/core-site.xml <configuration> <property> <name>fs.defaultFS</name> <value>hdfs://mycluster</value> </property> <property> <name>hadoop.tmp.dir</name> <value>/var/hadoop</value> </property> <property> <name>ha.zookeeper.quorum</name> <value>node-0001:2181,node-0002:2181,node-0003:2181</value> </property> <property> <name>hadoop.proxyuser.nfsuser.groups</name> <value>*</value> </property> <property> <name>hadoop.proxyuser.nfsuser.hosts</name> <value>*</value> </property> </configuration>

4、配置 hdfs-site.xml

[root@hadoop1 ~]# vim /usr/local/hadoop/etc/hadoop/hdfs-site.xml <configuration> <property> <name>dfs.nameservices</name> <value>mycluster</value> </property> <property> <name>dfs.ha.namenodes.mycluster</name> <value>nn1,nn2</value> </property> <property> <name>dfs.namenode.rpc-address.mycluster.nn1</name> <value>hadoop1:8020</value> </property> <property> <name>dfs.namenode.rpc-address.mycluster.nn2</name> <value>hadoop2:8020</value> </property> <property> <name>dfs.namenode.http-address.mycluster.nn1</name> <value>hadoop1:50070</value> </property> <property> <name>dfs.namenode.http-address.mycluster.nn2</name> <value>hadoop2:50070</value> </property> <property> <name>dfs.namenode.shared.edits.dir</name> <value>qjournal://node-0001:8485;node-0002:8485;node-0003:8485/mycluster</value> </property> <property> <name>dfs.journalnode.edits.dir</name> <value>/var/hadoop/journal</value> </property> <property> <name>dfs.client.failover.proxy.provider.mycluster</name> <value>org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider</value> </property> <property> <name>dfs.ha.fencing.methods</name> <value>sshfence</value> </property> <property> <name>dfs.ha.fencing.ssh.private-key-files</name> <value>/root/.ssh/id_rsa</value> </property> <property> <name>dfs.ha.automatic-failover.enabled</name> <value>true</value> </property> <property> <name>dfs.replication</name> <value>2</value> </property> <property> <name>dfs.hosts.exclude</name> <value>/usr/local/hadoop/etc/hadoop/exclude</value> </property> </configuration>

5、配置 mapred-site.xml

[root@hadoop1 ~]# vim /usr/local/hadoop/etc/hadoop/mapred-site.xml <configuration> <property> <name>mapreduce.framework.name</name> <value>yarn</value> </property> </configuration>

6、配置 yare-site.xml

[root@hadoop1 ~]# vim /usr/local/hadoop/etc/hadoop/yarn-site.xml <configuration> <property> <name>yarn.resourcemanager.ha.enabled</name> <value>true</value> </property> <property> <name>yarn.resourcemanager.recovery.enabled</name> <value>true</value> </property> <property> <name>yarn.resourcemanager.store.class</name> <value>org.apache.hadoop.yarn.server.resourcemanager.recovery.ZKRMStateStore</value> </property> <property> <name>yarn.resourcemanager.zk-address</name> <value>node-0001:2181,node-0002:2181,node-0003:2181</value> </property> <property> <name>yarn.resourcemanager.cluster-id</name> <value>yarn-ha</value> </property> <property> <name>yarn.resourcemanager.ha.rm-ids</name> <value>rm1,rm2</value> </property> <property> <name>yarn.resourcemanager.hostname.rm1</name> <value>hadoop1</value> </property> <property> <name>yarn.resourcemanager.hostname.rm2</name> <value>hadoop2</value> </property>  <property> <name>yarn.nodemanager.aux-services</name> <value>mapreduce_shuffle</value> </property> </configuration>

初始化启动集群

1、重启机器、在 node-0001，node-0002，node-0003 启动 zookeeper

[root@node-0001 ~]# /usr/local/zookeeper/bin/zkServer.sh start #---------------------------------------------------------------------------------------- [root@node-0002 ~]# /usr/local/zookeeper/bin/zkServer.sh start #---------------------------------------------------------------------------------------- [root@node-0003 ~]# /usr/local/zookeeper/bin/zkServer.sh start #---------------------------------------------------------------------------------------- [root@hadoop1 ~]# zkstats node-{0001..0003} node-0001 Mode: follower node-0002 Mode: leader node-0003 Mode: follower

2、清空实验数据并同步配置文件

[root@hadoop1 ~]# rm -rf /var/hadoop/* /usr/local/hadoop/logs [root@hadoop1 ~]# for i in hadoop2 node-{0001..0003};do rsync -av /etc/hosts ${i}:/etc/ rsync -aXSH --delete /var/hadoop ${i}:/var/ rsync -aXSH --delete /usr/local/hadoop ${i}:/usr/local/ done

3、在 node-0001，node-0002，node-0003 启动 journalnode 服务

[root@node-0001 ~]# /usr/local/hadoop/sbin/hadoop-daemon.sh start journalnode [root@node-0001 ~]# jps 1037 JournalNode #---------------------------------------------------------------------------------------- [root@node-0002 ~]# /usr/local/hadoop/sbin/hadoop-daemon.sh start journalnode #---------------------------------------------------------------------------------------- [root@node-0003 ~]# /usr/local/hadoop/sbin/hadoop-daemon.sh start journalnode

4、初始化（hadoop1 上执行）

[root@hadoop1 ~]# /usr/local/hadoop/bin/hdfs zkfc -formatZK [root@hadoop1 ~]# /usr/local/hadoop/bin/hdfs namenode -format [root@hadoop1 ~]# /usr/local/hadoop/bin/hdfs namenode -initializeSharedEdits [root@hadoop1 ~]# rsync -aXSH --delete /var/hadoop/dfs hadoop2:/var/hadoop/

5、停止在 node-0001，node-0002，node-0003 上的 journalnode 服务

[root@node-0001 ~]# /usr/local/hadoop/sbin/hadoop-daemon.sh stop journalnode #---------------------------------------------------------------------------------------- [root@node-0002 ~]# /usr/local/hadoop/sbin/hadoop-daemon.sh stop journalnode #---------------------------------------------------------------------------------------- [root@node-0003 ~]# /usr/local/hadoop/sbin/hadoop-daemon.sh stop journalnode

6、启动集群

#-------------------- 下面这条命令在 hadoop1 上执行 ---------------------------------------- [root@hadoop1 ~]# /usr/local/hadoop/sbin/start-all.sh #-------------------- 下面这条命令在 hadoop2 上执行 ---------------------------------------- [root@hadoop2 ~]# /usr/local/hadoop/sbin/yarn-daemon.sh start resourcemanager

验证集群

[root@hadoop1 ~]# /usr/local/hadoop/bin/hdfs haadmin -getServiceState nn1 [root@hadoop1 ~]# /usr/local/hadoop/bin/hdfs haadmin -getServiceState nn2 [root@hadoop1 ~]# /usr/local/hadoop/bin/yarn rmadmin -getServiceState rm1 [root@hadoop1 ~]# /usr/local/hadoop/bin/yarn rmadmin -getServiceState rm2 [root@hadoop1 ~]# /usr/local/hadoop/bin/hdfs dfsadmin -report [root@hadoop1 ~]# /usr/local/hadoop/bin/yarn node -list

使用高可用集群分析数据实验

[root@hadoop1 ~]# cd /usr/local/hadoop [root@hadoop1 hadoop]# ./bin/hadoop fs -mkdir /input [root@hadoop1 hadoop]# ./bin/hadoop fs -put *.txt /input/ [root@hadoop1 hadoop]# ./bin/hadoop jar share/hadoop/mapreduce/hadoop-mapreduce-examples-2.7.7.jar wordcount /input /output [root@hadoop1 hadoop]# ./bin/hadoop fs -cat /output/*

最新回复(0)