Hadoop集群管理

it2025-01-19 35

Hadoop集群管理

集群架构图例

#mermaid-svg-sz5RHwEBnuvkDMp0 .label{font-family:'trebuchet ms', verdana, arial;font-family:var(--mermaid-font-family);fill:#333;color:#333}#mermaid-svg-sz5RHwEBnuvkDMp0 .label text{fill:#333}#mermaid-svg-sz5RHwEBnuvkDMp0 .node rect,#mermaid-svg-sz5RHwEBnuvkDMp0 .node circle,#mermaid-svg-sz5RHwEBnuvkDMp0 .node ellipse,#mermaid-svg-sz5RHwEBnuvkDMp0 .node polygon,#mermaid-svg-sz5RHwEBnuvkDMp0 .node path{fill:#ECECFF;stroke:#9370db;stroke-width:1px}#mermaid-svg-sz5RHwEBnuvkDMp0 .node .label{text-align:center;fill:#333}#mermaid-svg-sz5RHwEBnuvkDMp0 .node.clickable{cursor:pointer}#mermaid-svg-sz5RHwEBnuvkDMp0 .arrowheadPath{fill:#333}#mermaid-svg-sz5RHwEBnuvkDMp0 .edgePath .path{stroke:#333;stroke-width:1.5px}#mermaid-svg-sz5RHwEBnuvkDMp0 .flowchart-link{stroke:#333;fill:none}#mermaid-svg-sz5RHwEBnuvkDMp0 .edgeLabel{background-color:#e8e8e8;text-align:center}#mermaid-svg-sz5RHwEBnuvkDMp0 .edgeLabel rect{opacity:0.9}#mermaid-svg-sz5RHwEBnuvkDMp0 .edgeLabel span{color:#333}#mermaid-svg-sz5RHwEBnuvkDMp0 .cluster rect{fill:#ffffde;stroke:#aa3;stroke-width:1px}#mermaid-svg-sz5RHwEBnuvkDMp0 .cluster text{fill:#333}#mermaid-svg-sz5RHwEBnuvkDMp0 div.mermaidTooltip{position:absolute;text-align:center;max-width:200px;padding:2px;font-family:'trebuchet ms', verdana, arial;font-family:var(--mermaid-font-family);font-size:12px;background:#ffffde;border:1px solid #aa3;border-radius:2px;pointer-events:none;z-index:100}#mermaid-svg-sz5RHwEBnuvkDMp0 .actor{stroke:#ccf;fill:#ECECFF}#mermaid-svg-sz5RHwEBnuvkDMp0 text.actor>tspan{fill:#000;stroke:none}#mermaid-svg-sz5RHwEBnuvkDMp0 .actor-line{stroke:grey}#mermaid-svg-sz5RHwEBnuvkDMp0 .messageLine0{stroke-width:1.5;stroke-dasharray:none;stroke:#333}#mermaid-svg-sz5RHwEBnuvkDMp0 .messageLine1{stroke-width:1.5;stroke-dasharray:2, 2;stroke:#333}#mermaid-svg-sz5RHwEBnuvkDMp0 #arrowhead path{fill:#333;stroke:#333}#mermaid-svg-sz5RHwEBnuvkDMp0 .sequenceNumber{fill:#fff}#mermaid-svg-sz5RHwEBnuvkDMp0 #sequencenumber{fill:#333}#mermaid-svg-sz5RHwEBnuvkDMp0 #crosshead path{fill:#333;stroke:#333}#mermaid-svg-sz5RHwEBnuvkDMp0 .messageText{fill:#333;stroke:#333}#mermaid-svg-sz5RHwEBnuvkDMp0 .labelBox{stroke:#ccf;fill:#ECECFF}#mermaid-svg-sz5RHwEBnuvkDMp0 .labelText,#mermaid-svg-sz5RHwEBnuvkDMp0 .labelText>tspan{fill:#000;stroke:none}#mermaid-svg-sz5RHwEBnuvkDMp0 .loopText,#mermaid-svg-sz5RHwEBnuvkDMp0 .loopText>tspan{fill:#000;stroke:none}#mermaid-svg-sz5RHwEBnuvkDMp0 .loopLine{stroke-width:2px;stroke-dasharray:2, 2;stroke:#ccf;fill:#ccf}#mermaid-svg-sz5RHwEBnuvkDMp0 .note{stroke:#aa3;fill:#fff5ad}#mermaid-svg-sz5RHwEBnuvkDMp0 .noteText,#mermaid-svg-sz5RHwEBnuvkDMp0 .noteText>tspan{fill:#000;stroke:none}#mermaid-svg-sz5RHwEBnuvkDMp0 .activation0{fill:#f4f4f4;stroke:#666}#mermaid-svg-sz5RHwEBnuvkDMp0 .activation1{fill:#f4f4f4;stroke:#666}#mermaid-svg-sz5RHwEBnuvkDMp0 .activation2{fill:#f4f4f4;stroke:#666}#mermaid-svg-sz5RHwEBnuvkDMp0 .mermaid-main-font{font-family:"trebuchet ms", verdana, arial;font-family:var(--mermaid-font-family)}#mermaid-svg-sz5RHwEBnuvkDMp0 .section{stroke:none;opacity:0.2}#mermaid-svg-sz5RHwEBnuvkDMp0 .section0{fill:rgba(102,102,255,0.49)}#mermaid-svg-sz5RHwEBnuvkDMp0 .section2{fill:#fff400}#mermaid-svg-sz5RHwEBnuvkDMp0 .section1,#mermaid-svg-sz5RHwEBnuvkDMp0 .section3{fill:#fff;opacity:0.2}#mermaid-svg-sz5RHwEBnuvkDMp0 .sectionTitle0{fill:#333}#mermaid-svg-sz5RHwEBnuvkDMp0 .sectionTitle1{fill:#333}#mermaid-svg-sz5RHwEBnuvkDMp0 .sectionTitle2{fill:#333}#mermaid-svg-sz5RHwEBnuvkDMp0 .sectionTitle3{fill:#333}#mermaid-svg-sz5RHwEBnuvkDMp0 .sectionTitle{text-anchor:start;font-size:11px;text-height:14px;font-family:'trebuchet ms', verdana, arial;font-family:var(--mermaid-font-family)}#mermaid-svg-sz5RHwEBnuvkDMp0 .grid .tick{stroke:#d3d3d3;opacity:0.8;shape-rendering:crispEdges}#mermaid-svg-sz5RHwEBnuvkDMp0 .grid .tick text{font-family:'trebuchet ms', verdana, arial;font-family:var(--mermaid-font-family)}#mermaid-svg-sz5RHwEBnuvkDMp0 .grid path{stroke-width:0}#mermaid-svg-sz5RHwEBnuvkDMp0 .today{fill:none;stroke:red;stroke-width:2px}#mermaid-svg-sz5RHwEBnuvkDMp0 .task{stroke-width:2}#mermaid-svg-sz5RHwEBnuvkDMp0 .taskText{text-anchor:middle;font-family:'trebuchet ms', verdana, arial;font-family:var(--mermaid-font-family)}#mermaid-svg-sz5RHwEBnuvkDMp0 .taskText:not([font-size]){font-size:11px}#mermaid-svg-sz5RHwEBnuvkDMp0 .taskTextOutsideRight{fill:#000;text-anchor:start;font-size:11px;font-family:'trebuchet ms', verdana, arial;font-family:var(--mermaid-font-family)}#mermaid-svg-sz5RHwEBnuvkDMp0 .taskTextOutsideLeft{fill:#000;text-anchor:end;font-size:11px}#mermaid-svg-sz5RHwEBnuvkDMp0 .task.clickable{cursor:pointer}#mermaid-svg-sz5RHwEBnuvkDMp0 .taskText.clickable{cursor:pointer;fill:#003163 !important;font-weight:bold}#mermaid-svg-sz5RHwEBnuvkDMp0 .taskTextOutsideLeft.clickable{cursor:pointer;fill:#003163 !important;font-weight:bold}#mermaid-svg-sz5RHwEBnuvkDMp0 .taskTextOutsideRight.clickable{cursor:pointer;fill:#003163 !important;font-weight:bold}#mermaid-svg-sz5RHwEBnuvkDMp0 .taskText0,#mermaid-svg-sz5RHwEBnuvkDMp0 .taskText1,#mermaid-svg-sz5RHwEBnuvkDMp0 .taskText2,#mermaid-svg-sz5RHwEBnuvkDMp0 .taskText3{fill:#fff}#mermaid-svg-sz5RHwEBnuvkDMp0 .task0,#mermaid-svg-sz5RHwEBnuvkDMp0 .task1,#mermaid-svg-sz5RHwEBnuvkDMp0 .task2,#mermaid-svg-sz5RHwEBnuvkDMp0 .task3{fill:#8a90dd;stroke:#534fbc}#mermaid-svg-sz5RHwEBnuvkDMp0 .taskTextOutside0,#mermaid-svg-sz5RHwEBnuvkDMp0 .taskTextOutside2{fill:#000}#mermaid-svg-sz5RHwEBnuvkDMp0 .taskTextOutside1,#mermaid-svg-sz5RHwEBnuvkDMp0 .taskTextOutside3{fill:#000}#mermaid-svg-sz5RHwEBnuvkDMp0 .active0,#mermaid-svg-sz5RHwEBnuvkDMp0 .active1,#mermaid-svg-sz5RHwEBnuvkDMp0 .active2,#mermaid-svg-sz5RHwEBnuvkDMp0 .active3{fill:#bfc7ff;stroke:#534fbc}#mermaid-svg-sz5RHwEBnuvkDMp0 .activeText0,#mermaid-svg-sz5RHwEBnuvkDMp0 .activeText1,#mermaid-svg-sz5RHwEBnuvkDMp0 .activeText2,#mermaid-svg-sz5RHwEBnuvkDMp0 .activeText3{fill:#000 !important}#mermaid-svg-sz5RHwEBnuvkDMp0 .done0,#mermaid-svg-sz5RHwEBnuvkDMp0 .done1,#mermaid-svg-sz5RHwEBnuvkDMp0 .done2,#mermaid-svg-sz5RHwEBnuvkDMp0 .done3{stroke:grey;fill:#d3d3d3;stroke-width:2}#mermaid-svg-sz5RHwEBnuvkDMp0 .doneText0,#mermaid-svg-sz5RHwEBnuvkDMp0 .doneText1,#mermaid-svg-sz5RHwEBnuvkDMp0 .doneText2,#mermaid-svg-sz5RHwEBnuvkDMp0 .doneText3{fill:#000 !important}#mermaid-svg-sz5RHwEBnuvkDMp0 .crit0,#mermaid-svg-sz5RHwEBnuvkDMp0 .crit1,#mermaid-svg-sz5RHwEBnuvkDMp0 .crit2,#mermaid-svg-sz5RHwEBnuvkDMp0 .crit3{stroke:#f88;fill:red;stroke-width:2}#mermaid-svg-sz5RHwEBnuvkDMp0 .activeCrit0,#mermaid-svg-sz5RHwEBnuvkDMp0 .activeCrit1,#mermaid-svg-sz5RHwEBnuvkDMp0 .activeCrit2,#mermaid-svg-sz5RHwEBnuvkDMp0 .activeCrit3{stroke:#f88;fill:#bfc7ff;stroke-width:2}#mermaid-svg-sz5RHwEBnuvkDMp0 .doneCrit0,#mermaid-svg-sz5RHwEBnuvkDMp0 .doneCrit1,#mermaid-svg-sz5RHwEBnuvkDMp0 .doneCrit2,#mermaid-svg-sz5RHwEBnuvkDMp0 .doneCrit3{stroke:#f88;fill:#d3d3d3;stroke-width:2;cursor:pointer;shape-rendering:crispEdges}#mermaid-svg-sz5RHwEBnuvkDMp0 .milestone{transform:rotate(45deg) scale(0.8, 0.8)}#mermaid-svg-sz5RHwEBnuvkDMp0 .milestoneText{font-style:italic}#mermaid-svg-sz5RHwEBnuvkDMp0 .doneCritText0,#mermaid-svg-sz5RHwEBnuvkDMp0 .doneCritText1,#mermaid-svg-sz5RHwEBnuvkDMp0 .doneCritText2,#mermaid-svg-sz5RHwEBnuvkDMp0 .doneCritText3{fill:#000 !important}#mermaid-svg-sz5RHwEBnuvkDMp0 .activeCritText0,#mermaid-svg-sz5RHwEBnuvkDMp0 .activeCritText1,#mermaid-svg-sz5RHwEBnuvkDMp0 .activeCritText2,#mermaid-svg-sz5RHwEBnuvkDMp0 .activeCritText3{fill:#000 !important}#mermaid-svg-sz5RHwEBnuvkDMp0 .titleText{text-anchor:middle;font-size:18px;fill:#000;font-family:'trebuchet ms', verdana, arial;font-family:var(--mermaid-font-family)}#mermaid-svg-sz5RHwEBnuvkDMp0 g.classGroup text{fill:#9370db;stroke:none;font-family:'trebuchet ms', verdana, arial;font-family:var(--mermaid-font-family);font-size:10px}#mermaid-svg-sz5RHwEBnuvkDMp0 g.classGroup text .title{font-weight:bolder}#mermaid-svg-sz5RHwEBnuvkDMp0 g.clickable{cursor:pointer}#mermaid-svg-sz5RHwEBnuvkDMp0 g.classGroup rect{fill:#ECECFF;stroke:#9370db}#mermaid-svg-sz5RHwEBnuvkDMp0 g.classGroup line{stroke:#9370db;stroke-width:1}#mermaid-svg-sz5RHwEBnuvkDMp0 .classLabel .box{stroke:none;stroke-width:0;fill:#ECECFF;opacity:0.5}#mermaid-svg-sz5RHwEBnuvkDMp0 .classLabel .label{fill:#9370db;font-size:10px}#mermaid-svg-sz5RHwEBnuvkDMp0 .relation{stroke:#9370db;stroke-width:1;fill:none}#mermaid-svg-sz5RHwEBnuvkDMp0 .dashed-line{stroke-dasharray:3}#mermaid-svg-sz5RHwEBnuvkDMp0 #compositionStart{fill:#9370db;stroke:#9370db;stroke-width:1}#mermaid-svg-sz5RHwEBnuvkDMp0 #compositionEnd{fill:#9370db;stroke:#9370db;stroke-width:1}#mermaid-svg-sz5RHwEBnuvkDMp0 #aggregationStart{fill:#ECECFF;stroke:#9370db;stroke-width:1}#mermaid-svg-sz5RHwEBnuvkDMp0 #aggregationEnd{fill:#ECECFF;stroke:#9370db;stroke-width:1}#mermaid-svg-sz5RHwEBnuvkDMp0 #dependencyStart{fill:#9370db;stroke:#9370db;stroke-width:1}#mermaid-svg-sz5RHwEBnuvkDMp0 #dependencyEnd{fill:#9370db;stroke:#9370db;stroke-width:1}#mermaid-svg-sz5RHwEBnuvkDMp0 #extensionStart{fill:#9370db;stroke:#9370db;stroke-width:1}#mermaid-svg-sz5RHwEBnuvkDMp0 #extensionEnd{fill:#9370db;stroke:#9370db;stroke-width:1}#mermaid-svg-sz5RHwEBnuvkDMp0 .commit-id,#mermaid-svg-sz5RHwEBnuvkDMp0 .commit-msg,#mermaid-svg-sz5RHwEBnuvkDMp0 .branch-label{fill:lightgrey;color:lightgrey;font-family:'trebuchet ms', verdana, arial;font-family:var(--mermaid-font-family)}#mermaid-svg-sz5RHwEBnuvkDMp0 .pieTitleText{text-anchor:middle;font-size:25px;fill:#000;font-family:'trebuchet ms', verdana, arial;font-family:var(--mermaid-font-family)}#mermaid-svg-sz5RHwEBnuvkDMp0 .slice{font-family:'trebuchet ms', verdana, arial;font-family:var(--mermaid-font-family)}#mermaid-svg-sz5RHwEBnuvkDMp0 g.stateGroup text{fill:#9370db;stroke:none;font-size:10px;font-family:'trebuchet ms', verdana, arial;font-family:var(--mermaid-font-family)}#mermaid-svg-sz5RHwEBnuvkDMp0 g.stateGroup text{fill:#9370db;fill:#333;stroke:none;font-size:10px}#mermaid-svg-sz5RHwEBnuvkDMp0 g.statediagram-cluster .cluster-label text{fill:#333}#mermaid-svg-sz5RHwEBnuvkDMp0 g.stateGroup .state-title{font-weight:bolder;fill:#000}#mermaid-svg-sz5RHwEBnuvkDMp0 g.stateGroup rect{fill:#ECECFF;stroke:#9370db}#mermaid-svg-sz5RHwEBnuvkDMp0 g.stateGroup line{stroke:#9370db;stroke-width:1}#mermaid-svg-sz5RHwEBnuvkDMp0 .transition{stroke:#9370db;stroke-width:1;fill:none}#mermaid-svg-sz5RHwEBnuvkDMp0 .stateGroup .composit{fill:white;border-bottom:1px}#mermaid-svg-sz5RHwEBnuvkDMp0 .stateGroup .alt-composit{fill:#e0e0e0;border-bottom:1px}#mermaid-svg-sz5RHwEBnuvkDMp0 .state-note{stroke:#aa3;fill:#fff5ad}#mermaid-svg-sz5RHwEBnuvkDMp0 .state-note text{fill:black;stroke:none;font-size:10px}#mermaid-svg-sz5RHwEBnuvkDMp0 .stateLabel .box{stroke:none;stroke-width:0;fill:#ECECFF;opacity:0.7}#mermaid-svg-sz5RHwEBnuvkDMp0 .edgeLabel text{fill:#333}#mermaid-svg-sz5RHwEBnuvkDMp0 .stateLabel text{fill:#000;font-size:10px;font-weight:bold;font-family:'trebuchet ms', verdana, arial;font-family:var(--mermaid-font-family)}#mermaid-svg-sz5RHwEBnuvkDMp0 .node circle.state-start{fill:black;stroke:black}#mermaid-svg-sz5RHwEBnuvkDMp0 .node circle.state-end{fill:black;stroke:white;stroke-width:1.5}#mermaid-svg-sz5RHwEBnuvkDMp0 #statediagram-barbEnd{fill:#9370db}#mermaid-svg-sz5RHwEBnuvkDMp0 .statediagram-cluster rect{fill:#ECECFF;stroke:#9370db;stroke-width:1px}#mermaid-svg-sz5RHwEBnuvkDMp0 .statediagram-cluster rect.outer{rx:5px;ry:5px}#mermaid-svg-sz5RHwEBnuvkDMp0 .statediagram-state .divider{stroke:#9370db}#mermaid-svg-sz5RHwEBnuvkDMp0 .statediagram-state .title-state{rx:5px;ry:5px}#mermaid-svg-sz5RHwEBnuvkDMp0 .statediagram-cluster.statediagram-cluster .inner{fill:white}#mermaid-svg-sz5RHwEBnuvkDMp0 .statediagram-cluster.statediagram-cluster-alt .inner{fill:#e0e0e0}#mermaid-svg-sz5RHwEBnuvkDMp0 .statediagram-cluster .inner{rx:0;ry:0}#mermaid-svg-sz5RHwEBnuvkDMp0 .statediagram-state rect.basic{rx:5px;ry:5px}#mermaid-svg-sz5RHwEBnuvkDMp0 .statediagram-state rect.divider{stroke-dasharray:10,10;fill:#efefef}#mermaid-svg-sz5RHwEBnuvkDMp0 .note-edge{stroke-dasharray:5}#mermaid-svg-sz5RHwEBnuvkDMp0 .statediagram-note rect{fill:#fff5ad;stroke:#aa3;stroke-width:1px;rx:0;ry:0}:root{--mermaid-font-family: '"trebuchet ms", verdana, arial';--mermaid-font-family: "Comic Sans MS", "Comic Sans", cursive}#mermaid-svg-sz5RHwEBnuvkDMp0 .error-icon{fill:#522}#mermaid-svg-sz5RHwEBnuvkDMp0 .error-text{fill:#522;stroke:#522}#mermaid-svg-sz5RHwEBnuvkDMp0 .edge-thickness-normal{stroke-width:2px}#mermaid-svg-sz5RHwEBnuvkDMp0 .edge-thickness-thick{stroke-width:3.5px}#mermaid-svg-sz5RHwEBnuvkDMp0 .edge-pattern-solid{stroke-dasharray:0}#mermaid-svg-sz5RHwEBnuvkDMp0 .edge-pattern-dashed{stroke-dasharray:3}#mermaid-svg-sz5RHwEBnuvkDMp0 .edge-pattern-dotted{stroke-dasharray:2}#mermaid-svg-sz5RHwEBnuvkDMp0 .marker{fill:#333}#mermaid-svg-sz5RHwEBnuvkDMp0 .marker.cross{stroke:#333} :root { --mermaid-font-family: "trebuchet ms", verdana, arial;} #mermaid-svg-sz5RHwEBnuvkDMp0 { color: rgba(0, 0, 0, 0.75); font: ; } hadoop1 Namenode SecondaryNamenode ResourceManager node-0001 DataNode NodeManager node-0002 DataNode NodeManager node-0003 DataNode NodeManager

重新初始化集群

警告：该方法会丢失所有数据

1、停止集群 /usr/local/hadoop/sbin/stop-all.sh 2、删除所有节点的 /var/hadoop/* 3、在 hadoop1 上重新格式化 /usr/local/hadoop/bin/hdfs namenode -format 4、启动集群 /usr/local/hadoop/sbin/start-all.sh

[root@hadoop1 ~]# /usr/local/hadoop/sbin/stop-all.sh [root@hadoop1 ~]# for i in hadoop1 node-{0001..0003};do ssh ${i} 'rm -rf /var/hadoop/*' done [root@hadoop1 ~]# /usr/local/hadoop/bin/hdfs namenode -format [root@hadoop1 ~]# /usr/local/hadoop/sbin/start-all.sh

增加新的节点

购买云主机

主机IP地址配置newnode192.168.1.54最低配置2核2G

新节点安装

在 hadoop1 上执行

[root@hadoop1 ~]# ssh-copy-id -i /root/.ssh/id_rsa.pub 192.168.1.54 [root@hadoop1 ~]# vim /etc/hosts 192.168.1.50 hadoop1 192.168.1.51 node-0001 192.168.1.52 node-0002 192.168.1.53 node-0003 192.168.1.54 newnode [root@hadoop1 ~]# for i in node-{0001..0003} newnode;do rsync -av /etc/hosts ${i}:/etc/ done [root@hadoop1 ~]# rsync -aXSH /usr/local/hadoop newnode:/usr/local/

在 newnode 节点执行

[root@newnode ~]# yum install -y java-1.8.0-openjdk-devel [root@newnode ~]# /usr/local/hadoop/sbin/hadoop-daemon.sh start datanode [root@newnode ~]# /usr/local/hadoop/bin/hdfs dfsadmin -setBalancerBandwidth 500000000 [root@newnode ~]# /usr/local/hadoop/sbin/start-balancer.sh [root@newnode ~]# /usr/local/hadoop/sbin/yarn-daemon.sh start nodemanager [root@newnode ~]# jps 1186 DataNode 1431 NodeManager 1535 Jps

验证集群(hadoop1上执行)

[root@hadoop1 ~]# /usr/local/hadoop/bin/hdfs dfsadmin -report ... ... ------------------------------------------------- Live datanodes (4): [root@hadoop1 ~]# /usr/local/hadoop/bin/yarn node -list ... ... Total Nodes:4

删除节点

配置数据迁移 hdfs-site.xml（hadoop1上做，不需要同步）

[root@hadoop1 ~]# vim /usr/local/hadoop/etc/hadoop/hdfs-site.xml <property> <name>dfs.hosts.exclude</name> <value>/usr/local/hadoop/etc/hadoop/exclude</value> </property>

配置排除主机列表，并迁移数据(hadoop1上执行)

# 在删除配置文件中添加 newnode [root@hadoop1 ~]# echo newnode >/usr/local/hadoop/etc/hadoop/exclude # 迁移数据 [root@hadoop1 ~]# /usr/local/hadoop/bin/hdfs dfsadmin -refreshNodes # 查看状态，仅当节点状态为 Decommissioned 时候才可以下线 [root@hadoop1 ~]# /usr/local/hadoop/bin/hdfs dfsadmin -report

下线节点（newnode执行）

[root@newnode ~]# /usr/local/hadoop/sbin/hadoop-daemon.sh stop datanode [root@newnode ~]# /usr/local/hadoop/sbin/yarn-daemon.sh stop nodemanager

NFS网关

NFS网关架构图

#mermaid-svg-Bt9jtRyppNE1Kuro .label{font-family:'trebuchet ms', verdana, arial;font-family:var(--mermaid-font-family);fill:#333;color:#333}#mermaid-svg-Bt9jtRyppNE1Kuro .label text{fill:#333}#mermaid-svg-Bt9jtRyppNE1Kuro .node rect,#mermaid-svg-Bt9jtRyppNE1Kuro .node circle,#mermaid-svg-Bt9jtRyppNE1Kuro .node ellipse,#mermaid-svg-Bt9jtRyppNE1Kuro .node polygon,#mermaid-svg-Bt9jtRyppNE1Kuro .node path{fill:#ECECFF;stroke:#9370db;stroke-width:1px}#mermaid-svg-Bt9jtRyppNE1Kuro .node .label{text-align:center;fill:#333}#mermaid-svg-Bt9jtRyppNE1Kuro .node.clickable{cursor:pointer}#mermaid-svg-Bt9jtRyppNE1Kuro .arrowheadPath{fill:#333}#mermaid-svg-Bt9jtRyppNE1Kuro .edgePath .path{stroke:#333;stroke-width:1.5px}#mermaid-svg-Bt9jtRyppNE1Kuro .flowchart-link{stroke:#333;fill:none}#mermaid-svg-Bt9jtRyppNE1Kuro .edgeLabel{background-color:#e8e8e8;text-align:center}#mermaid-svg-Bt9jtRyppNE1Kuro .edgeLabel rect{opacity:0.9}#mermaid-svg-Bt9jtRyppNE1Kuro .edgeLabel span{color:#333}#mermaid-svg-Bt9jtRyppNE1Kuro .cluster rect{fill:#ffffde;stroke:#aa3;stroke-width:1px}#mermaid-svg-Bt9jtRyppNE1Kuro .cluster text{fill:#333}#mermaid-svg-Bt9jtRyppNE1Kuro div.mermaidTooltip{position:absolute;text-align:center;max-width:200px;padding:2px;font-family:'trebuchet ms', verdana, arial;font-family:var(--mermaid-font-family);font-size:12px;background:#ffffde;border:1px solid #aa3;border-radius:2px;pointer-events:none;z-index:100}#mermaid-svg-Bt9jtRyppNE1Kuro .actor{stroke:#ccf;fill:#ECECFF}#mermaid-svg-Bt9jtRyppNE1Kuro text.actor>tspan{fill:#000;stroke:none}#mermaid-svg-Bt9jtRyppNE1Kuro .actor-line{stroke:grey}#mermaid-svg-Bt9jtRyppNE1Kuro .messageLine0{stroke-width:1.5;stroke-dasharray:none;stroke:#333}#mermaid-svg-Bt9jtRyppNE1Kuro .messageLine1{stroke-width:1.5;stroke-dasharray:2, 2;stroke:#333}#mermaid-svg-Bt9jtRyppNE1Kuro #arrowhead path{fill:#333;stroke:#333}#mermaid-svg-Bt9jtRyppNE1Kuro .sequenceNumber{fill:#fff}#mermaid-svg-Bt9jtRyppNE1Kuro #sequencenumber{fill:#333}#mermaid-svg-Bt9jtRyppNE1Kuro #crosshead path{fill:#333;stroke:#333}#mermaid-svg-Bt9jtRyppNE1Kuro .messageText{fill:#333;stroke:#333}#mermaid-svg-Bt9jtRyppNE1Kuro .labelBox{stroke:#ccf;fill:#ECECFF}#mermaid-svg-Bt9jtRyppNE1Kuro .labelText,#mermaid-svg-Bt9jtRyppNE1Kuro .labelText>tspan{fill:#000;stroke:none}#mermaid-svg-Bt9jtRyppNE1Kuro .loopText,#mermaid-svg-Bt9jtRyppNE1Kuro .loopText>tspan{fill:#000;stroke:none}#mermaid-svg-Bt9jtRyppNE1Kuro .loopLine{stroke-width:2px;stroke-dasharray:2, 2;stroke:#ccf;fill:#ccf}#mermaid-svg-Bt9jtRyppNE1Kuro .note{stroke:#aa3;fill:#fff5ad}#mermaid-svg-Bt9jtRyppNE1Kuro .noteText,#mermaid-svg-Bt9jtRyppNE1Kuro .noteText>tspan{fill:#000;stroke:none}#mermaid-svg-Bt9jtRyppNE1Kuro .activation0{fill:#f4f4f4;stroke:#666}#mermaid-svg-Bt9jtRyppNE1Kuro .activation1{fill:#f4f4f4;stroke:#666}#mermaid-svg-Bt9jtRyppNE1Kuro .activation2{fill:#f4f4f4;stroke:#666}#mermaid-svg-Bt9jtRyppNE1Kuro .mermaid-main-font{font-family:"trebuchet ms", verdana, arial;font-family:var(--mermaid-font-family)}#mermaid-svg-Bt9jtRyppNE1Kuro .section{stroke:none;opacity:0.2}#mermaid-svg-Bt9jtRyppNE1Kuro .section0{fill:rgba(102,102,255,0.49)}#mermaid-svg-Bt9jtRyppNE1Kuro .section2{fill:#fff400}#mermaid-svg-Bt9jtRyppNE1Kuro .section1,#mermaid-svg-Bt9jtRyppNE1Kuro .section3{fill:#fff;opacity:0.2}#mermaid-svg-Bt9jtRyppNE1Kuro .sectionTitle0{fill:#333}#mermaid-svg-Bt9jtRyppNE1Kuro .sectionTitle1{fill:#333}#mermaid-svg-Bt9jtRyppNE1Kuro .sectionTitle2{fill:#333}#mermaid-svg-Bt9jtRyppNE1Kuro .sectionTitle3{fill:#333}#mermaid-svg-Bt9jtRyppNE1Kuro .sectionTitle{text-anchor:start;font-size:11px;text-height:14px;font-family:'trebuchet ms', verdana, arial;font-family:var(--mermaid-font-family)}#mermaid-svg-Bt9jtRyppNE1Kuro .grid .tick{stroke:#d3d3d3;opacity:0.8;shape-rendering:crispEdges}#mermaid-svg-Bt9jtRyppNE1Kuro .grid .tick text{font-family:'trebuchet ms', verdana, arial;font-family:var(--mermaid-font-family)}#mermaid-svg-Bt9jtRyppNE1Kuro .grid path{stroke-width:0}#mermaid-svg-Bt9jtRyppNE1Kuro .today{fill:none;stroke:red;stroke-width:2px}#mermaid-svg-Bt9jtRyppNE1Kuro .task{stroke-width:2}#mermaid-svg-Bt9jtRyppNE1Kuro .taskText{text-anchor:middle;font-family:'trebuchet ms', verdana, arial;font-family:var(--mermaid-font-family)}#mermaid-svg-Bt9jtRyppNE1Kuro .taskText:not([font-size]){font-size:11px}#mermaid-svg-Bt9jtRyppNE1Kuro .taskTextOutsideRight{fill:#000;text-anchor:start;font-size:11px;font-family:'trebuchet ms', verdana, arial;font-family:var(--mermaid-font-family)}#mermaid-svg-Bt9jtRyppNE1Kuro .taskTextOutsideLeft{fill:#000;text-anchor:end;font-size:11px}#mermaid-svg-Bt9jtRyppNE1Kuro .task.clickable{cursor:pointer}#mermaid-svg-Bt9jtRyppNE1Kuro .taskText.clickable{cursor:pointer;fill:#003163 !important;font-weight:bold}#mermaid-svg-Bt9jtRyppNE1Kuro .taskTextOutsideLeft.clickable{cursor:pointer;fill:#003163 !important;font-weight:bold}#mermaid-svg-Bt9jtRyppNE1Kuro .taskTextOutsideRight.clickable{cursor:pointer;fill:#003163 !important;font-weight:bold}#mermaid-svg-Bt9jtRyppNE1Kuro .taskText0,#mermaid-svg-Bt9jtRyppNE1Kuro .taskText1,#mermaid-svg-Bt9jtRyppNE1Kuro .taskText2,#mermaid-svg-Bt9jtRyppNE1Kuro .taskText3{fill:#fff}#mermaid-svg-Bt9jtRyppNE1Kuro .task0,#mermaid-svg-Bt9jtRyppNE1Kuro .task1,#mermaid-svg-Bt9jtRyppNE1Kuro .task2,#mermaid-svg-Bt9jtRyppNE1Kuro .task3{fill:#8a90dd;stroke:#534fbc}#mermaid-svg-Bt9jtRyppNE1Kuro .taskTextOutside0,#mermaid-svg-Bt9jtRyppNE1Kuro .taskTextOutside2{fill:#000}#mermaid-svg-Bt9jtRyppNE1Kuro .taskTextOutside1,#mermaid-svg-Bt9jtRyppNE1Kuro .taskTextOutside3{fill:#000}#mermaid-svg-Bt9jtRyppNE1Kuro .active0,#mermaid-svg-Bt9jtRyppNE1Kuro .active1,#mermaid-svg-Bt9jtRyppNE1Kuro .active2,#mermaid-svg-Bt9jtRyppNE1Kuro .active3{fill:#bfc7ff;stroke:#534fbc}#mermaid-svg-Bt9jtRyppNE1Kuro .activeText0,#mermaid-svg-Bt9jtRyppNE1Kuro .activeText1,#mermaid-svg-Bt9jtRyppNE1Kuro .activeText2,#mermaid-svg-Bt9jtRyppNE1Kuro .activeText3{fill:#000 !important}#mermaid-svg-Bt9jtRyppNE1Kuro .done0,#mermaid-svg-Bt9jtRyppNE1Kuro .done1,#mermaid-svg-Bt9jtRyppNE1Kuro .done2,#mermaid-svg-Bt9jtRyppNE1Kuro .done3{stroke:grey;fill:#d3d3d3;stroke-width:2}#mermaid-svg-Bt9jtRyppNE1Kuro .doneText0,#mermaid-svg-Bt9jtRyppNE1Kuro .doneText1,#mermaid-svg-Bt9jtRyppNE1Kuro .doneText2,#mermaid-svg-Bt9jtRyppNE1Kuro .doneText3{fill:#000 !important}#mermaid-svg-Bt9jtRyppNE1Kuro .crit0,#mermaid-svg-Bt9jtRyppNE1Kuro .crit1,#mermaid-svg-Bt9jtRyppNE1Kuro .crit2,#mermaid-svg-Bt9jtRyppNE1Kuro .crit3{stroke:#f88;fill:red;stroke-width:2}#mermaid-svg-Bt9jtRyppNE1Kuro .activeCrit0,#mermaid-svg-Bt9jtRyppNE1Kuro .activeCrit1,#mermaid-svg-Bt9jtRyppNE1Kuro .activeCrit2,#mermaid-svg-Bt9jtRyppNE1Kuro .activeCrit3{stroke:#f88;fill:#bfc7ff;stroke-width:2}#mermaid-svg-Bt9jtRyppNE1Kuro .doneCrit0,#mermaid-svg-Bt9jtRyppNE1Kuro .doneCrit1,#mermaid-svg-Bt9jtRyppNE1Kuro .doneCrit2,#mermaid-svg-Bt9jtRyppNE1Kuro .doneCrit3{stroke:#f88;fill:#d3d3d3;stroke-width:2;cursor:pointer;shape-rendering:crispEdges}#mermaid-svg-Bt9jtRyppNE1Kuro .milestone{transform:rotate(45deg) scale(0.8, 0.8)}#mermaid-svg-Bt9jtRyppNE1Kuro .milestoneText{font-style:italic}#mermaid-svg-Bt9jtRyppNE1Kuro .doneCritText0,#mermaid-svg-Bt9jtRyppNE1Kuro .doneCritText1,#mermaid-svg-Bt9jtRyppNE1Kuro .doneCritText2,#mermaid-svg-Bt9jtRyppNE1Kuro .doneCritText3{fill:#000 !important}#mermaid-svg-Bt9jtRyppNE1Kuro .activeCritText0,#mermaid-svg-Bt9jtRyppNE1Kuro .activeCritText1,#mermaid-svg-Bt9jtRyppNE1Kuro .activeCritText2,#mermaid-svg-Bt9jtRyppNE1Kuro .activeCritText3{fill:#000 !important}#mermaid-svg-Bt9jtRyppNE1Kuro .titleText{text-anchor:middle;font-size:18px;fill:#000;font-family:'trebuchet ms', verdana, arial;font-family:var(--mermaid-font-family)}#mermaid-svg-Bt9jtRyppNE1Kuro g.classGroup text{fill:#9370db;stroke:none;font-family:'trebuchet ms', verdana, arial;font-family:var(--mermaid-font-family);font-size:10px}#mermaid-svg-Bt9jtRyppNE1Kuro g.classGroup text .title{font-weight:bolder}#mermaid-svg-Bt9jtRyppNE1Kuro g.clickable{cursor:pointer}#mermaid-svg-Bt9jtRyppNE1Kuro g.classGroup rect{fill:#ECECFF;stroke:#9370db}#mermaid-svg-Bt9jtRyppNE1Kuro g.classGroup line{stroke:#9370db;stroke-width:1}#mermaid-svg-Bt9jtRyppNE1Kuro .classLabel .box{stroke:none;stroke-width:0;fill:#ECECFF;opacity:0.5}#mermaid-svg-Bt9jtRyppNE1Kuro .classLabel .label{fill:#9370db;font-size:10px}#mermaid-svg-Bt9jtRyppNE1Kuro .relation{stroke:#9370db;stroke-width:1;fill:none}#mermaid-svg-Bt9jtRyppNE1Kuro .dashed-line{stroke-dasharray:3}#mermaid-svg-Bt9jtRyppNE1Kuro #compositionStart{fill:#9370db;stroke:#9370db;stroke-width:1}#mermaid-svg-Bt9jtRyppNE1Kuro #compositionEnd{fill:#9370db;stroke:#9370db;stroke-width:1}#mermaid-svg-Bt9jtRyppNE1Kuro #aggregationStart{fill:#ECECFF;stroke:#9370db;stroke-width:1}#mermaid-svg-Bt9jtRyppNE1Kuro #aggregationEnd{fill:#ECECFF;stroke:#9370db;stroke-width:1}#mermaid-svg-Bt9jtRyppNE1Kuro #dependencyStart{fill:#9370db;stroke:#9370db;stroke-width:1}#mermaid-svg-Bt9jtRyppNE1Kuro #dependencyEnd{fill:#9370db;stroke:#9370db;stroke-width:1}#mermaid-svg-Bt9jtRyppNE1Kuro #extensionStart{fill:#9370db;stroke:#9370db;stroke-width:1}#mermaid-svg-Bt9jtRyppNE1Kuro #extensionEnd{fill:#9370db;stroke:#9370db;stroke-width:1}#mermaid-svg-Bt9jtRyppNE1Kuro .commit-id,#mermaid-svg-Bt9jtRyppNE1Kuro .commit-msg,#mermaid-svg-Bt9jtRyppNE1Kuro .branch-label{fill:lightgrey;color:lightgrey;font-family:'trebuchet ms', verdana, arial;font-family:var(--mermaid-font-family)}#mermaid-svg-Bt9jtRyppNE1Kuro .pieTitleText{text-anchor:middle;font-size:25px;fill:#000;font-family:'trebuchet ms', verdana, arial;font-family:var(--mermaid-font-family)}#mermaid-svg-Bt9jtRyppNE1Kuro .slice{font-family:'trebuchet ms', verdana, arial;font-family:var(--mermaid-font-family)}#mermaid-svg-Bt9jtRyppNE1Kuro g.stateGroup text{fill:#9370db;stroke:none;font-size:10px;font-family:'trebuchet ms', verdana, arial;font-family:var(--mermaid-font-family)}#mermaid-svg-Bt9jtRyppNE1Kuro g.stateGroup text{fill:#9370db;fill:#333;stroke:none;font-size:10px}#mermaid-svg-Bt9jtRyppNE1Kuro g.statediagram-cluster .cluster-label text{fill:#333}#mermaid-svg-Bt9jtRyppNE1Kuro g.stateGroup .state-title{font-weight:bolder;fill:#000}#mermaid-svg-Bt9jtRyppNE1Kuro g.stateGroup rect{fill:#ECECFF;stroke:#9370db}#mermaid-svg-Bt9jtRyppNE1Kuro g.stateGroup line{stroke:#9370db;stroke-width:1}#mermaid-svg-Bt9jtRyppNE1Kuro .transition{stroke:#9370db;stroke-width:1;fill:none}#mermaid-svg-Bt9jtRyppNE1Kuro .stateGroup .composit{fill:white;border-bottom:1px}#mermaid-svg-Bt9jtRyppNE1Kuro .stateGroup .alt-composit{fill:#e0e0e0;border-bottom:1px}#mermaid-svg-Bt9jtRyppNE1Kuro .state-note{stroke:#aa3;fill:#fff5ad}#mermaid-svg-Bt9jtRyppNE1Kuro .state-note text{fill:black;stroke:none;font-size:10px}#mermaid-svg-Bt9jtRyppNE1Kuro .stateLabel .box{stroke:none;stroke-width:0;fill:#ECECFF;opacity:0.7}#mermaid-svg-Bt9jtRyppNE1Kuro .edgeLabel text{fill:#333}#mermaid-svg-Bt9jtRyppNE1Kuro .stateLabel text{fill:#000;font-size:10px;font-weight:bold;font-family:'trebuchet ms', verdana, arial;font-family:var(--mermaid-font-family)}#mermaid-svg-Bt9jtRyppNE1Kuro .node circle.state-start{fill:black;stroke:black}#mermaid-svg-Bt9jtRyppNE1Kuro .node circle.state-end{fill:black;stroke:white;stroke-width:1.5}#mermaid-svg-Bt9jtRyppNE1Kuro #statediagram-barbEnd{fill:#9370db}#mermaid-svg-Bt9jtRyppNE1Kuro .statediagram-cluster rect{fill:#ECECFF;stroke:#9370db;stroke-width:1px}#mermaid-svg-Bt9jtRyppNE1Kuro .statediagram-cluster rect.outer{rx:5px;ry:5px}#mermaid-svg-Bt9jtRyppNE1Kuro .statediagram-state .divider{stroke:#9370db}#mermaid-svg-Bt9jtRyppNE1Kuro .statediagram-state .title-state{rx:5px;ry:5px}#mermaid-svg-Bt9jtRyppNE1Kuro .statediagram-cluster.statediagram-cluster .inner{fill:white}#mermaid-svg-Bt9jtRyppNE1Kuro .statediagram-cluster.statediagram-cluster-alt .inner{fill:#e0e0e0}#mermaid-svg-Bt9jtRyppNE1Kuro .statediagram-cluster .inner{rx:0;ry:0}#mermaid-svg-Bt9jtRyppNE1Kuro .statediagram-state rect.basic{rx:5px;ry:5px}#mermaid-svg-Bt9jtRyppNE1Kuro .statediagram-state rect.divider{stroke-dasharray:10,10;fill:#efefef}#mermaid-svg-Bt9jtRyppNE1Kuro .note-edge{stroke-dasharray:5}#mermaid-svg-Bt9jtRyppNE1Kuro .statediagram-note rect{fill:#fff5ad;stroke:#aa3;stroke-width:1px;rx:0;ry:0}:root{--mermaid-font-family: '"trebuchet ms", verdana, arial';--mermaid-font-family: "Comic Sans MS", "Comic Sans", cursive}#mermaid-svg-Bt9jtRyppNE1Kuro .error-icon{fill:#522}#mermaid-svg-Bt9jtRyppNE1Kuro .error-text{fill:#522;stroke:#522}#mermaid-svg-Bt9jtRyppNE1Kuro .edge-thickness-normal{stroke-width:2px}#mermaid-svg-Bt9jtRyppNE1Kuro .edge-thickness-thick{stroke-width:3.5px}#mermaid-svg-Bt9jtRyppNE1Kuro .edge-pattern-solid{stroke-dasharray:0}#mermaid-svg-Bt9jtRyppNE1Kuro .edge-pattern-dashed{stroke-dasharray:3}#mermaid-svg-Bt9jtRyppNE1Kuro .edge-pattern-dotted{stroke-dasharray:2}#mermaid-svg-Bt9jtRyppNE1Kuro .marker{fill:#333}#mermaid-svg-Bt9jtRyppNE1Kuro .marker.cross{stroke:#333} :root { --mermaid-font-family: "trebuchet ms", verdana, arial;} #mermaid-svg-Bt9jtRyppNE1Kuro { color: rgba(0, 0, 0, 0.75); font: ; } NFS网关 HDFS集群 mount HDFS客户端 NFS服务 namenode datanode datanode datanode 客户端

购买云主机

主机IP地址配置nfsgw192.168.1.55最低配置1核1G

HDFS用户授权

hadoop1与nfsgw都要添加用户

[root@hadoop1 ~]# groupadd -g 800 nfsuser [root@hadoop1 ~]# useradd -g 800 -u 800 -r -d /var/hadoop nfsuser #---------------------------------------------------------------------------------------- [root@nfsgw ~]# groupadd -g 800 nfsuser [root@nfsgw ~]# useradd -g 800 -u 800 -r -d /var/hadoop nfsuser

HDFS集群授权

[root@hadoop1 ~]# vim /usr/local/hadoop/etc/hadoop/core-site.xml <configuration> <property> <name>fs.defaultFS</name> <value>hdfs://hadoop1:9000</value> </property> <property> <name>hadoop.tmp.dir</name> <value>/var/hadoop</value> </property> <property> <name>hadoop.proxyuser.nfsuser.groups</name> <value>*</value> </property> <property> <name>hadoop.proxyuser.nfsuser.hosts</name> <value>*</value> </property> </configuration> [root@hadoop1 ~]# /usr/local/hadoop/sbin/stop-all.sh [root@hadoop1 ~]# for i in node-{0001..0003};do rsync -avXSH /usr/local/hadoop/etc ${i}:/usr/local/hadoop/ done [root@hadoop1 ~]# /usr/local/hadoop/sbin/start-dfs.sh [root@hadoop1 ~]# jps 5925 NameNode 6122 SecondaryNameNode 6237 Jps [root@hadoop1 ~]# /usr/local/hadoop/bin/hdfs dfsadmin -report ... ... ------------------------------------------------- Live datanodes (3):

NFS网关服务

[root@nfsgw ~]# yum remove -y rpcbind nfs-utils [root@nfsgw ~]# vim /etc/hosts 192.168.1.50 hadoop1 192.168.1.51 node-0001 192.168.1.52 node-0002 192.168.1.53 node-0003 192.168.1.55 nfsgw [root@nfsgw ~]# yum install -y java-1.8.0-openjdk-devel [root@nfsgw ~]# rsync -aXSH --delete hadoop1:/usr/local/hadoop /usr/local/ [root@nfsgw ~]# vim /usr/local/hadoop/etc/hadoop/hdfs-site.xml <configuration> <property> <name>dfs.namenode.http-address</name> <value>hadoop1:50070</value> </property> <property> <name>dfs.namenode.secondary.http-address</name> <value>hadoop1:50090</value> </property> <property> <name>dfs.replication</name> <value>2</value> </property> <property> <name>dfs.hosts.exclude</name> <value>/usr/local/hadoop/etc/hadoop/exclude</value> </property> <property> <name>nfs.exports.allowed.hosts</name> <value>* rw</value> </property> <property> <name>nfs.dump.dir</name> <value>/var/nfstmp</value> </property> </configuration> [root@nfsgw ~]# mkdir /var/nfstmp [root@nfsgw ~]# chown nfsuser.nfsuser /var/nfstmp [root@nfsgw ~]# rm -rf /usr/local/hadoop/logs/* [root@nfsgw ~]# setfacl -m user:nfsuser:rwx /usr/local/hadoop/logs [root@nfsgw ~]# getfacl /usr/local/hadoop/logs [root@nfsgw ~]# cd /usr/local/hadoop/ [root@nfsgw hadoop]# ./sbin/hadoop-daemon.sh --script ./bin/hdfs start portmap [root@nfsgw hadoop]# jps 1376 Portmap 1416 Jps [root@nfsgw hadoop]# rm -rf /tmp/.hdfs-nfs [root@nfsgw hadoop]# sudo -u nfsuser ./sbin/hadoop-daemon.sh --script ./bin/hdfs start nfs3 [root@nfsgw hadoop]# sudo -u nfsuser jps 1452 Nfs3 1502 Jps

mount验证

[root@newnode ~]# yum install -y nfs-utils [root@newnode ~]# showmount -e 192.168.1.55 Export list for 192.168.1.55: / * [root@newnode ~]# mount -t nfs -o vers=3,proto=tcp,nolock,noacl,noatime,sync 192.168.1.55:/ /mnt/ [root@newnode ~]# df -h Filesystem Size Used Avail Use% Mounted on 192.168.1.55:/ 118G 15G 104G 13% /mnt

最新回复(0)