hadoop-2.2.0 Federation配置

HDFS中的Federation机制是为了扩展名字空间服务而产生的，确切的来说Federation是为了扩容来实现的，理论上HDFS没有上限，但是有物理机的限制以及JVM的限制。名字节点是联邦机制的，意味着各个namespace之间是相互独立的，不需要彼此协调操作。

hdfs-site.xml:

<?xml version="1.0" encoding="UTF-8"?>
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
<!--
  Licensed under the Apache License, Version 2.0 (the "License");
  you may not use this file except in compliance with the License.
  You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

  Unless required by applicable law or agreed to in writing, software
  distributed under the License is distributed on an "AS IS" BASIS,
  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  See the License for the specific language governing permissions and
  limitations under the License. See accompanying LICENSE file.
-->

<!-- Put site-specific property overrides in this file. -->

<configuration>

    <property>
        <name>dfs.nameservices</name>
        <value>ns1,ns2</value>
        <description>This configuration is for HDFS Federation</description>
    </property>

    <property>
        <name>dfs.namenode.rpc-address.ns1</name>
        <value>qzhong:8020</value>
    </property>

    <property>
        <name>dfs.namenode.rpc-address.ns2</name>
        <value>node100:8020</value>
    </property>

    <property>
        <name>dfs.namenode.http-address.ns1</name>
        <value>qzhong:50070</value>
    </property>

    <property>
        <name>dfs.namenode.http-address.ns2</name>
        <value>node100:50070</value>
    </property>

    <property>
        <name>dfs.namenode.secondaryhttp-address.ns1</name>
        <value>node27:50070</value>
    </property>

    <property>
        <name>dfs.namenode.secondaryhttp-address.ns2</name>
        <value>node101:50070</value>
    </property>

    <property>
        <name>dfs.namenode.name.dir.ns1</name>
        <value>/home/qzhong/hadoop2ns1</value>
    </property>

    <property>
        <name>dfs.namenode.edits.dir.ns1</name>
        <value>${dfs.namenode.name.dir.ns1}</value>
    </property>

    <property>
        <name>dfs.namenode.name.dir.ns2</name>
        <value>/home/qzhong/hadoop2ns2</value>
    </property>

    <property>
        <name>dfs.namenode.edits.dir.ns2</name>
        <value>${dfs.namenode.name.dir.ns2}</value>
    </property>

    <property>
        <name>dfs.hosts</name>
        <value>/home/qzhong/hadoop-2.2.0/etc/hadoop/slaves</value>
    </property>

    <property>
        <name>dfs.namenode.checkpoint.dir.ns1</name>
        <value>/home/qzhong/hadoopfederation</value>
    </property>

    <property>
        <name>dfs.namenode.checkpoint.edits.dir.ns1</name>
        <value>${dfs.namenode.checkpoint.dir.ns1}</value>
    </property>

    <property>
        <name>dfs.namenode.checkpoint.dir.ns2</name>
        <value>/home/qzhong/hadoopfederation</value>
    </property>

    <property>
        <name>dfs.namenode.checkpoint.edits.dir.ns2</name>
        <value>${dfs.namenode.checkpoint.dir.ns2}</value>
    </property>

    <property>
        <name>dfs.namenode.handler.count</name>
        <value>100</value>
    </property>

    <property>
        <name>dfs.blocksize</name>
        <value>268435456</value>
    </property>

    <property>
        <name>dfs.datanode.data.dir</name>
        <value>/home/qzhong/hadoopjournaldata</value>
    </property>

    <property>
        <name>dfs.replication</name>
        <value>3</value>
    </property>

</configuration>

core-site.xml:

<?xml version="1.0" encoding="UTF-8"?>
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
<!--
  Licensed under the Apache License, Version 2.0 (the "License");
  you may not use this file except in compliance with the License.
  You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

  Unless required by applicable law or agreed to in writing, software
  distributed under the License is distributed on an "AS IS" BASIS,
  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  See the License for the specific language governing permissions and
  limitations under the License. See accompanying LICENSE file.
-->

<!-- Put site-specific property overrides in this file. -->

<configuration>
    <property>
        <name>fs.default.name</name>
        <value>hdfs://qzhong:8020</value>
    </property>

    <property>
        <name>io.file.buffer.size</name>
        <value>131072</value>
    </property>
</configuration>

同时也可以刷新建立多个Namespace，这样HDFS既实现了Namespace扩展，也实现了dataNode扩展