HDFS的文件操作

格式化HDFS

命令：[email protected]:Hadoop$bin/hadoop namenode -format

启动HDFS

命令：[email protected]:hadoop$bin/start-dfs.sh

列出HDFS上的文件

命令：[email protected]:hadoop$bin/hadoop dfs -ls

使用hadoop API

public List<String[]>GetFileBolckHost(Configuration conf, String FileName) {

try {

List<String[]> list = new ArrayList<String[]>();

FileSystem hdfs = FileSystem.get(conf);

Path path = new Path(FileName);

FileStatus fileStatus = hdfs.getFileStatus(path);

BlockLocation[] blkLocations = hdfs.getFileBlockLocations(

fileStatus, 0,fileStatus.getLen());

int blkCount = blkLocations.length;

for (int i = 0; i < blkCount; i++) {

String[] hosts =blkLocations.getHosts();

list.add(hosts);

}

return list;

} catch (IOException e) {

e.printStackTrace();

}

return null;

}

在HDFS上创建目录

命令：[email protected]:hadoop$bin/hadoop dfs -mkdir /文件名

使用hadoop API

// 在HDFS新建文件

public FSDataOutputStream CreateFile(Configuration conf, StringFileName) {

try {

FileSystem hdfs = FileSystem.get(conf);

Path path = new Path(FileName);

FSDataOutputStream outputStream = hdfs.create(path);

return outputStream;

} catch (IOException e) {

e.printStackTrace();

}

return null;

}

上传一个文件到HDFS

命令：[email protected]:Hadoop$ bin/hadoopdfs -put 文件名/user/yourUserName/

使用hadoop API

// 上传文件到HDFS

public voidPutFile(Configuration conf, String srcFile, String dstFile) {

try {

FileSystem hdfs =FileSystem.get(conf);

Path srcPath = newPath(srcFile);

Path dstPath = newPath(dstFile);

hdfs.copyFromLocalFile(srcPath, dstPath);

} catch (IOExceptione) {

e.printStackTrace();

}

}

从 HDFS 中导出数据

命令：[email protected]:hadoop$ bin/hadoopdfs -cat foo

使用hadoop API

// 从HDFS读取文件

public voidReadFile(Configuration conf, String FileName) {

try {

FileSystem hdfs =FileSystem.get(conf);

FSDataInputStreamdis = hdfs.open(new Path(FileName));

IOUtils.copyBytes(dis, System.out, 4096, false);

dis.close();

} catch (IOExceptione) {

e.printStackTrace();

}

}

HDFS 的关闭

命令：[email protected]:hadoop$bin/stop-dfs.sh

HDFS全局状态信息

命令：bin/Hadoop dfsadmin -report

我们可以得到一份全局状态报告。这份报告包含了HDFS集群的基本信息，当然也有每台机器的一些情况。

以上讲的都是本地操作HDFS，都是基于在Ubuntu下并配置有hadoop环境下对HDFS的操作，作为客户端也可以在window系统下远程的对 HDFS进行操作，其实原理基本上差不多，只需要集群中namenode对外开放的IP和端口，就可以访问到HDFS

/**

* 对HDFS操作

* @author yujing

*

*/

public class Write {

public static voidmain(String[] args) {

try {

uploadTohdfs();

readHdfs();

getDirectoryFromHdfs();

} catch(FileNotFoundException e) {

e.printStackTrace();

} catch (IOExceptione) {

e.printStackTrace();

}

}

public static voiduploadTohdfs() throws FileNotFoundException, IOException {

String localSrc ="D://qq.txt";

String dst ="hdfs://192.168.1.11:9000/usr/yujing/test.txt";

InputStream in = newBufferedInputStream(new FileInputStream(localSrc));

Configuration conf =new Configuration();

FileSystem fs =FileSystem.get(URI.create(dst), conf);

OutputStream out =fs.create(new Path(dst), new Progressable() {

public voidprogress() {

System.out.println(".");

}

});

System.out.println("上传文件成功");

IOUtils.copyBytes(in,out, 4096, true);

}

/** 从HDFS上读取文件 */

private static voidreadHdfs() throws FileNotFoundException, IOException {

String dst ="hdfs://192.168.1.11:9000/usr/yujing/test.txt";

Configuration conf =new Configuration();

FileSystem fs =FileSystem.get(URI.create(dst), conf);

FSDataInputStreamhdfsInStream = fs.open(new Path(dst));

OutputStream out = newFileOutputStream("d:/qq-hdfs.txt");

byte[] ioBuffer = newbyte[1024];

int readLen =hdfsInStream.read(ioBuffer);

while (-1 != readLen){

out.write(ioBuffer, 0, readLen);

readLen =hdfsInStream.read(ioBuffer);

}

System.out.println("读文件成功");

out.close();

hdfsInStream.close();

fs.close();

}

/**

* 以append方式将内容添加到HDFS上文件的末尾;注意：文件更新，需要在hdfs-site.xml中添<property><name>dfs.

*append.support</name><value>true</value></property>

*/

private static voidappendToHdfs() throws FileNotFoundException,

IOException {

String dst ="hdfs://192.168.1.11:9000/usr/yujing/test.txt";

Configuration conf =new Configuration();

FileSystem fs =FileSystem.get(URI.create(dst), conf);

FSDataOutputStream out= fs.append(new Path(dst));

int readLen ="zhangzk add by hdfs java api".getBytes().length;

while (-1 != readLen){

out.write("zhangzk add by hdfs java api".getBytes(), 0,readLen);

}

out.close();

fs.close();

}

/** 从HDFS上删除文件 */

private static voiddeleteFromHdfs() throws FileNotFoundException,

IOException {

String dst ="hdfs://192.168.1.11:9000/usr/yujing";

Configuration conf =new Configuration();

FileSystem fs =FileSystem.get(URI.create(dst), conf);

fs.deleteOnExit(newPath(dst));

fs.close();

}

/** 遍历HDFS上的文件和目录 */

private static voidgetDirectoryFromHdfs() throws FileNotFoundException,

IOException {

String dst ="hdfs://192.168.1.11:9000/usr/yujing";

Configuration conf =new Configuration();

FileSystem fs =FileSystem.get(URI.create(dst), conf);

FileStatus fileList[]= fs.listStatus(new Path(dst));

int size =fileList.length;

for (int i = 0; i <size; i++) {

System.out.println("文件名name:" + fileList.getPath().getName()

+ "文件大小/t/tsize:" +fileList.getLen());

}

fs.close();

}

}

我们可以通过http://主机IP：50030就可以查看集群的所有信息，也可以查看到自己上传到HDFS上的文件。

更多精彩内容请关注：http://bbs.superwu.cn

关注超人学院微信二维码：