Hadoop文件系统基本的文件系统命令操作, 通过hadoop fs -help可以获取所有的命令的详细帮助文件。Java抽象类org.apache.hadoop.fs.FileSystem定义了hadoop的一个文件系统接口。该类是一个抽象类，通过以下两种静态工厂方法可以过去FileSystem实例：public static FileSystem.get(Configuration conf) throws IOExceptionpublic static FileSystem.get(URI uri, Configuration conf) throws IOException具体方法实现：1、public boolean mkdirs(Path f) throws IOException一次性新建所有目录（包括父目录）， f是完整的目录路径。2、public FSOutputStream create(Path f) throws IOException创建指定path对象的一个文件，返回一个用于写入数据的输出流create()有多个重载版本，允许我们指定是否强制覆盖已有的文件、文件备份数量、写入文件缓冲区大小、文件块大小以及文件权限。3、public boolean copyFromLocal(Path src, Path dst) throws IOException将本地文件拷贝到文件系统4、public boolean exists(Path f) throws IOException检查文件或目录是否存在5、public boolean delete(Path f, Boolean recursive)永久性删除指定的文件或目录，如果f是一个空目录或者文件，那么recursive的值就会被忽略。只有recursive＝true时，一个非空目录及其内容才会被删除。6、FileStatus类封装了文件系统中文件和目录的元数据，包括文件长度、块大小、备份、修改时间、所有者以及权限信息。
通过"FileStatus.getPath（）"可查看指定HDFS中某个目录下所有文件。
01packagehdfsTest;
02
03importjava.io.IOException;
04
05importorg.apache.hadoop.conf.Configuration;
06importorg.apache.hadoop.fs.FSDataOutputStream;
07importorg.apache.hadoop.fs.FileStatus;
08importorg.apache.hadoop.fs.FileSystem;
09importorg.apache.hadoop.fs.Path;
10
11publicclassOperatingFiles {
12//initialization
13staticConfiguration conf =newConfiguration();
14staticFileSystem hdfs;
15static{
16String path ="/usr/java/hadoop-1.0.3/conf/";
17conf.addResource(newPath(path +"core-site.xml"));
18conf.addResource(newPath(path +"hdfs-site.xml"));
19conf.addResource(newPath(path +"mapred-site.xml"));
20path ="/usr/java/hbase-0.90.3/conf/";
21conf.addResource(newPath(path +"hbase-site.xml"));
22try{
23hdfs = FileSystem.get(conf);
24}catch(IOException e) {
25e.printStackTrace();
26}
27}
2829//create a direction
30publicvoidcreateDir(String dir)throwsIOException {
31Path path =newPath(dir);
32hdfs.mkdirs(path);
33System.out.println("new dir \t"+ conf.get("fs.default.name") + dir);
34}
3536//copy from local file to HDFS file
37publicvoidcopyFile(String localSrc, String hdfsDst)throwsIOException{
38Path src =newPath(localSrc);
39Path dst =newPath(hdfsDst);
40hdfs.copyFromLocalFile(src, dst);
4142//list all the files in the current direction
43FileStatus files[] = hdfs.listStatus(dst);
44System.out.println("Upload to \t"+ conf.get("fs.default.name") + hdfsDst);
45for(FileStatus file : files) {
46System.out.println(file.getPath());
47}
48}
4950//create a new file
51publicvoidcreateFile(String fileName, String fileContent)throwsIOException {
52Path dst =newPath(fileName);
53byte[] bytes = fileContent.getBytes();
54FSDataOutputStream output = hdfs.create(dst);
55output.write(bytes);
56System.out.println("new file \t"+ conf.get("fs.default.name") + fileName);
57}
5859//list all files
60publicvoidlistFiles(String dirName)throwsIOException {
61Path f =newPath(dirName);
62FileStatus[] status = hdfs.listStatus(f);
63System.out.println(dirName +" has all files:");
64for(inti =0; i< status.length; i++) {
65System.out.println(status[i].getPath().toString());
66}
67}
68
69//judge a file existed? and delete it!
70publicvoiddeleteFile(String fileName)throwsIOException {
71Path f =newPath(fileName);
72booleanisExists = hdfs.exists(f);
73if(isExists) {//if exists, delete
74booleanisDel = hdfs.delete(f,true);
75System.out.println(fileName +" delete? \t"+ isDel);
76}else{
77System.out.println(fileName +" exist? \t"+ isExists);
78}
79}
80
81publicstaticvoidmain(String[] args)throwsIOException {
82OperatingFiles ofs =newOperatingFiles();
83System.out.println("\n=======create dir=======");
84String dir ="/test";
85ofs.createDir(dir);
86System.out.println("\n=======copy file=======");
87String src ="/home/ictclas/Configure.xml";
88ofs.copyFile(src, dir);
89System.out.println("\n=======create a file=======");
90String fileContent ="Hello, world! Just a test.";
91ofs.createFile(dir+"/word.txt", fileContent);
92}
93}
Using HDFS in java (0.20.0)Below is a code sample of how to read from and write to HDFS in java.1. Creating a configuration object:To be able to read from or write to HDFS, you need to create a Configuration object and pass configuration parameter to it using hadoop configuration files. // Conf object will read the HDFS configuration parameters from theseXML // files. You may specify the parameters for your own if you want. Configuration conf = new Configuration(); conf.addResource(new Path("/opt/hadoop-0.20.0/conf/core-site.xml")); conf.addResource(new Path("/opt/hadoop-0.20.0/conf/hdfs-site.xml")); If you do not assign the configurations to conf object (using hadoop xml file) your HDFS operation will be performed on the local file system and not on the HDFS.2. Adding file to HDFS:Create a FileSystem object and use a file stream to add a file. FileSystem fileSystem = FileSystem.get(conf); // Check if the file already exists Path path = new Path("/path/to/file.ext"); if (fileSystem.exists(path)) { System.out.println("File " + dest + " already exists"); return; } // Create a new file and write data to it. FSDataOutputStream out = fileSystem.create(path); InputStream in = new BufferedInputStream(new FileInputStream( new File(source))); byte[] b = new byte[1024]; int numBytes = 0; while ((numBytes = in.read(b)) > 0) { out.write(b, 0, numBytes); } // Close all the file descripters in.close(); out.close(); fileSystem.close();3. Reading file from HDFS:Create a file stream object to a file in HDFS and read it. FileSystem fileSystem = FileSystem.get(conf); Path path = new Path("/path/to/file.ext"); if (!fileSystem.exists(path)) { System.out.println("File does not exists"); return; } FSDataInputStream in = fileSystem.open(path); String filename = file.substring(file.lastIndexOf(‘/’) + 1, file.length()); OutputStream out = new BufferedOutputStream(new FileOutputStream( new File(filename))); byte[] b = new byte[1024]; int numBytes = 0; while ((numBytes = in.read(b)) > 0) { out.write(b, 0, numBytes); } in.close(); out.close(); fileSystem.close();3. Deleting file from HDFS:Create a file stream object to a file in HDFS and delete it. FileSystem fileSystem = FileSystem.get(conf); Path path = new Path("/path/to/file.ext"); if (!fileSystem.exists(path)) { System.out.println("File does not exists"); return; } // Delete file fileSystem.delete(new Path(file), true); fileSystem.close();3. Create dir in HDFS:Create a file stream object to a file in HDFS and read it. FileSystem fileSystem = FileSystem.get(conf); Path path = new Path(dir); if (fileSystem.exists(path)) { System.out.println("Dir " + dir + " already not exists"); return; } // Create directories fileSystem.mkdirs(path); fileSystem.close();Code:
001importjava.io.BufferedInputStream;
002importjava.io.BufferedOutputStream;
003importjava.io.File;
004importjava.io.FileInputStream;
005importjava.io.FileOutputStream;
006importjava.io.IOException;
007importjava.io.InputStream;
008importjava.io.OutputStream;
009
010importorg.apache.hadoop.conf.Configuration;
011importorg.apache.hadoop.fs.FSDataInputStream;
012importorg.apache.hadoop.fs.FSDataOutputStream;
013importorg.apache.hadoop.fs.FileSystem;
014importorg.apache.hadoop.fs.Path;
015
016publicclassHDFSClient {
017publicHDFSClient() {
018
019}
020
021publicvoidaddFile(String source, String dest)throwsIOException {
022Configuration conf =newConfiguration();
023
024// Conf object will read the HDFS configuration parameters from these
025// XML files.
026conf.addResource(newPath("/opt/hadoop-0.20.0/conf/core-site.xml"));
027conf.addResource(newPath("/opt/hadoop-0.20.0/conf/hdfs-site.xml"));
028
029FileSystem fileSystem = FileSystem.get(conf);
030
031// Get the filename out of the file path
032String filename = source.substring(source.lastIndexOf('/') +1,
033source.length());
034
035// Create the destination path including the filename.
036if(dest.charAt(dest.length() -1) !='/') {
037dest = dest +"/"+ filename;
038}else{
039dest = dest + filename;
040}
041
042// System.out.println("Adding file to " + destination);
043
044// Check if the file already exists
045Path path =newPath(dest);
046if(fileSystem.exists(path)) {
047System.out.println("File "+ dest +" already exists");
048return;
049}
050
051// Create a new file and write data to it.
052FSDataOutputStream out = fileSystem.create(path);
053InputStream in =newBufferedInputStream(newFileInputStream(
054newFile(source)));
055
056byte[] b =newbyte[1024];
057intnumBytes =0;
058while((numBytes = in.read(b)) >0) {
059out.write(b,0, numBytes);
060}
061
062// Close all the file descripters
063in.close();
064out.close();
065fileSystem.close();
066}
067
068publicvoidreadFile(String file)throwsIOException {
069Configuration conf =newConfiguration();
070conf.addResource(newPath("/opt/hadoop-0.20.0/conf/core-site.xml"));
071
072FileSystem fileSystem = FileSystem.get(conf);
073
074Path path =newPath(file);
075if(!fileSystem.exists(path)) {
076System.out.println("File "+ file +" does not exists");
077return;
078}
079
080FSDataInputStream in = fileSystem.open(path);
081
082String filename = file.substring(file.lastIndexOf('/') +1,
083file.length());
084
085OutputStream out =newBufferedOutputStream(newFileOutputStream(
086newFile(filename)));
087
088byte[] b =newbyte[1024];
089intnumBytes =0;
090while((numBytes = in.read(b)) >0) {
091out.write(b,0, numBytes);
092}
093
094in.close();
095out.close();
096fileSystem.close();
097}
098
099publicvoiddeleteFile(String file)throwsIOException {
100Configuration conf =newConfiguration();
101conf.addResource(newPath("/opt/hadoop-0.20.0/conf/core-site.xml"));
102
103FileSystem fileSystem = FileSystem.get(conf);
104
105Path path =newPath(file);
106if(!fileSystem.exists(path)) {
107System.out.println("File "+ file +" does not exists");
108return;
109}
110
111fileSystem.delete(newPath(file),true);
112
113fileSystem.close();
114}
115
116publicvoidmkdir(String dir)throwsIOException {
117Configuration conf =newConfiguration();
118conf.addResource(newPath("/opt/hadoop-0.20.0/conf/core-site.xml"));
119
120FileSystem fileSystem = FileSystem.get(conf);
121
122Path path =newPath(dir);
123if(fileSystem.exists(path)) {
124System.out.println("Dir "+ dir +" already not exists");
125return;
126}
127
128fileSystem.mkdirs(path);
129
130fileSystem.close();
131}
132
133publicstaticvoidmain(String[] args)throwsIOException {
134
135if(args.length <1) {
136System.out.println("Usage: hdfsclient add/read/delete/mkdir"+
137" [<local_path> <hdfs_path>]");
138System.exit(1);
139}
140
141HDFSClient client =newHDFSClient();
142if(args[0].equals("add")) {
143if(args.length <3) {
144System.out.println("Usage: hdfsclient add <local_path> "+
145"<hdfs_path>");
146System.exit(1);
147}
148
149client.addFile(args[1], args[2]);
150}elseif(args[0].equals("read")) {
151if(args.length <2) {
152System.out.println("Usage: hdfsclient read <hdfs_path>");
153System.exit(1);
154}
155
156client.readFile(args[1]);
157}elseif(args[0].equals("delete")) {
158if(args.length <2) {
159System.out.println("Usage: hdfsclient delete <hdfs_path>");
160System.exit(1);
161}
162
163client.deleteFile(args[1]);
164}elseif(args[0].equals("mkdir")) {
165if(args.length <2) {
166System.out.println("Usage: hdfsclient mkdir <hdfs_path>");
167System.exit(1);
168}
169
170client.mkdir(args[1]);
171}else{
172System.out.println("Usage: hdfsclient add/read/delete/mkdir"+
173" [<local_path> <hdfs_path>]");
174System.exit(1);
175}
176
177System.out.println("Done!");
178}
179}
from：http://smallwildpig.iteye.com/blog/1705039 Java对HDFS的操作
http://blog.rajeevsharma.in/2009/06/using-hdfs-in-java-0200.html Using HDFS in java (0.20.0)
一遍一遍的……你突然明白自己还活着，
