Java 封裝 HDFS API 操作
程式碼下載地址:點選下載
一:環境介紹
hadoop:2.6
Ubuntu:15.10
eclipse:3.8.1
二:操作包括
判斷某個資料夾是否存在 isExist(folder);
建立資料夾 mkdir(folder);
建立資料夾 mkdir(folder);
刪除資料夾 rmr(folder);
列出所有資料夾 ls(folder);
遞迴列出所有資料夾 lsr(folder);
上傳檔案 put(local, folder);
下載檔案 get(folder,local1);
刪除檔案 rm(folder);
顯示檔案 cat(folder);
列出所有資料夾 ls(folder);
遞迴列出所有資料夾 lsr(folder);
上傳檔案 put(local, folder);
下載檔案 get(folder,local1);
刪除檔案 rm(folder);
顯示檔案 cat(folder);
三:程式碼演示
package user_thing_tuijian;
import java.io.IOException;
import java.net.URI;
import java.net.URISyntaxException;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IOUtils;
public class hdfsGYT {
private static final String HDFS = "hdfs://127.0.0.1:9000/";
public hdfsGYT(String hdfs, Configuration conf ){
this.hdfsPath = hdfs;
this.conf = conf;
}
public hdfsGYT() {
// TODO Auto-generated constructor stub
}
private String hdfsPath;
private Configuration conf = new Configuration() ;
public static void main(String[] args) throws IOException, URISyntaxException{
hdfsGYT hdfsgyt = new hdfsGYT();
String folder = HDFS + "mr/groom_system/small2.csv";
String local = "/home/thinkgamer/Java/hadoop_shizhan/src/user_thing_tuijian/small2.csv";
String local1 = "/home/thinkgamer/Java/hadoop_shizhan/src/user_thing_tuijian";
//判斷某個資料夾是否存在
//hdfsgyt.isExist(folder);
//建立資料夾
//hdfsgyt.mkdir(folder);
//刪除資料夾
//hdfsgyt.rmr(folder);
//列出所有資料夾
//hdfsgyt.ls(folder);
//遞迴列出所有資料夾
//hdfsgyt.lsr(folder);
//上傳檔案
//hdfsgyt.put(local, folder);
//下載檔案
//hdfsgyt.get(folder,local1);
//刪除檔案
//hdfsgyt.rm(folder);
//顯示檔案
//hdfsgyt.cat(folder);
}
//顯示檔案
private void cat(String folder) throws IOException, URISyntaxException {
// 與hdfs建立聯絡
FileSystem fs = FileSystem.get(new URI(HDFS),new Configuration());
Path path = new Path(folder);
FSDataInputStream fsdis = null;
System.out.println("cat: " + folder);
try {
fsdis =fs.open(path);
IOUtils.copyBytes(fsdis, System.out, 4096, false);
} finally {
IOUtils.closeStream(fsdis);
fs.close();
}
}
//刪除檔案
private void rm(String folder) throws IOException, URISyntaxException {
//與hdfs建立聯絡
FileSystem fs = FileSystem.get(new URI(HDFS),new Configuration());
Path path = new Path(folder);
if(fs.deleteOnExit(path)){
fs.delete(path);
System.out.println("delete:" + folder);
}else{
System.out.println("The fiel is not exist!");
}
fs.close();
}
//下載檔案
private void get(String remote, String local) throws IllegalArgumentException, IOException, URISyntaxException {
// 建立聯絡
FileSystem fs = FileSystem.get(new URI(HDFS), new Configuration());
fs.copyToLocalFile(new Path(remote), new Path(local));
System.out.println("Get From : " + remote + " To :" + local);
fs.close();
}
//上傳檔案
private void put(String local, String remote) throws IOException, URISyntaxException {
// 建立聯絡
FileSystem fs = FileSystem.get(new URI(HDFS), new Configuration());
fs.copyFromLocalFile(new Path(local), new Path(remote));
System.out.println("Put :" + local + " To : " + remote);
fs.close();
}
//遞迴列出所有資料夾
private void lsr(String folder) throws IOException, URISyntaxException {
//與hdfs建立聯絡
FileSystem fs = FileSystem.get(new URI(HDFS),new Configuration());
Path path = new Path(folder);
//得到該目錄下的所有檔案
FileStatus[] fileList = fs.listStatus(path);
for (FileStatus f : fileList) {
System.out.printf("name: %s | folder: %s | size: %d\n", f.getPath(), f.isDir() , f.getLen());
try{
FileStatus[] fileListR = fs.listStatus(f.getPath());
for(FileStatus fr:fileListR){
System.out.printf("name: %s | folder: %s | size: %d\n", fr.getPath(), fr.isDir() , fr.getLen());
}
}finally{
continue;
}
}
fs.close();
}
//列出所有資料夾
private void ls(String folder) throws IOException, URISyntaxException {
//與hdfs建立聯絡
FileSystem fs = FileSystem.get(new URI(HDFS),new Configuration());
Path path = new Path(folder);
//得到該目錄下的所有檔案
FileStatus[] fileList = fs.listStatus(path);
for (FileStatus f : fileList) {
System.out.printf("name: %s | folder: %s | size: %d\n", f.getPath(), f.isDir() , f.getLen());
}
fs.close();
}
//刪除資料夾
private void rmr(String folder) throws IOException, URISyntaxException {
//與hdfs建立聯絡
FileSystem fs = FileSystem.get(new URI(HDFS),new Configuration());
Path path = new Path(folder);
fs.delete(path);
System.out.println("delete:" + folder);
fs.close();
}
//建立資料夾
public void mkdir(String folder) throws IOException, URISyntaxException {
//與hdfs建立聯絡
FileSystem fs = FileSystem.get(new URI(HDFS),new Configuration());
Path path = new Path(folder);
if (!fs.exists(path)) {
fs.mkdirs(path);
System.out.println("Create: " + folder);
}else{
System.out.println("it is have exist:" + folder);
}
fs.close();
}
//判斷某個資料夾是否存在
private void isExist(String folder) throws IOException, URISyntaxException {
//與hdfs建立聯絡
FileSystem fs = FileSystem.get(new URI(HDFS),new Configuration());
Path path = new Path(folder);
if(fs.exists(path)){
System.out.println("it is have exist:" + folder);
}else{
System.out.println("it is not exist:" + folder);
}
fs.close();
}
}
相關文章
- HDFS 05 - HDFS 常用的 Java API 操作JavaAPI
- Hadoop(十)HDFS API操作HadoopAPI
- seleniumAPI常用操作大全和 API再封裝API封裝
- 如何用JAVA程式碼操作HDFSJava
- axios封裝apiiOS封裝API
- 網路封裝APi封裝API
- java封裝Java封裝
- java 封裝Java封裝
- Java操作hdfs出現的問題Java
- Java API 讀取HDFS的單檔案JavaAPI
- Java API操作ESJavaAPI
- 使用java操作ranger,hdfs ranger授權操作,hive ranger授權操作JavaRangerHive
- Java(三)封裝Java封裝
- HDFS常用操作
- HDFS Shell操作
- 封裝操作cookie的方法封裝Cookie
- 4、hdfs api使用API
- Fetch API 簡單封裝API封裝
- 必應每日桌布API封裝API封裝
- mac系統上hdfs java api的簡單使用MacJavaAPI
- 使用 Java API 操作 elasticsearchJavaAPIElasticsearch
- 小程式API進行promise封裝APIPromise封裝
- 高德地圖 API 介面封裝 SDK地圖API封裝
- Vue — 請求模組、api模組封裝VueAPI封裝
- c# Lambda操作類封裝C#封裝
- 作業系統封裝操作作業系統封裝
- javascript操作cookie程式碼封裝JavaScriptCookie封裝
- 如何封裝資料庫操作封裝資料庫
- Java學習-封裝Java封裝
- hadoop: hdfs API示例HadoopAPI
- HBase篇--HBase操作Api和Java操作Hbase相關ApiAPIJava
- Java之POI操作,封裝ExcelUtil實現Excel匯入匯出Java封裝Excel
- java內省api的操作JavaAPI
- Java API 操作Docker淺談JavaAPIDocker
- ElasticSearch 2.3.3 java API操作二ElasticsearchJavaAPI
- 用JAVA的API操作HBASEJavaAPI
- java操作redis。jedis使用apiJavaRedisAPI
- [譯]使用 Proxy 更好的封裝 Storage API封裝API