HDFS的javaApi

1.安装jdk(1.8)
2.创建maven项目,配置pom.xml(创建对应的依赖)

<dependencies>
        <dependency>
            <groupId>org.apache.hadoop</groupId>
            <artifactId>hadoop-common</artifactId>
            <version>3.2.1</version>
        </dependency>
        <dependency>
            <groupId>org.apache.hadoop</groupId>
            <artifactId>hadoop-client</artifactId>
            <version>3.2.1</version>
        </dependency>
        <dependency>
            <groupId>org.apache.hadoop</groupId>
            <artifactId>hadoop-hdfs</artifactId>
            <version>3.2.1</version>
        </dependency>
        <dependency>
            <groupId>junit</groupId>
            <artifactId>junit</artifactId>
            <version>4.13</version>
        </dependency>
    </dependencies>

3.在项目的src/main/resources目录下，新建一个文件，命名为“log4j.properties”

# 控制台输出配置
log4j.appender.Console=org.apache.log4j.ConsoleAppender
log4j.appender.Console.layout=org.apache.log4j.PatternLayout
log4j.appender.Console.layout.ConversionPattern=%d [%t] %p [%c] - %m%n
# 指定日志的输出级别与输出端
log4j.rootLogger=info,Console

4.编写代码实现
4.1配置全局变量

    FileSystem fs=null;
    Configuration conf=null;
    URI uri=null;
    Logger log;

4.2建立连接

  @Before
    public void testInit() throws Exception{
        uri=new URI("hdfs://node-1:8020");
        conf=new Configuration();
        String user="xdd";
        log=LogManager.getLogger(hadooptest.class);
        fs=FileSystem.get(uri,conf,user);
        log.error("初始化成功!");
    }

4.3 运行后释放资源

@After
    public void testClose() throws Exception{
        if(fs!=null){
            log.error("释放资源结束!");
            fs.close();
        }
    }

4.3新建文件

 //新建文件
    @Test
    public void hdmkdir() throws Exception{
        boolean flag=fs.mkdirs(new Path("test1"));
        log.error("创建成功:"+flag);
    }

4.4删除文件

//删除文件
    @Test
    public void hddel() throws Exception{
        boolean flag=fs.delete(new Path("test1"),true);
        log.error("成功删除:"+flag);
    }

4.5上传文件

//上传资源
    @Test
    public void hdput() throws Exception{
        fs.copyFromLocalFile(new Path("D:/software/coffe.jpg"),new Path("/zxm1/put.jpg"));
        log.error("上传成功");
    }

4.6下载文件

  //下载资源
    @Test
    public void hddown() throws Exception{
        fs.copyToLocalFile(new Path("/zxm1/put.jpg"),new Path("D:/software/down.jpg"));
        log.error("下载成功!");
    }

4.7遍历判断是否为文件

 @Test
    public void hdindAll() throws Exception{
        FileStatus[] ls=fs.listStatus(new Path("/"));
        for (FileStatus fS: ls){
            if(fS.isFile()){
                System.out.println("文件:"+fS.getPath().getName());
            }else {
                System.out.println("目录:"+fS.getPath().getName());
            }
        }
    }

5.整体代码
5.1 hadooptest.java

package mytest;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.Path;
import org.apache.log4j.LogManager;
import org.apache.log4j.Logger;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
import java.net.URI;

public class hadooptest {
    FileSystem fs=null;
    Configuration conf=null;
    URI uri=null;
    Logger log=null;
    @Before
    public void testInit() throws Exception{
        uri=new URI("hdfs://node-1:8020");
        conf=new Configuration();
        String user="xdd";
        log=LogManager.getLogger(hadooptest.class);
        fs=FileSystem.get(uri,conf,user);
        log.error("初始化成功!");
    }
    @After
    public void testClose() throws Exception{
        if(fs!=null){
            log.error("释放资源结束!");
            fs.close();
        }
    }
    //新建文件
    @Test
    public void hdmkdir() throws Exception{
        boolean flag=fs.mkdirs(new Path("test1"));
        log.error("创建成功:"+flag);
    }

    //删除文件
    @Test
    public void hddel() throws Exception{
        boolean flag=fs.delete(new Path("test1"),true);
        log.error("成功删除:"+flag);
    }

    //上传资源
    @Test
    public void hdput() throws Exception{
        fs.copyFromLocalFile(new Path("D:/software/coffe.jpg"),new Path("/zxm1/put.jpg"));
        log.error("上传成功");
    }

    //下载资源
    @Test
    public void hddown() throws Exception{
        fs.copyToLocalFile(new Path("/zxm1/put.jpg"),new Path("D:/software/down.jpg"));
        log.error("下载成功!");
    }

    //判断文件
    @Test
    public void hdindAll() throws Exception{
        FileStatus[] ls=fs.listStatus(new Path("/"));
        for (FileStatus fS: ls){
            if(fS.isFile()){
                System.out.println("文件:"+fS.getPath().getName());
            }else {
                System.out.println("目录:"+fS.getPath().getName());
            }
        }
    }

}

5.2  pom.xml

<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
    <modelVersion>4.0.0</modelVersion>

    <groupId>org.example</groupId>
    <artifactId>hadoop</artifactId>
    <version>1.0-SNAPSHOT</version>

    <dependencies>
        <dependency>
            <groupId>org.apache.hadoop</groupId>
            <artifactId>hadoop-common</artifactId>
            <version>3.2.1</version>
        </dependency>
        <dependency>
            <groupId>org.apache.hadoop</groupId>
            <artifactId>hadoop-client</artifactId>
            <version>3.2.1</version>
        </dependency>
        <dependency>
            <groupId>org.apache.hadoop</groupId>
            <artifactId>hadoop-hdfs</artifactId>
            <version>3.2.1</version>
        </dependency>
        <dependency>
            <groupId>junit</groupId>
            <artifactId>junit</artifactId>
            <version>4.13</version>
        </dependency>
    </dependencies>

</project>

5.3 log4j.properties

# 控制台输出配置
log4j.appender.Console=org.apache.log4j.ConsoleAppender
log4j.appender.Console.layout=org.apache.log4j.PatternLayout
log4j.appender.Console.layout.ConversionPattern=%d [%t] %p [%c] - %m%n
# 指定日志的输出级别与输出端
log4j.rootLogger=info,Console

标签：HDFS,fs,log,javaApi,hadoop,new,apache,org
来源： https://blog.csdn.net/weixin_46327726/article/details/122085075