Hadoop基础-04-HDFS API编程
目录
HDFS API编程
开发环境搭建
porn.xml
4.0.0
org.example
untitled4
1.0-SNAPSHOT
untitled4
http://www.example.com
UTF-8
1.7
1.7
2.6.0-cdh5.15.1
cloudera
https://repository.cloudera.com/artifactory/cloudera-repos/
org.apache.hadoop
hadoop-client
2.6.0-cdh5.15.1
junit
junit
4.11
test
maven-clean-plugin
3.1.0
maven-resources-plugin
3.0.2
maven-compiler-plugin
3.8.0
maven-surefire-plugin
2.22.1
maven-jar-plugin
3.0.2
maven-install-plugin
2.5.2
maven-deploy-plugin
2.8.2
maven-site-plugin
3.7.1
maven-project-info-reports-plugin
3.0.0
等下载完毕就可以了
最后速度太慢,我就挂梯子了
这个就是安装完毕后的样子
HelloWorld
package org.example;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import java.net.URI;
//使用java API来操作文件系统
public class HDFSapp {
public static void main(String[] args) throws Exception {
Configuration configuration =new Configuration();
FileSystem fileSystem= FileSystem.get(new URI("hdfs://hadoop000:8020"), configuration);
Path path= new Path("/hdfsapi/test");
boolean result= fileSystem.mkdirs(path);
System.out.println(result);
}
}
第一次出现了报错Permission denied
Exception in thread "main" org.apache.hadoop.security.AccessControlException: Permission denied: user=jacksun, access=WRITE, inode="/":hadoop:supergroup:drwxr-xr-x
at org.apache.hadoop.hdfs.server.namenode.DefaultAuthorizationProvider.checkFsPermission(DefaultAuthorizationProvider.java:279)
at org.apache.hadoop.hdfs.server.namenode.DefaultAuthorizationProvider.chec
-rw-r--r-- 1 hadoop supergroup 1366 2020-08-17 21:35 /README.txt
drwxr-xr-x - hadoop supergroup 0 2020-08-17 21:48 /hdfs-test
-rw-r--r-- 1 hadoop supergroup 181367942 2020-08-17 21:59 /jdk-8u91-linux-x64.tar.gz
原因我们不是supergroup
组里面的没有写权限
解决办法加入用户hadoop到里面
package org.example;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import java.net.URI;
//使用java API来操作文件系统
public class HDFSapp {
public static void main(String[] args) throws Exception {
Configuration configuration =new Configuration();
FileSystem fileSystem= FileSystem.get(new URI("hdfs://hadoop000:8020"), configuration, "hadoop");
Path path= new Path("/hdfsapi/test");
boolean result= fileSystem.mkdirs(path);
System.out.println(result);
}
}
true
Process finished with exit code 0
[hadoop@hadoop000 sbin]$ hadoop fs -ls /
Found 4 items
-rw-r--r-- 1 hadoop supergroup 1366 2020-08-17 21:35 /README.txt
drwxr-xr-x - hadoop supergroup 0 2020-08-17 21:48 /hdfs-test
drwxr-xr-x - hadoop supergroup 0 2020-08-19 09:08 /hdfsapi
-rw-r--r-- 1 hadoop supergroup 181367942 2020-08-17 21:59 /jdk-8u91-linux-x64.tar.gz
jUnit封装
package org.example;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
import java.net.URI;
//使用java API来操作文件系统
public class HDFSapp {
public static final String HDFS_PATH="hdfs://hadoop000:8020";
FileSystem fileSystem=null;
Configuration configuration=null;
@Before
public void setUp() throws Exception{
System.out.println("=======setUp=======");
//fileSyetem参数(指定uri,客户端指定的配置参数,客户端的身份即用户名)
configuration=new Configuration();
fileSystem=FileSystem.get(new URI(HDFS_PATH),configuration,"hadoop");
}
@Test
public void mkdir() throws Exception{
fileSystem.mkdirs(new Path("/hdfsapi/test2"));
}
@After
public void tearDown(){
configuration=null;
fileSystem=null;
System.out.println("=======tearDown=======");
}
// public static void main(String[] args) throws Exception {
// Configuration configuration =new Configuration();
// FileSystem fileSystem= FileSystem.get(new URI("hdfs://hadoop000:8020"), configuration, "hadoop");
// Path path= new Path("/hdfsapi/test");
// boolean result= fileSystem.mkdirs(path);
// System.out.println(result);
// }
}
[hadoop@hadoop000 sbin]$ hadoop fs -ls /hdfsapi
Found 2 items
drwxr-xr-x - hadoop supergroup 0 2020-08-19 09:08 /hdfsapi/test
drwxr-xr-x - hadoop supergroup 0 2020-08-19 09:44 /hdfsapi/test2
查看HDFS文件内容
//查看HDFS内容
@Test
public void text() throws Exception{
FSDataInputStream in=fileSystem.open(new Path("/README.txt"));
IOUtils.copyBytes(in,System.out,1024);
}
创建文件
//创建文件
@Test
public void create() throws Exception{
FSDataOutputStream out=fileSystem.create(new Path("/hdfsapi/b.txt"));
out.writeUTF("hello world replication");
out.flush();
out.close();
}
[hadoop@hadoop000 sbin]$ hadoop dfs -ls /hdfsapi
DEPRECATED: Use of this script to execute hdfs command is deprecated.
Instead use the hdfs command for it.
Found 4 items
-rw-r--r-- 1 hadoop supergroup 25 2020-08-21 09:48 /hdfsapi/b.txt
-rw-r--r-- 3 hadoop supergroup 13 2020-08-19 10:07 /hdfsapi/c.txt
drwxr-xr-x - hadoop supergroup 0 2020-08-19 09:08 /hdfsapi/test
drwxr-xr-x - hadoop supergroup 0 2020-08-19 09:44 /hdfsapi/test2
[hadoop@hadoop000 sbin]$
副本系数剖析
[hadoop@hadoop000 sbin]$ hadoop fs -ls /
Found 4 items
-rw-r--r-- 1 hadoop supergroup 1366 2020-08-17 21:35 /README.txt
drwxr-xr-x - hadoop supergroup 0 2020-08-17 21:48 /hdfs-test
drwxr-xr-x - hadoop supergroup 0 2020-08-19 10:07 /hdfsapi
-rw-r--r-- 1 hadoop supergroup 181367942 2020-08-17 21:59 /jdk-8u91-linux-x64.tar.gz
[hadoop@hadoop000 sbin]$ hadoop fs -ls /hdfsapi
Found 3 items
-rw-r--r-- 3 hadoop supergroup 13 2020-08-19 10:07 /hdfsapi/a.txt
drwxr-xr-x - hadoop supergroup 0 2020-08-19 09:08 /hdfsapi/test
drwxr-xr-x - hadoop supergroup 0 2020-08-19 09:44 /hdfsapi/test2
我们发现一个细节
是用ssh在服务器上创建的文件副本系数有1个
但是使用java创建的副本系数创建的副本洗漱有3个
那是因为在idea的配置里面我们设置的是3个
在代码中添加下面的 configuration.set("dfs.replication","1");
可以实现对副本洗漱的控制
@Before
public void setUp() throws Exception{
System.out.println("=======setUp=======");
//fileSyetem参数(指定uri,客户端指定的配置参数,客户端的身份即用户名)
configuration=new Configuration();
configuration.set("dfs.replication","1");
fileSystem=FileSystem.get(new URI(HDFS_PATH),configuration,"hadoop");
}
[hadoop@hadoop000 hadoop]$ hadoop fs -ls /hdfsapi
Found 4 items
-rw-r--r-- 3 hadoop supergroup 13 2020-08-19 10:07 /hdfsapi/a.txt
-rw-r--r-- 1 hadoop supergroup 0 2020-08-19 10:46 /hdfsapi/b.txt
drwxr-xr-x - hadoop supergroup 0 2020-08-19 09:08 /hdfsapi/test
drwxr-xr-x - hadoop supergroup 0 2020-08-19 09:44 /hdfsapi/test2
重命名
//重命名
@Test
public void rename() throws Exception{
Path oldPath=new Path("/hdfsapi/a.txt");
Path newPath=new Path("/hdfsapi/c.txt");
boolean result =fileSystem.rename(oldPath,newPath);
System.out.println(result);
}
[hadoop@hadoop000 sbin]$ hadoop fs -ls /hdfsapi/
Found 4 items
-rw-r--r-- 3 hadoop supergroup 13 2020-08-19 10:07 /hdfsapi/a.txt
-rw-r--r-- 1 hadoop supergroup 25 2020-08-19 10:46 /hdfsapi/b.txt
drwxr-xr-x - hadoop supergroup 0 2020-08-19 09:08 /hdfsapi/test
drwxr-xr-x - hadoop supergroup 0 2020-08-19 09:44 /hdfsapi/test2
[hadoop@hadoop000 sbin]$ hadoop fs -ls /hdfsapi/
Found 4 items
-rw-r--r-- 1 hadoop supergroup 25 2020-08-19 10:46 /hdfsapi/b.txt
-rw-r--r-- 3 hadoop supergroup 13 2020-08-19 10:07 /hdfsapi/c.txt
drwxr-xr-x - hadoop supergroup 0 2020-08-19 09:08 /hdfsapi/test
drwxr-xr-x - hadoop supergroup 0 2020-08-19 09:44 /hdfsapi/test2
上传本地文件到HDFS
//上传本地文件到HDFS系统
@Test
public void copyFromLocalFile() throws Exception{
Path src =new Path("/Users/jacksun/data/local.txt");
Path dst =new Path("/hdfsapi/");
fileSystem.copyFromLocalFile(src,dst);
}
[hadoop@hadoop000 sbin]$ hadoop dfs -ls /hdfsapi
DEPRECATED: Use of this script to execute hdfs command is deprecated.
Instead use the hdfs command for it.
Found 4 items
-rw-r--r-- 1 hadoop supergroup 25 2020-08-21 09:48 /hdfsapi/b.txt
-rw-r--r-- 3 hadoop supergroup 13 2020-08-19 10:07 /hdfsapi/c.txt
drwxr-xr-x - hadoop supergroup 0 2020-08-19 09:08 /hdfsapi/test
drwxr-xr-x - hadoop supergroup 0 2020-08-19 09:44 /hdfsapi/test2
[hadoop@hadoop000 sbin]$ hadoop dfs -ls /hdfsapi
DEPRECATED: Use of this script to execute hdfs command is deprecated.
Instead use the hdfs command for it.
Found 5 items
-rw-r--r-- 1 hadoop supergroup 25 2020-08-21 09:48 /hdfsapi/b.txt
-rw-r--r-- 3 hadoop supergroup 13 2020-08-19 10:07 /hdfsapi/c.txt
-rw-r--r-- 1 hadoop supergroup 18 2020-08-21 09:52 /hdfsapi/local.txt
drwxr-xr-x - hadoop supergroup 0 2020-08-19 09:08 /hdfsapi/test
drwxr-xr-x - hadoop supergroup 0 2020-08-19 09:44 /hdfsapi/test2
带进度的上传大文件
//上传本地文件到HDFS系统(大文件,进度条)
@Test
public void copyFromLocalBigFile() throws Exception{
InputStream in =new BufferedInputStream(new FileInputStream(new File("/Users/jacksun/data/music.ape")));
FSDataOutputStream out =fileSystem.create(new Path("/hdfsapi/music.ape"),
new Progressable() {
@Override
public void progress() {
System.out.print("-");
}
}
);
IOUtils.copyBytes(in,out,1024);
}
[hadoop@hadoop000 sbin]$ hadoop dfs -ls /hdfsapi
DEPRECATED: Use of this script to execute hdfs command is deprecated.
Instead use the hdfs command for it.
Found 6 items
-rw-r--r-- 1 hadoop supergroup 25 2020-08-21 09:48 /hdfsapi/b.txt
-rw-r--r-- 3 hadoop supergroup 13 2020-08-19 10:07 /hdfsapi/c.txt
-rw-r--r-- 1 hadoop supergroup 18 2020-08-21 09:52 /hdfsapi/local.txt
-rw-r--r-- 1 hadoop supergroup 48097077 2020-08-21 09:52 /hdfsapi/music.ape
drwxr-xr-x - hadoop supergroup 0 2020-08-19 09:08 /hdfsapi/test
drwxr-xr-x - hadoop supergroup 0 2020-08-19 09:44 /hdfsapi/test2
下载文件
// 从HDFS系统下载文件到本地
@Test
public void copyToLocalFile() throws Exception{
Path src =new Path("/hdfsapi/hello.txt");
Path dst =new Path("/Users/jacksun/data");
fileSystem.copyFromLocalFile(src,dst);
}
列出文件夹下的所有内容
//列出文件列表
@Test
public void listFiles() throws Exception {
FileStatus[] statuses = fileSystem.listStatus(new Path("/hdfsapi"));
for (FileStatus file : statuses) {
printFileStatus(file);
}
}
private void printFileStatus(FileStatus file) {
String isDir = file.isDirectory() ? "文件夹" : "文件";
String permission = file.getPermission().toString();
short replication = file.getReplication();
long length = file.getLen();
String path = file.getPath().toString();
System.out.println(isDir + "\t" + permission + "\t" +
replication + "\t" + length + "\t" + path);
}
=======setUp=======
log4j:WARN No appenders could be found for logger (org.apache.hadoop.metrics2.lib.MutableMetricsFactory).
log4j:WARN Please initialize the log4j system properly.
log4j:WARN See http://logging.apache.org/log4j/1.2/faq.html#noconfig for more info.
文件 rw-r--r-- 1 25 hdfs://hadoop000:8020/hdfsapi/b.txt
文件 rw-r--r-- 3 13 hdfs://hadoop000:8020/hdfsapi/c.txt
文件 rw-r--r-- 1 18 hdfs://hadoop000:8020/hdfsapi/local.txt
文件 rw-r--r-- 1 48097077 hdfs://hadoop000:8020/hdfsapi/music.ape
文件夹 rwxr-xr-x 0 0 hdfs://hadoop000:8020/hdfsapi/test
文件夹 rwxr-xr-x 0 0 hdfs://hadoop000:8020/hdfsapi/test2
=======tearDown=======
递归列出文件夹下的所有文件
//递归列出所有文件
@Test
public void listFileRecursive() throws Exception {
RemoteIterator files = fileSystem.listFiles(
new Path("/hdfsapi"), true);
while (files.hasNext()) {
LocatedFileStatus file = files.next();
printFileStatus(file);
}
}
=======setUp=======
log4j:WARN No appenders could be found for logger (org.apache.hadoop.metrics2.lib.MutableMetricsFactory).
log4j:WARN Please initialize the log4j system properly.
log4j:WARN See http://logging.apache.org/log4j/1.2/faq.html#noconfig for more info.
文件 rw-r--r-- 1 25 hdfs://hadoop000:8020/hdfsapi/b.txt
文件 rw-r--r-- 3 13 hdfs://hadoop000:8020/hdfsapi/c.txt
文件 rw-r--r-- 1 18 hdfs://hadoop000:8020/hdfsapi/local.txt
文件 rw-r--r-- 1 48097077 hdfs://hadoop000:8020/hdfsapi/music.ape
查看文件块信息
// 查看文件块信息
@Test
public void getFielBlockLocations() throws Exception {
FileStatus fileStatus = fileSystem.getFileStatus(
new Path("/hdfsapi/test/a.txt"));
BlockLocation[] blocks = fileSystem.getFileBlockLocations(fileStatus,
0, fileStatus.getLen());
for (BlockLocation block : blocks) {
for (String name : block.getNames()) {
System.out.println(name + ":" + block.getOffset() + ":" +
block.getLength() + ":" + block.getHosts());
}
}
}
删除文件
//删除文件
@Test
public void delete() throws Exception {
fileSystem.delete(new Path("/hdfsapi/test/idea.dmg"), true);
}
Found 6 items
-rw-r--r-- 1 hadoop supergroup 25 2020-08-21 09:48 /hdfsapi/b.txt
-rw-r--r-- 3 hadoop supergroup 13 2020-08-19 10:07 /hdfsapi/c.txt
-rw-r--r-- 1 hadoop supergroup 18 2020-08-21 09:52 /hdfsapi/local.txt
-rw-r--r-- 1 hadoop supergroup 48097077 2020-08-21 09:52 /hdfsapi/music.ape
drwxr-xr-x - hadoop supergroup 0 2020-08-19 09:08 /hdfsapi/test
drwxr-xr-x - hadoop supergroup 0 2020-08-19 09:44 /hdfsapi/test2
[hadoop@hadoop000 sbin]$ hadoop dfs -ls /hdfsapi
DEPRECATED: Use of this script to execute hdfs command is deprecated.
Instead use the hdfs command for it.
Found 5 items
-rw-r--r-- 1 hadoop supergroup 25 2020-08-21 09:57 /hdfsapi/b.txt
-rw-r--r-- 3 hadoop supergroup 13 2020-08-19 10:07 /hdfsapi/c.txt
-rw-r--r-- 1 hadoop supergroup 48097077 2020-08-21 09:57 /hdfsapi/music.ape
drwxr-xr-x - hadoop supergroup 0 2020-08-19 09:08 /hdfsapi/test
drwxr-xr-x - hadoop supergroup 0 2020-08-19 09:44 /hdfsapi/test2
所有文件
package org.example;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.*;
import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.util.Progressable;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
import java.io.BufferedInputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.InputStream;
import java.net.URI;
//使用java API来操作文件系统
public class HDFSapp {
public static final String HDFS_PATH="hdfs://hadoop000:8020";
FileSystem fileSystem=null;
Configuration configuration=null;
@Before
public void setUp() throws Exception{
System.out.println("=======setUp=======");
//fileSyetem参数(指定uri,客户端指定的配置参数,客户端的身份即用户名)
configuration=new Configuration();
configuration.set("dfs.replication","1");
fileSystem=FileSystem.get(new URI(HDFS_PATH),configuration,"hadoop");
}
//创建文件夹
@Test
public void mkdir() throws Exception{
fileSystem.mkdirs(new Path("/hdfsapi/test2"));
}
//查看HDFS内容
@Test
public void text() throws Exception{
FSDataInputStream in=fileSystem.open(new Path("/README.txt"));
IOUtils.copyBytes(in,System.out,1024);
}
//========================================
//创建文件
@Test
public void create() throws Exception{
FSDataOutputStream out=fileSystem.create(new Path("/hdfsapi/b.txt"));
out.writeUTF("hello world replication");
out.flush();
out.close();
}
//重命名
@Test
public void rename() throws Exception{
Path oldPath=new Path("/hdfsapi/a.txt");
Path newPath=new Path("/hdfsapi/c.txt");
boolean result =fileSystem.rename(oldPath,newPath);
System.out.println(result);
}
//上传本地文件到HDFS系统
@Test
public void copyFromLocalFile() throws Exception{
Path src =new Path("/Users/jacksun/data/local.txt");
Path dst =new Path("/hdfsapi/");
fileSystem.copyFromLocalFile(src,dst);
}
//上传本地文件到HDFS系统(大文件,进度条)
@Test
public void copyFromLocalBigFile() throws Exception{
InputStream in =new BufferedInputStream(new FileInputStream(new File("/Users/jacksun/data/music.ape")));
FSDataOutputStream out =fileSystem.create(new Path("/hdfsapi/music.ape"),
new Progressable() {
@Override
public void progress() {
System.out.print("-");
}
}
);
IOUtils.copyBytes(in,out,1024);
}
// 从HDFS系统下载文件到本地
@Test
public void copyToLocalFile() throws Exception{
Path src =new Path("/hdfsapi/hello.txt");
Path dst =new Path("/Users/jacksun/data");
fileSystem.copyFromLocalFile(src,dst);
}
//列出文件列表
@Test
public void listFiles() throws Exception {
FileStatus[] statuses = fileSystem.listStatus(new Path("/hdfsapi"));
for (FileStatus file : statuses) {
printFileStatus(file);
}
}
private void printFileStatus(FileStatus file) {
String isDir = file.isDirectory() ? "文件夹" : "文件";
String permission = file.getPermission().toString();
short replication = file.getReplication();
long length = file.getLen();
String path = file.getPath().toString();
System.out.println(isDir + "\t" + permission + "\t" +
replication + "\t" + length + "\t" + path);
}
//递归列出所有文件
@Test
public void listFileRecursive() throws Exception {
RemoteIterator files = fileSystem.listFiles(
new Path("/hdfsapi"), true);
while (files.hasNext()) {
LocatedFileStatus file = files.next();
printFileStatus(file);
}
}
// 查看文件块信息
@Test
public void getFielBlockLocations() throws Exception {
FileStatus fileStatus = fileSystem.getFileStatus(
new Path("/hdfsapi/test/a.txt"));
BlockLocation[] blocks = fileSystem.getFileBlockLocations(fileStatus,
0, fileStatus.getLen());
for (BlockLocation block : blocks) {
for (String name : block.getNames()) {
System.out.println(name + ":" + block.getOffset() + ":" +
block.getLength() + ":" + block.getHosts());
}
}
}
//删除文件
@Test
public void delete() throws Exception {
fileSystem.delete(new Path("/hdfsapi/local.txt"), true);
}
@After
public void tearDown(){
configuration=null;
fileSystem=null;
System.out.println("=======tearDown=======");
}
// public static void main(String[] args) throws Exception {
// Configuration configuration =new Configuration();
// FileSystem fileSystem= FileSystem.get(new URI("hdfs://hadoop000:8020"), configuration, "hadoop");
// Path path= new Path("/hdfsapi/test");
// boolean result= fileSystem.mkdirs(path);
// System.out.println(result);
// }
}
原文:https://www.cnblogs.com/hiszm/p/13524969.html