From 95bfec4931260eb7b73c97861c4929623c8321d6 Mon Sep 17 00:00:00 2001 From: limd Date: Sat, 26 Sep 2020 01:38:48 +0800 Subject: hadoop: filesystem cannot create file issues: https://github.com/chrislusf/seaweedfs/issues/1494 --- other/java/hdfs3/src/main/java/seaweed/hdfs/SeaweedFileSystemStore.java | 2 ++ 1 file changed, 2 insertions(+) (limited to 'other/java/hdfs3') diff --git a/other/java/hdfs3/src/main/java/seaweed/hdfs/SeaweedFileSystemStore.java b/other/java/hdfs3/src/main/java/seaweed/hdfs/SeaweedFileSystemStore.java index 53185367a..2d9f3ccb9 100644 --- a/other/java/hdfs3/src/main/java/seaweed/hdfs/SeaweedFileSystemStore.java +++ b/other/java/hdfs3/src/main/java/seaweed/hdfs/SeaweedFileSystemStore.java @@ -202,6 +202,8 @@ public class SeaweedFileSystemStore { .clearGroupName() .addAllGroupName(Arrays.asList(userGroupInformation.getGroupNames())) ); + + filerClient.createEntry(getParentDirectory(path), entry.build()); } return new SeaweedOutputStream(filerGrpcClient, path, entry, writePosition, bufferSize, replication); -- cgit v1.2.3 From ac162fc85769cb1b2a1f8694f9644eae7d0ce6c8 Mon Sep 17 00:00:00 2001 From: limd Date: Sat, 10 Oct 2020 13:07:27 +0800 Subject: hdfs: Hadoop on SeaweedFS: create empty file --- .../src/main/java/seaweed/hdfs/SeaweedFileSystemStore.java | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) (limited to 'other/java/hdfs3') diff --git a/other/java/hdfs3/src/main/java/seaweed/hdfs/SeaweedFileSystemStore.java b/other/java/hdfs3/src/main/java/seaweed/hdfs/SeaweedFileSystemStore.java index 2d9f3ccb9..c76160bd2 100644 --- a/other/java/hdfs3/src/main/java/seaweed/hdfs/SeaweedFileSystemStore.java +++ b/other/java/hdfs3/src/main/java/seaweed/hdfs/SeaweedFileSystemStore.java @@ -8,14 +8,10 @@ import org.apache.hadoop.fs.permission.FsPermission; import org.apache.hadoop.security.UserGroupInformation; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import seaweedfs.client.FilerClient; -import seaweedfs.client.FilerGrpcClient; -import seaweedfs.client.FilerProto; -import seaweedfs.client.SeaweedRead; +import seaweedfs.client.*; import java.io.FileNotFoundException; import java.io.IOException; -import java.io.InputStream; import java.io.OutputStream; import java.util.ArrayList; import java.util.Arrays; @@ -202,8 +198,7 @@ public class SeaweedFileSystemStore { .clearGroupName() .addAllGroupName(Arrays.asList(userGroupInformation.getGroupNames())) ); - - filerClient.createEntry(getParentDirectory(path), entry.build()); + SeaweedWrite.writeMeta(filerGrpcClient, getParentDirectory(path), entry); } return new SeaweedOutputStream(filerGrpcClient, path, entry, writePosition, bufferSize, replication); @@ -280,4 +275,4 @@ public class SeaweedFileSystemStore { } -} +} \ No newline at end of file -- cgit v1.2.3 From c709059b690e2e5c70b262f4c884bb784cd1781a Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Tue, 3 Nov 2020 00:46:00 -0800 Subject: HCFS: add close() to SeaweedFileSystem.java --- .../java/hdfs3/src/main/java/seaweed/hdfs/SeaweedFileSystem.java | 6 ++++++ .../hdfs3/src/main/java/seaweed/hdfs/SeaweedFileSystemStore.java | 8 ++++++++ 2 files changed, 14 insertions(+) (limited to 'other/java/hdfs3') diff --git a/other/java/hdfs3/src/main/java/seaweed/hdfs/SeaweedFileSystem.java b/other/java/hdfs3/src/main/java/seaweed/hdfs/SeaweedFileSystem.java index ca67c3874..69df55cf1 100644 --- a/other/java/hdfs3/src/main/java/seaweed/hdfs/SeaweedFileSystem.java +++ b/other/java/hdfs3/src/main/java/seaweed/hdfs/SeaweedFileSystem.java @@ -65,6 +65,12 @@ public class SeaweedFileSystem extends FileSystem { } + @Override + public void close() throws IOException { + this.seaweedFileSystemStore.close(); + super.close(); + } + @Override public FSDataInputStream open(Path path, int bufferSize) throws IOException { diff --git a/other/java/hdfs3/src/main/java/seaweed/hdfs/SeaweedFileSystemStore.java b/other/java/hdfs3/src/main/java/seaweed/hdfs/SeaweedFileSystemStore.java index 23556a578..cec309b3e 100644 --- a/other/java/hdfs3/src/main/java/seaweed/hdfs/SeaweedFileSystemStore.java +++ b/other/java/hdfs3/src/main/java/seaweed/hdfs/SeaweedFileSystemStore.java @@ -36,6 +36,14 @@ public class SeaweedFileSystemStore { this.conf = conf; } + public void close() { + try { + this.filerGrpcClient.shutdown(); + } catch (InterruptedException e) { + e.printStackTrace(); + } + } + public static String getParentDirectory(Path path) { return path.isRoot() ? "/" : path.getParent().toUri().getPath(); } -- cgit v1.2.3 From 4f676aa7d3ff26de2226f628d8534ca569098a15 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Sun, 8 Nov 2020 22:06:06 -0800 Subject: 2.09 --- other/java/hdfs3/dependency-reduced-pom.xml | 2 +- other/java/hdfs3/pom.xml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) (limited to 'other/java/hdfs3') diff --git a/other/java/hdfs3/dependency-reduced-pom.xml b/other/java/hdfs3/dependency-reduced-pom.xml index c4847a9b9..c09ab0040 100644 --- a/other/java/hdfs3/dependency-reduced-pom.xml +++ b/other/java/hdfs3/dependency-reduced-pom.xml @@ -309,7 +309,7 @@ - 1.5.2 + 1.5.3 3.1.1 diff --git a/other/java/hdfs3/pom.xml b/other/java/hdfs3/pom.xml index 9e668bba8..d579daa76 100644 --- a/other/java/hdfs3/pom.xml +++ b/other/java/hdfs3/pom.xml @@ -5,7 +5,7 @@ 4.0.0 - 1.5.2 + 1.5.3 3.1.1 -- cgit v1.2.3 From 7d249808407588431f00424dbb2ca599cc9441c6 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Fri, 13 Nov 2020 11:37:50 -0800 Subject: HCFS: 1.5.4 --- other/java/hdfs3/dependency-reduced-pom.xml | 2 +- other/java/hdfs3/pom.xml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) (limited to 'other/java/hdfs3') diff --git a/other/java/hdfs3/dependency-reduced-pom.xml b/other/java/hdfs3/dependency-reduced-pom.xml index c09ab0040..262c3ca80 100644 --- a/other/java/hdfs3/dependency-reduced-pom.xml +++ b/other/java/hdfs3/dependency-reduced-pom.xml @@ -309,7 +309,7 @@ - 1.5.3 + 1.5.4 3.1.1 diff --git a/other/java/hdfs3/pom.xml b/other/java/hdfs3/pom.xml index d579daa76..1952305e9 100644 --- a/other/java/hdfs3/pom.xml +++ b/other/java/hdfs3/pom.xml @@ -5,7 +5,7 @@ 4.0.0 - 1.5.3 + 1.5.4 3.1.1 -- cgit v1.2.3 From 4737df597db3664d9d47e7f47ec6204a832e4c78 Mon Sep 17 00:00:00 2001 From: limd Date: Tue, 17 Nov 2020 19:05:57 +0800 Subject: HCFS: 1. add replication parameter 2. fix close sequence --- other/java/hdfs3/src/main/java/seaweed/hdfs/SeaweedFileSystem.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) (limited to 'other/java/hdfs3') diff --git a/other/java/hdfs3/src/main/java/seaweed/hdfs/SeaweedFileSystem.java b/other/java/hdfs3/src/main/java/seaweed/hdfs/SeaweedFileSystem.java index 69df55cf1..fb1f4c53b 100644 --- a/other/java/hdfs3/src/main/java/seaweed/hdfs/SeaweedFileSystem.java +++ b/other/java/hdfs3/src/main/java/seaweed/hdfs/SeaweedFileSystem.java @@ -25,6 +25,7 @@ public class SeaweedFileSystem extends FileSystem { public static final String FS_SEAWEED_FILER_PORT = "fs.seaweed.filer.port"; public static final int FS_SEAWEED_DEFAULT_PORT = 8888; public static final String FS_SEAWEED_BUFFER_SIZE = "fs.seaweed.buffer.size"; + public static final String FS_SEAWEED_REPLICATION = "fs.seaweed.replication"; public static final int FS_SEAWEED_DEFAULT_BUFFER_SIZE = 4 * 1024 * 1024; private static final Logger LOG = LoggerFactory.getLogger(SeaweedFileSystem.class); @@ -67,8 +68,8 @@ public class SeaweedFileSystem extends FileSystem { @Override public void close() throws IOException { - this.seaweedFileSystemStore.close(); super.close(); + this.seaweedFileSystemStore.close(); } @Override @@ -97,7 +98,7 @@ public class SeaweedFileSystem extends FileSystem { path = qualify(path); try { - String replicaPlacement = String.format("%03d", replication - 1); + String replicaPlacement = this.getConf().get(FS_SEAWEED_REPLICATION, String.format("%03d", replication - 1)); int seaweedBufferSize = this.getConf().getInt(FS_SEAWEED_BUFFER_SIZE, FS_SEAWEED_DEFAULT_BUFFER_SIZE); OutputStream outputStream = seaweedFileSystemStore.createFile(path, overwrite, permission, seaweedBufferSize, replicaPlacement); return new FSDataOutputStream(outputStream, statistics); -- cgit v1.2.3 From da04bb3d1bb60d92fdacfb2edd8c8bdba2643038 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Wed, 18 Nov 2020 01:46:38 -0800 Subject: 1.5.6 --- other/java/hdfs3/dependency-reduced-pom.xml | 2 +- other/java/hdfs3/pom.xml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) (limited to 'other/java/hdfs3') diff --git a/other/java/hdfs3/dependency-reduced-pom.xml b/other/java/hdfs3/dependency-reduced-pom.xml index 262c3ca80..20b52e20f 100644 --- a/other/java/hdfs3/dependency-reduced-pom.xml +++ b/other/java/hdfs3/dependency-reduced-pom.xml @@ -309,7 +309,7 @@ - 1.5.4 + 1.5.6 3.1.1 diff --git a/other/java/hdfs3/pom.xml b/other/java/hdfs3/pom.xml index 1952305e9..85d8db859 100644 --- a/other/java/hdfs3/pom.xml +++ b/other/java/hdfs3/pom.xml @@ -5,7 +5,7 @@ 4.0.0 - 1.5.4 + 1.5.6 3.1.1 -- cgit v1.2.3