From 95bfec4931260eb7b73c97861c4929623c8321d6 Mon Sep 17 00:00:00 2001 From: limd Date: Sat, 26 Sep 2020 01:38:48 +0800 Subject: hadoop: filesystem cannot create file issues: https://github.com/chrislusf/seaweedfs/issues/1494 --- other/java/hdfs3/src/main/java/seaweed/hdfs/SeaweedFileSystemStore.java | 2 ++ 1 file changed, 2 insertions(+) (limited to 'other/java') diff --git a/other/java/hdfs3/src/main/java/seaweed/hdfs/SeaweedFileSystemStore.java b/other/java/hdfs3/src/main/java/seaweed/hdfs/SeaweedFileSystemStore.java index 53185367a..2d9f3ccb9 100644 --- a/other/java/hdfs3/src/main/java/seaweed/hdfs/SeaweedFileSystemStore.java +++ b/other/java/hdfs3/src/main/java/seaweed/hdfs/SeaweedFileSystemStore.java @@ -202,6 +202,8 @@ public class SeaweedFileSystemStore { .clearGroupName() .addAllGroupName(Arrays.asList(userGroupInformation.getGroupNames())) ); + + filerClient.createEntry(getParentDirectory(path), entry.build()); } return new SeaweedOutputStream(filerGrpcClient, path, entry, writePosition, bufferSize, replication); -- cgit v1.2.3 From ac162fc85769cb1b2a1f8694f9644eae7d0ce6c8 Mon Sep 17 00:00:00 2001 From: limd Date: Sat, 10 Oct 2020 13:07:27 +0800 Subject: hdfs: Hadoop on SeaweedFS: create empty file --- .../src/main/java/seaweed/hdfs/SeaweedFileSystemStore.java | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) (limited to 'other/java') diff --git a/other/java/hdfs3/src/main/java/seaweed/hdfs/SeaweedFileSystemStore.java b/other/java/hdfs3/src/main/java/seaweed/hdfs/SeaweedFileSystemStore.java index 2d9f3ccb9..c76160bd2 100644 --- a/other/java/hdfs3/src/main/java/seaweed/hdfs/SeaweedFileSystemStore.java +++ b/other/java/hdfs3/src/main/java/seaweed/hdfs/SeaweedFileSystemStore.java @@ -8,14 +8,10 @@ import org.apache.hadoop.fs.permission.FsPermission; import org.apache.hadoop.security.UserGroupInformation; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import seaweedfs.client.FilerClient; -import seaweedfs.client.FilerGrpcClient; -import seaweedfs.client.FilerProto; -import seaweedfs.client.SeaweedRead; +import seaweedfs.client.*; import java.io.FileNotFoundException; import java.io.IOException; -import java.io.InputStream; import java.io.OutputStream; import java.util.ArrayList; import java.util.Arrays; @@ -202,8 +198,7 @@ public class SeaweedFileSystemStore { .clearGroupName() .addAllGroupName(Arrays.asList(userGroupInformation.getGroupNames())) ); - - filerClient.createEntry(getParentDirectory(path), entry.build()); + SeaweedWrite.writeMeta(filerGrpcClient, getParentDirectory(path), entry); } return new SeaweedOutputStream(filerGrpcClient, path, entry, writePosition, bufferSize, replication); @@ -280,4 +275,4 @@ public class SeaweedFileSystemStore { } -} +} \ No newline at end of file -- cgit v1.2.3 From c709059b690e2e5c70b262f4c884bb784cd1781a Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Tue, 3 Nov 2020 00:46:00 -0800 Subject: HCFS: add close() to SeaweedFileSystem.java --- .../java/hdfs2/src/main/java/seaweed/hdfs/SeaweedFileSystem.java | 6 ++++++ .../hdfs2/src/main/java/seaweed/hdfs/SeaweedFileSystemStore.java | 8 ++++++++ .../java/hdfs3/src/main/java/seaweed/hdfs/SeaweedFileSystem.java | 6 ++++++ .../hdfs3/src/main/java/seaweed/hdfs/SeaweedFileSystemStore.java | 8 ++++++++ 4 files changed, 28 insertions(+) (limited to 'other/java') diff --git a/other/java/hdfs2/src/main/java/seaweed/hdfs/SeaweedFileSystem.java b/other/java/hdfs2/src/main/java/seaweed/hdfs/SeaweedFileSystem.java index ca67c3874..69df55cf1 100644 --- a/other/java/hdfs2/src/main/java/seaweed/hdfs/SeaweedFileSystem.java +++ b/other/java/hdfs2/src/main/java/seaweed/hdfs/SeaweedFileSystem.java @@ -65,6 +65,12 @@ public class SeaweedFileSystem extends FileSystem { } + @Override + public void close() throws IOException { + this.seaweedFileSystemStore.close(); + super.close(); + } + @Override public FSDataInputStream open(Path path, int bufferSize) throws IOException { diff --git a/other/java/hdfs2/src/main/java/seaweed/hdfs/SeaweedFileSystemStore.java b/other/java/hdfs2/src/main/java/seaweed/hdfs/SeaweedFileSystemStore.java index 23556a578..cec309b3e 100644 --- a/other/java/hdfs2/src/main/java/seaweed/hdfs/SeaweedFileSystemStore.java +++ b/other/java/hdfs2/src/main/java/seaweed/hdfs/SeaweedFileSystemStore.java @@ -36,6 +36,14 @@ public class SeaweedFileSystemStore { this.conf = conf; } + public void close() { + try { + this.filerGrpcClient.shutdown(); + } catch (InterruptedException e) { + e.printStackTrace(); + } + } + public static String getParentDirectory(Path path) { return path.isRoot() ? "/" : path.getParent().toUri().getPath(); } diff --git a/other/java/hdfs3/src/main/java/seaweed/hdfs/SeaweedFileSystem.java b/other/java/hdfs3/src/main/java/seaweed/hdfs/SeaweedFileSystem.java index ca67c3874..69df55cf1 100644 --- a/other/java/hdfs3/src/main/java/seaweed/hdfs/SeaweedFileSystem.java +++ b/other/java/hdfs3/src/main/java/seaweed/hdfs/SeaweedFileSystem.java @@ -65,6 +65,12 @@ public class SeaweedFileSystem extends FileSystem { } + @Override + public void close() throws IOException { + this.seaweedFileSystemStore.close(); + super.close(); + } + @Override public FSDataInputStream open(Path path, int bufferSize) throws IOException { diff --git a/other/java/hdfs3/src/main/java/seaweed/hdfs/SeaweedFileSystemStore.java b/other/java/hdfs3/src/main/java/seaweed/hdfs/SeaweedFileSystemStore.java index 23556a578..cec309b3e 100644 --- a/other/java/hdfs3/src/main/java/seaweed/hdfs/SeaweedFileSystemStore.java +++ b/other/java/hdfs3/src/main/java/seaweed/hdfs/SeaweedFileSystemStore.java @@ -36,6 +36,14 @@ public class SeaweedFileSystemStore { this.conf = conf; } + public void close() { + try { + this.filerGrpcClient.shutdown(); + } catch (InterruptedException e) { + e.printStackTrace(); + } + } + public static String getParentDirectory(Path path) { return path.isRoot() ? "/" : path.getParent().toUri().getPath(); } -- cgit v1.2.3 From 4f676aa7d3ff26de2226f628d8534ca569098a15 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Sun, 8 Nov 2020 22:06:06 -0800 Subject: 2.09 --- other/java/client/pom.xml | 2 +- other/java/client/pom.xml.deploy | 2 +- other/java/client/pom_debug.xml | 2 +- other/java/hdfs2/dependency-reduced-pom.xml | 2 +- other/java/hdfs2/pom.xml | 2 +- other/java/hdfs3/dependency-reduced-pom.xml | 2 +- other/java/hdfs3/pom.xml | 2 +- 7 files changed, 7 insertions(+), 7 deletions(-) (limited to 'other/java') diff --git a/other/java/client/pom.xml b/other/java/client/pom.xml index efbf304c4..09f891a8a 100644 --- a/other/java/client/pom.xml +++ b/other/java/client/pom.xml @@ -5,7 +5,7 @@ com.github.chrislusf seaweedfs-client - 1.5.2 + 1.5.3 org.sonatype.oss diff --git a/other/java/client/pom.xml.deploy b/other/java/client/pom.xml.deploy index 9efc21373..28a62d66f 100644 --- a/other/java/client/pom.xml.deploy +++ b/other/java/client/pom.xml.deploy @@ -5,7 +5,7 @@ com.github.chrislusf seaweedfs-client - 1.5.2 + 1.5.3 org.sonatype.oss diff --git a/other/java/client/pom_debug.xml b/other/java/client/pom_debug.xml index 04ff52730..78e4986cf 100644 --- a/other/java/client/pom_debug.xml +++ b/other/java/client/pom_debug.xml @@ -5,7 +5,7 @@ com.github.chrislusf seaweedfs-client - 1.5.2 + 1.5.3 org.sonatype.oss diff --git a/other/java/hdfs2/dependency-reduced-pom.xml b/other/java/hdfs2/dependency-reduced-pom.xml index 229fa673c..4be9ff180 100644 --- a/other/java/hdfs2/dependency-reduced-pom.xml +++ b/other/java/hdfs2/dependency-reduced-pom.xml @@ -301,7 +301,7 @@ - 1.5.2 + 1.5.3 2.9.2 diff --git a/other/java/hdfs2/pom.xml b/other/java/hdfs2/pom.xml index 1b73b2811..478ae40ad 100644 --- a/other/java/hdfs2/pom.xml +++ b/other/java/hdfs2/pom.xml @@ -5,7 +5,7 @@ 4.0.0 - 1.5.2 + 1.5.3 2.9.2 diff --git a/other/java/hdfs3/dependency-reduced-pom.xml b/other/java/hdfs3/dependency-reduced-pom.xml index c4847a9b9..c09ab0040 100644 --- a/other/java/hdfs3/dependency-reduced-pom.xml +++ b/other/java/hdfs3/dependency-reduced-pom.xml @@ -309,7 +309,7 @@ - 1.5.2 + 1.5.3 3.1.1 diff --git a/other/java/hdfs3/pom.xml b/other/java/hdfs3/pom.xml index 9e668bba8..d579daa76 100644 --- a/other/java/hdfs3/pom.xml +++ b/other/java/hdfs3/pom.xml @@ -5,7 +5,7 @@ 4.0.0 - 1.5.2 + 1.5.3 3.1.1 -- cgit v1.2.3 From c32b6f4d9d99d175863e778dcc0da8ef3da27d96 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Wed, 11 Nov 2020 23:46:42 -0800 Subject: prepare for path based configuration --- other/java/client/src/main/proto/filer.proto | 8 ++++++++ 1 file changed, 8 insertions(+) (limited to 'other/java') diff --git a/other/java/client/src/main/proto/filer.proto b/other/java/client/src/main/proto/filer.proto index 11c29e6ec..8829062f4 100644 --- a/other/java/client/src/main/proto/filer.proto +++ b/other/java/client/src/main/proto/filer.proto @@ -348,3 +348,11 @@ message KvPutRequest { message KvPutResponse { string error = 1; } + +// path-based configurations +message FilerPathConfiguration { + string location = 1; + string collection = 2; + string replication = 3; + string ttl = 4; +} -- cgit v1.2.3 From a4dfa00030b7c7e0aeeaee7c00da203dc9cd0ed0 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Thu, 12 Nov 2020 00:39:59 -0800 Subject: adjust filer conf --- other/java/client/src/main/proto/filer.proto | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) (limited to 'other/java') diff --git a/other/java/client/src/main/proto/filer.proto b/other/java/client/src/main/proto/filer.proto index 8829062f4..f75caec4e 100644 --- a/other/java/client/src/main/proto/filer.proto +++ b/other/java/client/src/main/proto/filer.proto @@ -350,9 +350,18 @@ message KvPutResponse { } // path-based configurations -message FilerPathConfiguration { - string location = 1; - string collection = 2; - string replication = 3; - string ttl = 4; +message FilerConf { + int32 version = 1; + message PathConf { + string location_prefix = 1; + string collection = 2; + string replication = 3; + string ttl = 4; + enum DiskType { + HDD = 0; + SSD = 1; + } + DiskType disk_type = 5; + } + repeated PathConf locations = 2; } -- cgit v1.2.3 From 559dfacdac0d3851541468b60b9046fe9d9f2b51 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Thu, 12 Nov 2020 23:24:39 -0800 Subject: adding test example --- other/java/unzip/pom.xml | 32 +++++++++++++++ .../src/main/java/com/example/test/Example.java | 46 ++++++++++++++++++++++ 2 files changed, 78 insertions(+) create mode 100644 other/java/unzip/pom.xml create mode 100644 other/java/unzip/src/main/java/com/example/test/Example.java (limited to 'other/java') diff --git a/other/java/unzip/pom.xml b/other/java/unzip/pom.xml new file mode 100644 index 000000000..1f86bb688 --- /dev/null +++ b/other/java/unzip/pom.xml @@ -0,0 +1,32 @@ + + + 4.0.0 + + org.example + unzip + 1.0-SNAPSHOT + + + com.github.chrislusf + seaweedfs-client + 1.5.3 + compile + + + com.github.chrislusf + seaweedfs-hadoop2-client + 1.5.3 + compile + + + org.apache.hadoop + hadoop-common + 2.9.2 + compile + + + + + \ No newline at end of file diff --git a/other/java/unzip/src/main/java/com/example/test/Example.java b/other/java/unzip/src/main/java/com/example/test/Example.java new file mode 100644 index 000000000..393ccb5ab --- /dev/null +++ b/other/java/unzip/src/main/java/com/example/test/Example.java @@ -0,0 +1,46 @@ +package com.example.test; + +import seaweed.hdfs.SeaweedInputStream; +import seaweedfs.client.FilerClient; +import seaweedfs.client.FilerGrpcClient; + +import java.io.FileInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.util.zip.ZipEntry; +import java.util.zip.ZipInputStream; + +public class Example { + + public static FilerClient filerClient = new FilerClient("localhost", 18888); + public static FilerGrpcClient filerGrpcClient = new FilerGrpcClient("localhost", 18888); + + public static void main(String[] args) throws IOException { + + // 本地模式，速度很快 + parseZip("/Users/chris/tmp/test.zip"); + + // swfs读取，慢 + SeaweedInputStream seaweedInputStream = new SeaweedInputStream( + filerGrpcClient, + new org.apache.hadoop.fs.FileSystem.Statistics(""), + "/", + filerClient.lookupEntry("/", "test.zip") + ); + parseZip(seaweedInputStream); + + } + + public static void parseZip(String filename) throws IOException { + FileInputStream fileInputStream = new FileInputStream(filename); + parseZip(fileInputStream); + } + + public static void parseZip(InputStream is) throws IOException { + ZipInputStream zin = new ZipInputStream(is); + ZipEntry ze; + while ((ze = zin.getNextEntry()) != null) { + System.out.println(ze.getName()); + } + } +} -- cgit v1.2.3 From a2962604ad0bf83b26c692c72ddd40e648fb804d Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Thu, 12 Nov 2020 23:25:11 -0800 Subject: add volume id cache --- .../src/main/java/seaweedfs/client/ChunkCache.java | 1 - .../main/java/seaweedfs/client/SeaweedRead.java | 19 ++++++++---- .../main/java/seaweedfs/client/VolumeIdCache.java | 36 ++++++++++++++++++++++ 3 files changed, 49 insertions(+), 7 deletions(-) create mode 100644 other/java/client/src/main/java/seaweedfs/client/VolumeIdCache.java (limited to 'other/java') diff --git a/other/java/client/src/main/java/seaweedfs/client/ChunkCache.java b/other/java/client/src/main/java/seaweedfs/client/ChunkCache.java index 7afa2dca0..58870d742 100644 --- a/other/java/client/src/main/java/seaweedfs/client/ChunkCache.java +++ b/other/java/client/src/main/java/seaweedfs/client/ChunkCache.java @@ -15,7 +15,6 @@ public class ChunkCache { } this.cache = CacheBuilder.newBuilder() .maximumSize(maxEntries) - .weakValues() .expireAfterAccess(1, TimeUnit.HOURS) .build(); } diff --git a/other/java/client/src/main/java/seaweedfs/client/SeaweedRead.java b/other/java/client/src/main/java/seaweedfs/client/SeaweedRead.java index ab2407dec..a9ddd51db 100644 --- a/other/java/client/src/main/java/seaweedfs/client/SeaweedRead.java +++ b/other/java/client/src/main/java/seaweedfs/client/SeaweedRead.java @@ -19,6 +19,7 @@ public class SeaweedRead { private static final Logger LOG = LoggerFactory.getLogger(SeaweedRead.class); static ChunkCache chunkCache = new ChunkCache(4); + static VolumeIdCache volumeIdCache = new VolumeIdCache(4 * 1024); // returns bytesRead public static long read(FilerGrpcClient filerGrpcClient, List visibleIntervals, @@ -30,13 +31,19 @@ public class SeaweedRead { FilerProto.LookupVolumeRequest.Builder lookupRequest = FilerProto.LookupVolumeRequest.newBuilder(); for (ChunkView chunkView : chunkViews) { String vid = parseVolumeId(chunkView.fileId); - lookupRequest.addVolumeIds(vid); + if (volumeIdCache.getLocations(vid)==null){ + lookupRequest.addVolumeIds(vid); + } } - FilerProto.LookupVolumeResponse lookupResponse = filerGrpcClient - .getBlockingStub().lookupVolume(lookupRequest.build()); - - Map vid2Locations = lookupResponse.getLocationsMapMap(); + if (lookupRequest.getVolumeIdsCount()>0){ + FilerProto.LookupVolumeResponse lookupResponse = filerGrpcClient + .getBlockingStub().lookupVolume(lookupRequest.build()); + Map vid2Locations = lookupResponse.getLocationsMapMap(); + for (Map.Entry entry : vid2Locations.entrySet()) { + volumeIdCache.setLocations(entry.getKey(), entry.getValue()); + } + } //TODO parallel this long readCount = 0; @@ -50,7 +57,7 @@ public class SeaweedRead { startOffset += gap; } - FilerProto.Locations locations = vid2Locations.get(parseVolumeId(chunkView.fileId)); + FilerProto.Locations locations = volumeIdCache.getLocations(parseVolumeId(chunkView.fileId)); if (locations == null || locations.getLocationsCount() == 0) { LOG.error("failed to locate {}", chunkView.fileId); // log here! diff --git a/other/java/client/src/main/java/seaweedfs/client/VolumeIdCache.java b/other/java/client/src/main/java/seaweedfs/client/VolumeIdCache.java new file mode 100644 index 000000000..38daa14ac --- /dev/null +++ b/other/java/client/src/main/java/seaweedfs/client/VolumeIdCache.java @@ -0,0 +1,36 @@ +package seaweedfs.client; + +import com.google.common.cache.Cache; +import com.google.common.cache.CacheBuilder; + +import java.util.concurrent.TimeUnit; + +public class VolumeIdCache { + + private Cache cache = null; + + public VolumeIdCache(int maxEntries) { + if (maxEntries == 0) { + return; + } + this.cache = CacheBuilder.newBuilder() + .maximumSize(maxEntries) + .expireAfterAccess(1, TimeUnit.HOURS) + .build(); + } + + public FilerProto.Locations getLocations(String volumeId) { + if (this.cache == null) { + return null; + } + return this.cache.getIfPresent(volumeId); + } + + public void setLocations(String volumeId, FilerProto.Locations locations) { + if (this.cache == null) { + return; + } + this.cache.put(volumeId, locations); + } + +} -- cgit v1.2.3 From 7d249808407588431f00424dbb2ca599cc9441c6 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Fri, 13 Nov 2020 11:37:50 -0800 Subject: HCFS: 1.5.4 --- other/java/client/pom.xml | 2 +- other/java/client/pom.xml.deploy | 2 +- other/java/client/pom_debug.xml | 2 +- other/java/hdfs2/dependency-reduced-pom.xml | 2 +- other/java/hdfs2/pom.xml | 2 +- other/java/hdfs3/dependency-reduced-pom.xml | 2 +- other/java/hdfs3/pom.xml | 2 +- other/java/unzip/pom.xml | 4 ++-- other/java/unzip/src/main/java/com/example/test/Example.java | 10 ++++++++++ 9 files changed, 19 insertions(+), 9 deletions(-) (limited to 'other/java') diff --git a/other/java/client/pom.xml b/other/java/client/pom.xml index 09f891a8a..5a4bbaead 100644 --- a/other/java/client/pom.xml +++ b/other/java/client/pom.xml @@ -5,7 +5,7 @@ com.github.chrislusf seaweedfs-client - 1.5.3 + 1.5.4 org.sonatype.oss diff --git a/other/java/client/pom.xml.deploy b/other/java/client/pom.xml.deploy index 28a62d66f..e24bcca9f 100644 --- a/other/java/client/pom.xml.deploy +++ b/other/java/client/pom.xml.deploy @@ -5,7 +5,7 @@ com.github.chrislusf seaweedfs-client - 1.5.3 + 1.5.4 org.sonatype.oss diff --git a/other/java/client/pom_debug.xml b/other/java/client/pom_debug.xml index 78e4986cf..724c3c3b9 100644 --- a/other/java/client/pom_debug.xml +++ b/other/java/client/pom_debug.xml @@ -5,7 +5,7 @@ com.github.chrislusf seaweedfs-client - 1.5.3 + 1.5.4 org.sonatype.oss diff --git a/other/java/hdfs2/dependency-reduced-pom.xml b/other/java/hdfs2/dependency-reduced-pom.xml index 4be9ff180..b598d8402 100644 --- a/other/java/hdfs2/dependency-reduced-pom.xml +++ b/other/java/hdfs2/dependency-reduced-pom.xml @@ -301,7 +301,7 @@ - 1.5.3 + 1.5.4 2.9.2 diff --git a/other/java/hdfs2/pom.xml b/other/java/hdfs2/pom.xml index 478ae40ad..de518a0dc 100644 --- a/other/java/hdfs2/pom.xml +++ b/other/java/hdfs2/pom.xml @@ -5,7 +5,7 @@ 4.0.0 - 1.5.3 + 1.5.4 2.9.2 diff --git a/other/java/hdfs3/dependency-reduced-pom.xml b/other/java/hdfs3/dependency-reduced-pom.xml index c09ab0040..262c3ca80 100644 --- a/other/java/hdfs3/dependency-reduced-pom.xml +++ b/other/java/hdfs3/dependency-reduced-pom.xml @@ -309,7 +309,7 @@ - 1.5.3 + 1.5.4 3.1.1 diff --git a/other/java/hdfs3/pom.xml b/other/java/hdfs3/pom.xml index d579daa76..1952305e9 100644 --- a/other/java/hdfs3/pom.xml +++ b/other/java/hdfs3/pom.xml @@ -5,7 +5,7 @@ 4.0.0 - 1.5.3 + 1.5.4 3.1.1 diff --git a/other/java/unzip/pom.xml b/other/java/unzip/pom.xml index 1f86bb688..7cbb56ec3 100644 --- a/other/java/unzip/pom.xml +++ b/other/java/unzip/pom.xml @@ -11,13 +11,13 @@ com.github.chrislusf seaweedfs-client - 1.5.3 + 1.5.4 compile com.github.chrislusf seaweedfs-hadoop2-client - 1.5.3 + 1.5.4 compile diff --git a/other/java/unzip/src/main/java/com/example/test/Example.java b/other/java/unzip/src/main/java/com/example/test/Example.java index 393ccb5ab..3d22329a8 100644 --- a/other/java/unzip/src/main/java/com/example/test/Example.java +++ b/other/java/unzip/src/main/java/com/example/test/Example.java @@ -17,9 +17,14 @@ public class Example { public static void main(String[] args) throws IOException { + long startTime = System.currentTimeMillis(); // 本地模式，速度很快 parseZip("/Users/chris/tmp/test.zip"); + long startTime2 = System.currentTimeMillis(); + + long localProcessTime = startTime2 - startTime; + // swfs读取，慢 SeaweedInputStream seaweedInputStream = new SeaweedInputStream( filerGrpcClient, @@ -29,6 +34,11 @@ public class Example { ); parseZip(seaweedInputStream); + long swProcessTime = System.currentTimeMillis() - startTime2; + + System.out.println("Local time: " + localProcessTime); + System.out.println("SeaweedFS time: " + swProcessTime); + } public static void parseZip(String filename) throws IOException { -- cgit v1.2.3 From 0d5355c6141f4d5ff83933f0b7803e88ae1df90b Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Fri, 13 Nov 2020 11:43:25 -0800 Subject: rename --- other/java/examples/pom.xml | 32 +++++++++++++ .../src/main/java/com/example/test/Example.java | 56 ++++++++++++++++++++++ other/java/unzip/pom.xml | 32 ------------- .../src/main/java/com/example/test/Example.java | 56 ---------------------- 4 files changed, 88 insertions(+), 88 deletions(-) create mode 100644 other/java/examples/pom.xml create mode 100644 other/java/examples/src/main/java/com/example/test/Example.java delete mode 100644 other/java/unzip/pom.xml delete mode 100644 other/java/unzip/src/main/java/com/example/test/Example.java (limited to 'other/java') diff --git a/other/java/examples/pom.xml b/other/java/examples/pom.xml new file mode 100644 index 000000000..7cbb56ec3 --- /dev/null +++ b/other/java/examples/pom.xml @@ -0,0 +1,32 @@ + + + 4.0.0 + + org.example + unzip + 1.0-SNAPSHOT + + + com.github.chrislusf + seaweedfs-client + 1.5.4 + compile + + + com.github.chrislusf + seaweedfs-hadoop2-client + 1.5.4 + compile + + + org.apache.hadoop + hadoop-common + 2.9.2 + compile + + + + + \ No newline at end of file diff --git a/other/java/examples/src/main/java/com/example/test/Example.java b/other/java/examples/src/main/java/com/example/test/Example.java new file mode 100644 index 000000000..3d22329a8 --- /dev/null +++ b/other/java/examples/src/main/java/com/example/test/Example.java @@ -0,0 +1,56 @@ +package com.example.test; + +import seaweed.hdfs.SeaweedInputStream; +import seaweedfs.client.FilerClient; +import seaweedfs.client.FilerGrpcClient; + +import java.io.FileInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.util.zip.ZipEntry; +import java.util.zip.ZipInputStream; + +public class Example { + + public static FilerClient filerClient = new FilerClient("localhost", 18888); + public static FilerGrpcClient filerGrpcClient = new FilerGrpcClient("localhost", 18888); + + public static void main(String[] args) throws IOException { + + long startTime = System.currentTimeMillis(); + // 本地模式，速度很快 + parseZip("/Users/chris/tmp/test.zip"); + + long startTime2 = System.currentTimeMillis(); + + long localProcessTime = startTime2 - startTime; + + // swfs读取，慢 + SeaweedInputStream seaweedInputStream = new SeaweedInputStream( + filerGrpcClient, + new org.apache.hadoop.fs.FileSystem.Statistics(""), + "/", + filerClient.lookupEntry("/", "test.zip") + ); + parseZip(seaweedInputStream); + + long swProcessTime = System.currentTimeMillis() - startTime2; + + System.out.println("Local time: " + localProcessTime); + System.out.println("SeaweedFS time: " + swProcessTime); + + } + + public static void parseZip(String filename) throws IOException { + FileInputStream fileInputStream = new FileInputStream(filename); + parseZip(fileInputStream); + } + + public static void parseZip(InputStream is) throws IOException { + ZipInputStream zin = new ZipInputStream(is); + ZipEntry ze; + while ((ze = zin.getNextEntry()) != null) { + System.out.println(ze.getName()); + } + } +} diff --git a/other/java/unzip/pom.xml b/other/java/unzip/pom.xml deleted file mode 100644 index 7cbb56ec3..000000000 --- a/other/java/unzip/pom.xml +++ /dev/null @@ -1,32 +0,0 @@ - - - 4.0.0 - - org.example - unzip - 1.0-SNAPSHOT - - - com.github.chrislusf - seaweedfs-client - 1.5.4 - compile - - - com.github.chrislusf - seaweedfs-hadoop2-client - 1.5.4 - compile - - - org.apache.hadoop - hadoop-common - 2.9.2 - compile - - - - - \ No newline at end of file diff --git a/other/java/unzip/src/main/java/com/example/test/Example.java b/other/java/unzip/src/main/java/com/example/test/Example.java deleted file mode 100644 index 3d22329a8..000000000 --- a/other/java/unzip/src/main/java/com/example/test/Example.java +++ /dev/null @@ -1,56 +0,0 @@ -package com.example.test; - -import seaweed.hdfs.SeaweedInputStream; -import seaweedfs.client.FilerClient; -import seaweedfs.client.FilerGrpcClient; - -import java.io.FileInputStream; -import java.io.IOException; -import java.io.InputStream; -import java.util.zip.ZipEntry; -import java.util.zip.ZipInputStream; - -public class Example { - - public static FilerClient filerClient = new FilerClient("localhost", 18888); - public static FilerGrpcClient filerGrpcClient = new FilerGrpcClient("localhost", 18888); - - public static void main(String[] args) throws IOException { - - long startTime = System.currentTimeMillis(); - // 本地模式，速度很快 - parseZip("/Users/chris/tmp/test.zip"); - - long startTime2 = System.currentTimeMillis(); - - long localProcessTime = startTime2 - startTime; - - // swfs读取，慢 - SeaweedInputStream seaweedInputStream = new SeaweedInputStream( - filerGrpcClient, - new org.apache.hadoop.fs.FileSystem.Statistics(""), - "/", - filerClient.lookupEntry("/", "test.zip") - ); - parseZip(seaweedInputStream); - - long swProcessTime = System.currentTimeMillis() - startTime2; - - System.out.println("Local time: " + localProcessTime); - System.out.println("SeaweedFS time: " + swProcessTime); - - } - - public static void parseZip(String filename) throws IOException { - FileInputStream fileInputStream = new FileInputStream(filename); - parseZip(fileInputStream); - } - - public static void parseZip(InputStream is) throws IOException { - ZipInputStream zin = new ZipInputStream(is); - ZipEntry ze; - while ((ze = zin.getNextEntry()) != null) { - System.out.println(ze.getName()); - } - } -} -- cgit v1.2.3 From 3e362451d226d9e19b4b652a02926dedc02f6cf9 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Fri, 13 Nov 2020 12:10:55 -0800 Subject: add example of watch files --- .../main/java/seaweedfs/client/FilerClient.java | 15 ++++-- .../src/main/java/com/example/test/Example.java | 56 ---------------------- .../java/com/seaweedfs/examples/UnzipFile.java | 54 +++++++++++++++++++++ .../java/com/seaweedfs/examples/WatchFiles.java | 42 ++++++++++++++++ 4 files changed, 108 insertions(+), 59 deletions(-) delete mode 100644 other/java/examples/src/main/java/com/example/test/Example.java create mode 100644 other/java/examples/src/main/java/com/seaweedfs/examples/UnzipFile.java create mode 100644 other/java/examples/src/main/java/com/seaweedfs/examples/WatchFiles.java (limited to 'other/java') diff --git a/other/java/client/src/main/java/seaweedfs/client/FilerClient.java b/other/java/client/src/main/java/seaweedfs/client/FilerClient.java index 035b2c852..7338d5bee 100644 --- a/other/java/client/src/main/java/seaweedfs/client/FilerClient.java +++ b/other/java/client/src/main/java/seaweedfs/client/FilerClient.java @@ -275,9 +275,9 @@ public class FilerClient { try { FilerProto.CreateEntryResponse createEntryResponse = filerGrpcClient.getBlockingStub().createEntry(FilerProto.CreateEntryRequest.newBuilder() - .setDirectory(parent) - .setEntry(entry) - .build()); + .setDirectory(parent) + .setEntry(entry) + .build()); if (Strings.isNullOrEmpty(createEntryResponse.getError())) { return true; } @@ -333,4 +333,13 @@ public class FilerClient { return true; } + public Iterator watch(String prefix, String clientName, long sinceNs) { + return filerGrpcClient.getBlockingStub().subscribeMetadata(FilerProto.SubscribeMetadataRequest.newBuilder() + .setPathPrefix(prefix) + .setClientName(clientName) + .setSinceNs(sinceNs) + .build() + ); + } + } diff --git a/other/java/examples/src/main/java/com/example/test/Example.java b/other/java/examples/src/main/java/com/example/test/Example.java deleted file mode 100644 index 3d22329a8..000000000 --- a/other/java/examples/src/main/java/com/example/test/Example.java +++ /dev/null @@ -1,56 +0,0 @@ -package com.example.test; - -import seaweed.hdfs.SeaweedInputStream; -import seaweedfs.client.FilerClient; -import seaweedfs.client.FilerGrpcClient; - -import java.io.FileInputStream; -import java.io.IOException; -import java.io.InputStream; -import java.util.zip.ZipEntry; -import java.util.zip.ZipInputStream; - -public class Example { - - public static FilerClient filerClient = new FilerClient("localhost", 18888); - public static FilerGrpcClient filerGrpcClient = new FilerGrpcClient("localhost", 18888); - - public static void main(String[] args) throws IOException { - - long startTime = System.currentTimeMillis(); - // 本地模式，速度很快 - parseZip("/Users/chris/tmp/test.zip"); - - long startTime2 = System.currentTimeMillis(); - - long localProcessTime = startTime2 - startTime; - - // swfs读取，慢 - SeaweedInputStream seaweedInputStream = new SeaweedInputStream( - filerGrpcClient, - new org.apache.hadoop.fs.FileSystem.Statistics(""), - "/", - filerClient.lookupEntry("/", "test.zip") - ); - parseZip(seaweedInputStream); - - long swProcessTime = System.currentTimeMillis() - startTime2; - - System.out.println("Local time: " + localProcessTime); - System.out.println("SeaweedFS time: " + swProcessTime); - - } - - public static void parseZip(String filename) throws IOException { - FileInputStream fileInputStream = new FileInputStream(filename); - parseZip(fileInputStream); - } - - public static void parseZip(InputStream is) throws IOException { - ZipInputStream zin = new ZipInputStream(is); - ZipEntry ze; - while ((ze = zin.getNextEntry()) != null) { - System.out.println(ze.getName()); - } - } -} diff --git a/other/java/examples/src/main/java/com/seaweedfs/examples/UnzipFile.java b/other/java/examples/src/main/java/com/seaweedfs/examples/UnzipFile.java new file mode 100644 index 000000000..0529a5c73 --- /dev/null +++ b/other/java/examples/src/main/java/com/seaweedfs/examples/UnzipFile.java @@ -0,0 +1,54 @@ +package com.seaweedfs.examples; + +import seaweed.hdfs.SeaweedInputStream; +import seaweedfs.client.FilerClient; +import seaweedfs.client.FilerGrpcClient; + +import java.io.FileInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.util.zip.ZipEntry; +import java.util.zip.ZipInputStream; + +public class UnzipFile { + + public static void main(String[] args) throws IOException { + + FilerGrpcClient filerGrpcClient = new FilerGrpcClient("localhost", 18888); + FilerClient filerClient = new FilerClient(filerGrpcClient); + + long startTime = System.currentTimeMillis(); + parseZip("/Users/chris/tmp/test.zip"); + + long startTime2 = System.currentTimeMillis(); + + long localProcessTime = startTime2 - startTime; + + SeaweedInputStream seaweedInputStream = new SeaweedInputStream( + filerGrpcClient, + new org.apache.hadoop.fs.FileSystem.Statistics(""), + "/", + filerClient.lookupEntry("/", "test.zip") + ); + parseZip(seaweedInputStream); + + long swProcessTime = System.currentTimeMillis() - startTime2; + + System.out.println("Local time: " + localProcessTime); + System.out.println("SeaweedFS time: " + swProcessTime); + + } + + public static void parseZip(String filename) throws IOException { + FileInputStream fileInputStream = new FileInputStream(filename); + parseZip(fileInputStream); + } + + public static void parseZip(InputStream is) throws IOException { + ZipInputStream zin = new ZipInputStream(is); + ZipEntry ze; + while ((ze = zin.getNextEntry()) != null) { + System.out.println(ze.getName()); + } + } +} diff --git a/other/java/examples/src/main/java/com/seaweedfs/examples/WatchFiles.java b/other/java/examples/src/main/java/com/seaweedfs/examples/WatchFiles.java new file mode 100644 index 000000000..c4f4c81b0 --- /dev/null +++ b/other/java/examples/src/main/java/com/seaweedfs/examples/WatchFiles.java @@ -0,0 +1,42 @@ +package com.seaweedfs.examples; + +import seaweedfs.client.FilerClient; +import seaweedfs.client.FilerGrpcClient; +import seaweedfs.client.FilerProto; + +import java.io.IOException; +import java.util.Iterator; + +public class WatchFiles { + + public static void main(String[] args) throws IOException { + FilerGrpcClient filerGrpcClient = new FilerGrpcClient("localhost", 18888); + FilerClient filerClient = new FilerClient(filerGrpcClient); + + Iterator watch = filerClient.watch( + "/buckets", + "exampleClient", + System.currentTimeMillis() * 1000000L + ); + + while (watch.hasNext()) { + FilerProto.SubscribeMetadataResponse event = watch.next(); + FilerProto.EventNotification notification = event.getEventNotification(); + if (notification.getNewParentPath() != null) { + // move an entry to a new directory, possibly with a new name + if (notification.hasOldEntry() && notification.hasNewEntry()) { + System.out.println("move " + event.getDirectory() + "/" + notification.getOldEntry().getName() + " to " + notification.getNewParentPath() + "/" + notification.getNewEntry().getName()); + } else { + System.out.println("this should not happen."); + } + } else if (notification.hasNewEntry() && !notification.hasOldEntry()) { + System.out.println("create entry " + event.getDirectory() + "/" + notification.getNewEntry().getName()); + } else if (!notification.hasNewEntry() && notification.hasOldEntry()) { + System.out.println("delete entry " + event.getDirectory() + "/" + notification.getOldEntry().getName()); + } else if (notification.hasNewEntry() && notification.hasOldEntry()) { + System.out.println("updated entry " + event.getDirectory() + "/" + notification.getNewEntry().getName()); + } + } + + } +} -- cgit v1.2.3 From 824e96ffcc3903260479b26aa0f27e07fa23a795 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Fri, 13 Nov 2020 12:47:46 -0800 Subject: updates --- .../java/com/seaweedfs/examples/WatchFiles.java | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) (limited to 'other/java') diff --git a/other/java/examples/src/main/java/com/seaweedfs/examples/WatchFiles.java b/other/java/examples/src/main/java/com/seaweedfs/examples/WatchFiles.java index c4f4c81b0..e489cb3b1 100644 --- a/other/java/examples/src/main/java/com/seaweedfs/examples/WatchFiles.java +++ b/other/java/examples/src/main/java/com/seaweedfs/examples/WatchFiles.java @@ -1,38 +1,42 @@ package com.seaweedfs.examples; import seaweedfs.client.FilerClient; -import seaweedfs.client.FilerGrpcClient; import seaweedfs.client.FilerProto; import java.io.IOException; +import java.util.Date; import java.util.Iterator; public class WatchFiles { public static void main(String[] args) throws IOException { - FilerGrpcClient filerGrpcClient = new FilerGrpcClient("localhost", 18888); - FilerClient filerClient = new FilerClient(filerGrpcClient); + + FilerClient filerClient = new FilerClient("localhost", 18888); + + long sinceNs = (System.currentTimeMillis() - 3600 * 1000) * 1000000L; Iterator watch = filerClient.watch( "/buckets", - "exampleClient", - System.currentTimeMillis() * 1000000L + "exampleClientName", + sinceNs ); + System.out.println("Connected to filer, subscribing from " + new Date()); + while (watch.hasNext()) { FilerProto.SubscribeMetadataResponse event = watch.next(); FilerProto.EventNotification notification = event.getEventNotification(); - if (notification.getNewParentPath() != null) { + if (!event.getDirectory().equals(notification.getNewParentPath())) { // move an entry to a new directory, possibly with a new name if (notification.hasOldEntry() && notification.hasNewEntry()) { - System.out.println("move " + event.getDirectory() + "/" + notification.getOldEntry().getName() + " to " + notification.getNewParentPath() + "/" + notification.getNewEntry().getName()); + System.out.println("moved " + event.getDirectory() + "/" + notification.getOldEntry().getName() + " to " + notification.getNewParentPath() + "/" + notification.getNewEntry().getName()); } else { System.out.println("this should not happen."); } } else if (notification.hasNewEntry() && !notification.hasOldEntry()) { - System.out.println("create entry " + event.getDirectory() + "/" + notification.getNewEntry().getName()); + System.out.println("created entry " + event.getDirectory() + "/" + notification.getNewEntry().getName()); } else if (!notification.hasNewEntry() && notification.hasOldEntry()) { - System.out.println("delete entry " + event.getDirectory() + "/" + notification.getOldEntry().getName()); + System.out.println("deleted entry " + event.getDirectory() + "/" + notification.getOldEntry().getName()); } else if (notification.hasNewEntry() && notification.hasOldEntry()) { System.out.println("updated entry " + event.getDirectory() + "/" + notification.getNewEntry().getName()); } -- cgit v1.2.3 From 95c0de285d907cbd826ba6ce97f7c4994c16ffd5 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Sun, 15 Nov 2020 16:58:48 -0800 Subject: refactoring --- other/java/client/src/main/proto/filer.proto | 1 + 1 file changed, 1 insertion(+) (limited to 'other/java') diff --git a/other/java/client/src/main/proto/filer.proto b/other/java/client/src/main/proto/filer.proto index f75caec4e..4d9398897 100644 --- a/other/java/client/src/main/proto/filer.proto +++ b/other/java/client/src/main/proto/filer.proto @@ -362,6 +362,7 @@ message FilerConf { SSD = 1; } DiskType disk_type = 5; + bool fsync = 6; } repeated PathConf locations = 2; } -- cgit v1.2.3 From 61d96fde011f81fb1addfd1a6599a4081c14b5e2 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Sun, 15 Nov 2020 21:26:04 -0800 Subject: protect against edge cases when locations expires --- .../client/src/main/java/seaweedfs/client/SeaweedRead.java | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) (limited to 'other/java') diff --git a/other/java/client/src/main/java/seaweedfs/client/SeaweedRead.java b/other/java/client/src/main/java/seaweedfs/client/SeaweedRead.java index a9ddd51db..2b530d2dd 100644 --- a/other/java/client/src/main/java/seaweedfs/client/SeaweedRead.java +++ b/other/java/client/src/main/java/seaweedfs/client/SeaweedRead.java @@ -28,20 +28,26 @@ public class SeaweedRead { List chunkViews = viewFromVisibles(visibleIntervals, position, bufferLength); + Map knownLocations = new HashMap<>(); + FilerProto.LookupVolumeRequest.Builder lookupRequest = FilerProto.LookupVolumeRequest.newBuilder(); for (ChunkView chunkView : chunkViews) { String vid = parseVolumeId(chunkView.fileId); - if (volumeIdCache.getLocations(vid)==null){ + FilerProto.Locations locations = volumeIdCache.getLocations(vid); + if (locations == null) { lookupRequest.addVolumeIds(vid); + } else { + knownLocations.put(vid, locations); } } - if (lookupRequest.getVolumeIdsCount()>0){ + if (lookupRequest.getVolumeIdsCount() > 0) { FilerProto.LookupVolumeResponse lookupResponse = filerGrpcClient .getBlockingStub().lookupVolume(lookupRequest.build()); Map vid2Locations = lookupResponse.getLocationsMapMap(); - for (Map.Entry entry : vid2Locations.entrySet()) { + for (Map.Entry entry : vid2Locations.entrySet()) { volumeIdCache.setLocations(entry.getKey(), entry.getValue()); + knownLocations.put(entry.getKey(), entry.getValue()); } } @@ -57,7 +63,7 @@ public class SeaweedRead { startOffset += gap; } - FilerProto.Locations locations = volumeIdCache.getLocations(parseVolumeId(chunkView.fileId)); + FilerProto.Locations locations = knownLocations.get(parseVolumeId(chunkView.fileId)); if (locations == null || locations.getLocationsCount() == 0) { LOG.error("failed to locate {}", chunkView.fileId); // log here! -- cgit v1.2.3 From fe0105967520ea9072fc19b0a94216fddb28f4f5 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Sun, 15 Nov 2020 21:27:21 -0800 Subject: volume id cache: reduce cache time --- other/java/client/src/main/java/seaweedfs/client/VolumeIdCache.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'other/java') diff --git a/other/java/client/src/main/java/seaweedfs/client/VolumeIdCache.java b/other/java/client/src/main/java/seaweedfs/client/VolumeIdCache.java index 38daa14ac..fd2649cc2 100644 --- a/other/java/client/src/main/java/seaweedfs/client/VolumeIdCache.java +++ b/other/java/client/src/main/java/seaweedfs/client/VolumeIdCache.java @@ -15,7 +15,7 @@ public class VolumeIdCache { } this.cache = CacheBuilder.newBuilder() .maximumSize(maxEntries) - .expireAfterAccess(1, TimeUnit.HOURS) + .expireAfterAccess(5, TimeUnit.MINUTES) .build(); } -- cgit v1.2.3 From 5f19e81dabf7e7bc4f7e9d5506139d36b981501c Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Mon, 16 Nov 2020 16:50:12 -0800 Subject: filer confi: support hierachical configuration --- other/java/client/src/main/proto/filer.proto | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) (limited to 'other/java') diff --git a/other/java/client/src/main/proto/filer.proto b/other/java/client/src/main/proto/filer.proto index 4d9398897..42787df79 100644 --- a/other/java/client/src/main/proto/filer.proto +++ b/other/java/client/src/main/proto/filer.proto @@ -358,8 +358,9 @@ message FilerConf { string replication = 3; string ttl = 4; enum DiskType { - HDD = 0; - SSD = 1; + NONE = 0; + HDD = 1; + SSD = 2; } DiskType disk_type = 5; bool fsync = 6; -- cgit v1.2.3 From dc304342b2a5113d402833b024c2123ac70fb162 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Tue, 17 Nov 2020 01:00:02 -0800 Subject: fs.configure: configurable volume growth --- other/java/client/src/main/proto/filer.proto | 1 + 1 file changed, 1 insertion(+) (limited to 'other/java') diff --git a/other/java/client/src/main/proto/filer.proto b/other/java/client/src/main/proto/filer.proto index 42787df79..f61b62fa3 100644 --- a/other/java/client/src/main/proto/filer.proto +++ b/other/java/client/src/main/proto/filer.proto @@ -364,6 +364,7 @@ message FilerConf { } DiskType disk_type = 5; bool fsync = 6; + uint32 volume_growth_count = 7; } repeated PathConf locations = 2; } -- cgit v1.2.3 From 4737df597db3664d9d47e7f47ec6204a832e4c78 Mon Sep 17 00:00:00 2001 From: limd Date: Tue, 17 Nov 2020 19:05:57 +0800 Subject: HCFS: 1. add replication parameter 2. fix close sequence --- other/java/hdfs2/src/main/java/seaweed/hdfs/SeaweedFileSystem.java | 5 +++-- other/java/hdfs3/src/main/java/seaweed/hdfs/SeaweedFileSystem.java | 5 +++-- 2 files changed, 6 insertions(+), 4 deletions(-) (limited to 'other/java') diff --git a/other/java/hdfs2/src/main/java/seaweed/hdfs/SeaweedFileSystem.java b/other/java/hdfs2/src/main/java/seaweed/hdfs/SeaweedFileSystem.java index 69df55cf1..fb1f4c53b 100644 --- a/other/java/hdfs2/src/main/java/seaweed/hdfs/SeaweedFileSystem.java +++ b/other/java/hdfs2/src/main/java/seaweed/hdfs/SeaweedFileSystem.java @@ -25,6 +25,7 @@ public class SeaweedFileSystem extends FileSystem { public static final String FS_SEAWEED_FILER_PORT = "fs.seaweed.filer.port"; public static final int FS_SEAWEED_DEFAULT_PORT = 8888; public static final String FS_SEAWEED_BUFFER_SIZE = "fs.seaweed.buffer.size"; + public static final String FS_SEAWEED_REPLICATION = "fs.seaweed.replication"; public static final int FS_SEAWEED_DEFAULT_BUFFER_SIZE = 4 * 1024 * 1024; private static final Logger LOG = LoggerFactory.getLogger(SeaweedFileSystem.class); @@ -67,8 +68,8 @@ public class SeaweedFileSystem extends FileSystem { @Override public void close() throws IOException { - this.seaweedFileSystemStore.close(); super.close(); + this.seaweedFileSystemStore.close(); } @Override @@ -97,7 +98,7 @@ public class SeaweedFileSystem extends FileSystem { path = qualify(path); try { - String replicaPlacement = String.format("%03d", replication - 1); + String replicaPlacement = this.getConf().get(FS_SEAWEED_REPLICATION, String.format("%03d", replication - 1)); int seaweedBufferSize = this.getConf().getInt(FS_SEAWEED_BUFFER_SIZE, FS_SEAWEED_DEFAULT_BUFFER_SIZE); OutputStream outputStream = seaweedFileSystemStore.createFile(path, overwrite, permission, seaweedBufferSize, replicaPlacement); return new FSDataOutputStream(outputStream, statistics); diff --git a/other/java/hdfs3/src/main/java/seaweed/hdfs/SeaweedFileSystem.java b/other/java/hdfs3/src/main/java/seaweed/hdfs/SeaweedFileSystem.java index 69df55cf1..fb1f4c53b 100644 --- a/other/java/hdfs3/src/main/java/seaweed/hdfs/SeaweedFileSystem.java +++ b/other/java/hdfs3/src/main/java/seaweed/hdfs/SeaweedFileSystem.java @@ -25,6 +25,7 @@ public class SeaweedFileSystem extends FileSystem { public static final String FS_SEAWEED_FILER_PORT = "fs.seaweed.filer.port"; public static final int FS_SEAWEED_DEFAULT_PORT = 8888; public static final String FS_SEAWEED_BUFFER_SIZE = "fs.seaweed.buffer.size"; + public static final String FS_SEAWEED_REPLICATION = "fs.seaweed.replication"; public static final int FS_SEAWEED_DEFAULT_BUFFER_SIZE = 4 * 1024 * 1024; private static final Logger LOG = LoggerFactory.getLogger(SeaweedFileSystem.class); @@ -67,8 +68,8 @@ public class SeaweedFileSystem extends FileSystem { @Override public void close() throws IOException { - this.seaweedFileSystemStore.close(); super.close(); + this.seaweedFileSystemStore.close(); } @Override @@ -97,7 +98,7 @@ public class SeaweedFileSystem extends FileSystem { path = qualify(path); try { - String replicaPlacement = String.format("%03d", replication - 1); + String replicaPlacement = this.getConf().get(FS_SEAWEED_REPLICATION, String.format("%03d", replication - 1)); int seaweedBufferSize = this.getConf().getInt(FS_SEAWEED_BUFFER_SIZE, FS_SEAWEED_DEFAULT_BUFFER_SIZE); OutputStream outputStream = seaweedFileSystemStore.createFile(path, overwrite, permission, seaweedBufferSize, replicaPlacement); return new FSDataOutputStream(outputStream, statistics); -- cgit v1.2.3 From da04bb3d1bb60d92fdacfb2edd8c8bdba2643038 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Wed, 18 Nov 2020 01:46:38 -0800 Subject: 1.5.6 --- other/java/client/pom.xml | 7 ++++++- other/java/client/pom.xml.deploy | 2 +- other/java/client/pom_debug.xml | 2 +- other/java/examples/pom.xml | 4 ++-- other/java/hdfs2/dependency-reduced-pom.xml | 2 +- other/java/hdfs2/pom.xml | 2 +- other/java/hdfs3/dependency-reduced-pom.xml | 2 +- other/java/hdfs3/pom.xml | 2 +- 8 files changed, 14 insertions(+), 9 deletions(-) (limited to 'other/java') diff --git a/other/java/client/pom.xml b/other/java/client/pom.xml index 5a4bbaead..4bfc5ab8f 100644 --- a/other/java/client/pom.xml +++ b/other/java/client/pom.xml @@ -5,7 +5,7 @@ com.github.chrislusf seaweedfs-client - 1.5.4 + 1.5.6 org.sonatype.oss @@ -68,6 +68,11 @@ 4.13.1 test + + javax.annotation + javax.annotation-api + 1.3.2 + diff --git a/other/java/client/pom.xml.deploy b/other/java/client/pom.xml.deploy index e24bcca9f..c3c960a28 100644 --- a/other/java/client/pom.xml.deploy +++ b/other/java/client/pom.xml.deploy @@ -5,7 +5,7 @@ com.github.chrislusf seaweedfs-client - 1.5.4 + 1.5.6 org.sonatype.oss diff --git a/other/java/client/pom_debug.xml b/other/java/client/pom_debug.xml index 724c3c3b9..acdf621a5 100644 --- a/other/java/client/pom_debug.xml +++ b/other/java/client/pom_debug.xml @@ -5,7 +5,7 @@ com.github.chrislusf seaweedfs-client - 1.5.4 + 1.5.6 org.sonatype.oss diff --git a/other/java/examples/pom.xml b/other/java/examples/pom.xml index 7cbb56ec3..f7c48d0ab 100644 --- a/other/java/examples/pom.xml +++ b/other/java/examples/pom.xml @@ -11,13 +11,13 @@ com.github.chrislusf seaweedfs-client - 1.5.4 + 1.5.6 compile com.github.chrislusf seaweedfs-hadoop2-client - 1.5.4 + 1.5.6 compile diff --git a/other/java/hdfs2/dependency-reduced-pom.xml b/other/java/hdfs2/dependency-reduced-pom.xml index b598d8402..f7873a435 100644 --- a/other/java/hdfs2/dependency-reduced-pom.xml +++ b/other/java/hdfs2/dependency-reduced-pom.xml @@ -301,7 +301,7 @@ - 1.5.4 + 1.5.6 2.9.2 diff --git a/other/java/hdfs2/pom.xml b/other/java/hdfs2/pom.xml index de518a0dc..bda0eba56 100644 --- a/other/java/hdfs2/pom.xml +++ b/other/java/hdfs2/pom.xml @@ -5,7 +5,7 @@ 4.0.0 - 1.5.4 + 1.5.6 2.9.2 diff --git a/other/java/hdfs3/dependency-reduced-pom.xml b/other/java/hdfs3/dependency-reduced-pom.xml index 262c3ca80..20b52e20f 100644 --- a/other/java/hdfs3/dependency-reduced-pom.xml +++ b/other/java/hdfs3/dependency-reduced-pom.xml @@ -309,7 +309,7 @@ - 1.5.4 + 1.5.6 3.1.1 diff --git a/other/java/hdfs3/pom.xml b/other/java/hdfs3/pom.xml index 1952305e9..85d8db859 100644 --- a/other/java/hdfs3/pom.xml +++ b/other/java/hdfs3/pom.xml @@ -5,7 +5,7 @@ 4.0.0 - 1.5.4 + 1.5.6 3.1.1 -- cgit v1.2.3