From 43fd11278ef811856551904b531bcc91821d0c9f Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Tue, 31 Aug 2021 23:23:08 -0700 Subject: support follow additional path prefixes --- other/java/client/src/main/proto/filer.proto | 1 + 1 file changed, 1 insertion(+) (limited to 'other/java/client/src') diff --git a/other/java/client/src/main/proto/filer.proto b/other/java/client/src/main/proto/filer.proto index bb461936c..7b1838565 100644 --- a/other/java/client/src/main/proto/filer.proto +++ b/other/java/client/src/main/proto/filer.proto @@ -313,6 +313,7 @@ message SubscribeMetadataRequest { string path_prefix = 2; int64 since_ns = 3; int32 signature = 4; + repeated string path_prefixes = 6; } message SubscribeMetadataResponse { string directory = 1; -- cgit v1.2.3 From 9fb278c92c90e4d57e675f5b52ac2263dbd18f2b Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Mon, 6 Sep 2021 11:33:56 -0700 Subject: Hadoop: avoid case insensitive on windows --- .../main/java/seaweedfs/client/FilerClient.java | 37 +++++++++++----------- .../main/java/seaweedfs/client/SeaweedUtil.java | 13 ++++++++ 2 files changed, 31 insertions(+), 19 deletions(-) (limited to 'other/java/client/src') diff --git a/other/java/client/src/main/java/seaweedfs/client/FilerClient.java b/other/java/client/src/main/java/seaweedfs/client/FilerClient.java index e70f6befa..12c6da631 100644 --- a/other/java/client/src/main/java/seaweedfs/client/FilerClient.java +++ b/other/java/client/src/main/java/seaweedfs/client/FilerClient.java @@ -4,7 +4,6 @@ import com.google.common.base.Strings; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.File; import java.util.ArrayList; import java.util.Arrays; import java.util.Iterator; @@ -108,9 +107,9 @@ public class FilerClient extends FilerGrpcClient { if ("/".equals(path)) { return true; } - File pathFile = new File(path); - String parent = pathFile.getParent().replace('\\','/'); - String name = pathFile.getName(); + String[] dirAndName = SeaweedUtil.toDirAndName(path); + String parent = dirAndName[0]; + String name = dirAndName[1]; mkdirs(parent, mode, uid, gid, userName, groupNames); @@ -129,22 +128,22 @@ public class FilerClient extends FilerGrpcClient { public boolean mv(String oldPath, String newPath) { - File oldPathFile = new File(oldPath); - String oldParent = oldPathFile.getParent().replace('\\','/'); - String oldName = oldPathFile.getName(); + String[] oldDirAndName = SeaweedUtil.toDirAndName(oldPath); + String oldParent = oldDirAndName[0]; + String oldName = oldDirAndName[1]; - File newPathFile = new File(newPath); - String newParent = newPathFile.getParent().replace('\\','/'); - String newName = newPathFile.getName(); + String[] newDirAndName = SeaweedUtil.toDirAndName(newPath); + String newParent = newDirAndName[0]; + String newName = newDirAndName[1]; return atomicRenameEntry(oldParent, oldName, newParent, newName); } public boolean exists(String path){ - File pathFile = new File(path); - String parent = pathFile.getParent(); - String entryName = pathFile.getName(); + String[] dirAndName = SeaweedUtil.toDirAndName(path); + String parent = dirAndName[0]; + String entryName = dirAndName[1]; if(parent == null) { parent = path; entryName =""; @@ -155,9 +154,9 @@ public class FilerClient extends FilerGrpcClient { public boolean rm(String path, boolean isRecursive, boolean ignoreRecusiveError) { - File pathFile = new File(path); - String parent = pathFile.getParent().replace('\\','/'); - String name = pathFile.getName(); + String[] dirAndName = SeaweedUtil.toDirAndName(path); + String parent = dirAndName[0]; + String name = dirAndName[1]; return deleteEntry( parent, @@ -176,9 +175,9 @@ public class FilerClient extends FilerGrpcClient { public boolean touch(String path, long modifiedTimeSecond, int mode, int uid, int gid, String userName, String[] groupNames) { - File pathFile = new File(path); - String parent = pathFile.getParent().replace('\\','/'); - String name = pathFile.getName(); + String[] dirAndName = SeaweedUtil.toDirAndName(path); + String parent = dirAndName[0]; + String name = dirAndName[1]; FilerProto.Entry entry = lookupEntry(parent, name); if (entry == null) { diff --git a/other/java/client/src/main/java/seaweedfs/client/SeaweedUtil.java b/other/java/client/src/main/java/seaweedfs/client/SeaweedUtil.java index c465d935f..6e1905370 100644 --- a/other/java/client/src/main/java/seaweedfs/client/SeaweedUtil.java +++ b/other/java/client/src/main/java/seaweedfs/client/SeaweedUtil.java @@ -27,4 +27,17 @@ public class SeaweedUtil { public static CloseableHttpClient getClosableHttpClient() { return httpClient; } + + public static String[] toDirAndName(String fullpath) { + if (fullpath.endsWith("/")) { + fullpath = fullpath.substring(0, fullpath.length() - 1); + } + if (fullpath.length() == 0) { + return new String[]{"/", ""}; + } + int sep = fullpath.lastIndexOf("/"); + String parent = sep == 0 ? "/" : fullpath.substring(0, sep); + String name = fullpath.substring(sep + 1); + return new String[]{parent, name}; + } } -- cgit v1.2.3 From 728ed21a80c815f3577d37e516655d171453590f Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Mon, 6 Sep 2021 11:39:10 -0700 Subject: just in case --- other/java/client/src/main/java/seaweedfs/client/SeaweedUtil.java | 3 +++ 1 file changed, 3 insertions(+) (limited to 'other/java/client/src') diff --git a/other/java/client/src/main/java/seaweedfs/client/SeaweedUtil.java b/other/java/client/src/main/java/seaweedfs/client/SeaweedUtil.java index 6e1905370..66ea9a98d 100644 --- a/other/java/client/src/main/java/seaweedfs/client/SeaweedUtil.java +++ b/other/java/client/src/main/java/seaweedfs/client/SeaweedUtil.java @@ -29,6 +29,9 @@ public class SeaweedUtil { } public static String[] toDirAndName(String fullpath) { + if (fullpath == null) { + return new String[]{"/", ""}; + } if (fullpath.endsWith("/")) { fullpath = fullpath.substring(0, fullpath.length() - 1); } -- cgit v1.2.3 From fe4794fe9210073670fad8612a57bbbe4197d8f9 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Mon, 6 Sep 2021 11:40:51 -0700 Subject: minor --- .../java/client/src/main/java/seaweedfs/client/FilerClient.java | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) (limited to 'other/java/client/src') diff --git a/other/java/client/src/main/java/seaweedfs/client/FilerClient.java b/other/java/client/src/main/java/seaweedfs/client/FilerClient.java index 12c6da631..e962cbbcc 100644 --- a/other/java/client/src/main/java/seaweedfs/client/FilerClient.java +++ b/other/java/client/src/main/java/seaweedfs/client/FilerClient.java @@ -141,15 +141,12 @@ public class FilerClient extends FilerGrpcClient { } public boolean exists(String path){ + String[] dirAndName = SeaweedUtil.toDirAndName(path); String parent = dirAndName[0]; String entryName = dirAndName[1]; - if(parent == null) { - parent = path; - entryName =""; - } - return lookupEntry(parent, entryName) != null; + return lookupEntry(parent, entryName) != null; } public boolean rm(String path, boolean isRecursive, boolean ignoreRecusiveError) { @@ -167,10 +164,12 @@ public class FilerClient extends FilerGrpcClient { } public boolean touch(String path, int mode) { + String currentUser = System.getProperty("user.name"); long now = System.currentTimeMillis() / 1000L; return touch(path, now, mode, 0, 0, currentUser, new String[]{}); + } public boolean touch(String path, long modifiedTimeSecond, int mode, int uid, int gid, String userName, String[] groupNames) { -- cgit v1.2.3 From 0128239c0f8829bb20edd1df257840dcbfa37c0e Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Tue, 7 Sep 2021 16:43:54 -0700 Subject: handle ipv6 addresses --- .../client/src/main/java/seaweedfs/client/FilerGrpcClient.java | 2 +- .../client/src/main/java/seaweedfs/client/SeaweedUtil.java | 10 ++++++++++ 2 files changed, 11 insertions(+), 1 deletion(-) (limited to 'other/java/client/src') diff --git a/other/java/client/src/main/java/seaweedfs/client/FilerGrpcClient.java b/other/java/client/src/main/java/seaweedfs/client/FilerGrpcClient.java index 6c57e2e0d..56aa35876 100644 --- a/other/java/client/src/main/java/seaweedfs/client/FilerGrpcClient.java +++ b/other/java/client/src/main/java/seaweedfs/client/FilerGrpcClient.java @@ -54,7 +54,7 @@ public class FilerGrpcClient { .negotiationType(NegotiationType.TLS) .sslContext(sslContext)); - filerAddress = String.format("%s:%d", host, grpcPort - 10000); + filerAddress = SeaweedUtil.joinHostPort(host, grpcPort - 10000); FilerProto.GetFilerConfigurationResponse filerConfigurationResponse = this.getBlockingStub().getFilerConfiguration( diff --git a/other/java/client/src/main/java/seaweedfs/client/SeaweedUtil.java b/other/java/client/src/main/java/seaweedfs/client/SeaweedUtil.java index 66ea9a98d..027e49b96 100644 --- a/other/java/client/src/main/java/seaweedfs/client/SeaweedUtil.java +++ b/other/java/client/src/main/java/seaweedfs/client/SeaweedUtil.java @@ -43,4 +43,14 @@ public class SeaweedUtil { String name = fullpath.substring(sep + 1); return new String[]{parent, name}; } + + public static String joinHostPort(String host, int port) { + if (host.startsWith("[") && host.endsWith("]")) { + return host + ":" + port; + } + if (host.indexOf(':')>=0) { + return "[" + host + "]:" + port; + } + return host + ":" + port; + } } -- cgit v1.2.3 From e5fc35ed0c970fea060a5b3b7a3f5efb5af425d6 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Sun, 12 Sep 2021 22:47:52 -0700 Subject: change server address from string to a type --- other/java/client/src/main/java/seaweedfs/client/FilerClient.java | 6 +++++- .../client/src/main/java/seaweedfs/client/FilerGrpcClient.java | 8 ++++---- other/java/client/src/main/proto/filer.proto | 4 ++-- 3 files changed, 11 insertions(+), 7 deletions(-) (limited to 'other/java/client/src') diff --git a/other/java/client/src/main/java/seaweedfs/client/FilerClient.java b/other/java/client/src/main/java/seaweedfs/client/FilerClient.java index e962cbbcc..84ee66e7d 100644 --- a/other/java/client/src/main/java/seaweedfs/client/FilerClient.java +++ b/other/java/client/src/main/java/seaweedfs/client/FilerClient.java @@ -14,7 +14,11 @@ public class FilerClient extends FilerGrpcClient { private static final Logger LOG = LoggerFactory.getLogger(FilerClient.class); public FilerClient(String host, int grpcPort) { - super(host, grpcPort); + super(host, grpcPort-10000, grpcPort); + } + + public FilerClient(String host, int port, int grpcPort) { + super(host, port, grpcPort); } public static String toFileId(FilerProto.FileId fid) { diff --git a/other/java/client/src/main/java/seaweedfs/client/FilerGrpcClient.java b/other/java/client/src/main/java/seaweedfs/client/FilerGrpcClient.java index 56aa35876..54e7ccb68 100644 --- a/other/java/client/src/main/java/seaweedfs/client/FilerGrpcClient.java +++ b/other/java/client/src/main/java/seaweedfs/client/FilerGrpcClient.java @@ -40,11 +40,11 @@ public class FilerGrpcClient { private int volumeServerAccess = VOLUME_SERVER_ACCESS_DIRECT; private String filerAddress; - public FilerGrpcClient(String host, int grpcPort) { - this(host, grpcPort, sslContext); + public FilerGrpcClient(String host, int port, int grpcPort) { + this(host, port, grpcPort, sslContext); } - public FilerGrpcClient(String host, int grpcPort, SslContext sslContext) { + public FilerGrpcClient(String host, int port, int grpcPort, SslContext sslContext) { this(sslContext == null ? ManagedChannelBuilder.forAddress(host, grpcPort).usePlaintext() @@ -54,7 +54,7 @@ public class FilerGrpcClient { .negotiationType(NegotiationType.TLS) .sslContext(sslContext)); - filerAddress = SeaweedUtil.joinHostPort(host, grpcPort - 10000); + filerAddress = SeaweedUtil.joinHostPort(host, port); FilerProto.GetFilerConfigurationResponse filerConfigurationResponse = this.getBlockingStub().getFilerConfiguration( diff --git a/other/java/client/src/main/proto/filer.proto b/other/java/client/src/main/proto/filer.proto index 7b1838565..bb4b6cc15 100644 --- a/other/java/client/src/main/proto/filer.proto +++ b/other/java/client/src/main/proto/filer.proto @@ -238,13 +238,12 @@ message AssignVolumeRequest { message AssignVolumeResponse { string file_id = 1; - string url = 2; - string public_url = 3; int32 count = 4; string auth = 5; string collection = 6; string replication = 7; string error = 8; + Location location = 9; } message LookupVolumeRequest { @@ -258,6 +257,7 @@ message Locations { message Location { string url = 1; string public_url = 2; + uint32 grpc_port = 3; } message LookupVolumeResponse { map locations_map = 1; -- cgit v1.2.3 From 20ac710ceb221a9172f5e4868f045ae210c0262a Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Mon, 13 Sep 2021 02:16:09 -0700 Subject: 2.68 --- other/java/client/src/main/java/seaweedfs/client/SeaweedWrite.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'other/java/client/src') diff --git a/other/java/client/src/main/java/seaweedfs/client/SeaweedWrite.java b/other/java/client/src/main/java/seaweedfs/client/SeaweedWrite.java index 3f9d79b99..db7f1a19c 100644 --- a/other/java/client/src/main/java/seaweedfs/client/SeaweedWrite.java +++ b/other/java/client/src/main/java/seaweedfs/client/SeaweedWrite.java @@ -59,7 +59,7 @@ public class SeaweedWrite { String fileId = response.getFileId(); String auth = response.getAuth(); - String targetUrl = filerClient.getChunkUrl(fileId, response.getUrl(), response.getPublicUrl()); + String targetUrl = filerClient.getChunkUrl(fileId, response.getLocation().getUrl(), response.getLocation().getPublicUrl()); ByteString cipherKeyString = com.google.protobuf.ByteString.EMPTY; byte[] cipherKey = null; -- cgit v1.2.3 From 5ca0a551acd89932bdb79871682527186ed28f40 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Sat, 9 Oct 2021 05:38:15 -0700 Subject: java: adjust cache expiration policy for long running java processes --- other/java/client/src/main/java/seaweedfs/client/ChunkCache.java | 2 +- other/java/client/src/main/java/seaweedfs/client/VolumeIdCache.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) (limited to 'other/java/client/src') diff --git a/other/java/client/src/main/java/seaweedfs/client/ChunkCache.java b/other/java/client/src/main/java/seaweedfs/client/ChunkCache.java index 58870d742..84b2beeb1 100644 --- a/other/java/client/src/main/java/seaweedfs/client/ChunkCache.java +++ b/other/java/client/src/main/java/seaweedfs/client/ChunkCache.java @@ -15,7 +15,7 @@ public class ChunkCache { } this.cache = CacheBuilder.newBuilder() .maximumSize(maxEntries) - .expireAfterAccess(1, TimeUnit.HOURS) + .expireAfterWrite(1, TimeUnit.HOURS) .build(); } diff --git a/other/java/client/src/main/java/seaweedfs/client/VolumeIdCache.java b/other/java/client/src/main/java/seaweedfs/client/VolumeIdCache.java index 86263fff9..6f3d9d8c1 100644 --- a/other/java/client/src/main/java/seaweedfs/client/VolumeIdCache.java +++ b/other/java/client/src/main/java/seaweedfs/client/VolumeIdCache.java @@ -15,7 +15,7 @@ public class VolumeIdCache { } this.cache = CacheBuilder.newBuilder() .maximumSize(maxEntries) - .expireAfterAccess(5, TimeUnit.MINUTES) + .expireAfterWrite(5, TimeUnit.MINUTES) .build(); } -- cgit v1.2.3 From 73369906390467bb2e069a7db542801294507fc5 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Sat, 16 Oct 2021 16:03:16 -0700 Subject: faster file read for large files --- .../src/main/java/seaweedfs/client/ReadChunks.java | 109 ++++++++++++++++++ .../main/java/seaweedfs/client/SeaweedRead.java | 90 +-------------- .../java/seaweedfs/client/SeaweedReadTest.java | 123 +++++++++++++++++++-- 3 files changed, 223 insertions(+), 99 deletions(-) create mode 100644 other/java/client/src/main/java/seaweedfs/client/ReadChunks.java (limited to 'other/java/client/src') diff --git a/other/java/client/src/main/java/seaweedfs/client/ReadChunks.java b/other/java/client/src/main/java/seaweedfs/client/ReadChunks.java new file mode 100644 index 000000000..2eba4f808 --- /dev/null +++ b/other/java/client/src/main/java/seaweedfs/client/ReadChunks.java @@ -0,0 +1,109 @@ +package seaweedfs.client; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Comparator; +import java.util.List; + +public class ReadChunks { + + public static List readResolvedChunks(List chunkList) throws IOException { + List points = new ArrayList<>(chunkList.size() * 2); + for (FilerProto.FileChunk chunk : chunkList) { + points.add(new Point(chunk.getOffset(), chunk, true)); + points.add(new Point(chunk.getOffset() + chunk.getSize(), chunk, false)); + } + Collections.sort(points, new Comparator() { + @Override + public int compare(Point a, Point b) { + int x = (int) (a.x - b.x); + if (a.x != b.x) { + return (int) (a.x - b.x); + } + if (a.ts != b.ts) { + return (int) (a.ts - b.ts); + } + if (!a.isStart) { + return -1; + } + return 1; + } + }); + + long prevX = 0; + List visibles = new ArrayList<>(); + ArrayList queue = new ArrayList<>(); + for (Point point : points) { + if (point.isStart) { + if (queue.size() > 0) { + int lastIndex = queue.size() - 1; + Point lastPoint = queue.get(lastIndex); + if (point.x != prevX && lastPoint.ts < point.ts) { + addToVisibles(visibles, prevX, lastPoint, point); + prevX = point.x; + } + } + // insert into queue + for (int i = queue.size(); i >= 0; i--) { + if (i == 0 || queue.get(i - 1).ts <= point.ts) { + if (i == queue.size()) { + prevX = point.x; + } + queue.add(i, point); + break; + } + } + } else { + int lastIndex = queue.size() - 1; + int index = lastIndex; + Point startPoint = null; + for (; index >= 0; index--) { + startPoint = queue.get(index); + if (startPoint.ts == point.ts) { + queue.remove(index); + break; + } + } + if (index == lastIndex && startPoint != null) { + addToVisibles(visibles, prevX, startPoint, point); + prevX = point.x; + } + } + } + + return visibles; + + } + + private static void addToVisibles(List visibles, long prevX, Point startPoint, Point point) { + if (prevX < point.x) { + FilerProto.FileChunk chunk = startPoint.chunk; + visibles.add(new SeaweedRead.VisibleInterval( + prevX, + point.x, + chunk.getFileId(), + chunk.getMtime(), + prevX - chunk.getOffset(), + chunk.getOffset() == prevX && chunk.getSize() == prevX - startPoint.x, + chunk.getCipherKey().toByteArray(), + chunk.getIsCompressed() + )); + } + } + + static class Point { + long x; + long ts; + FilerProto.FileChunk chunk; + boolean isStart; + + public Point(long x, FilerProto.FileChunk chunk, boolean isStart) { + this.x = x; + this.ts = chunk.getMtime(); + this.chunk = chunk; + this.isStart = isStart; + } + } + +} diff --git a/other/java/client/src/main/java/seaweedfs/client/SeaweedRead.java b/other/java/client/src/main/java/seaweedfs/client/SeaweedRead.java index 011462a17..41033befb 100644 --- a/other/java/client/src/main/java/seaweedfs/client/SeaweedRead.java +++ b/other/java/client/src/main/java/seaweedfs/client/SeaweedRead.java @@ -226,96 +226,8 @@ public class SeaweedRead { chunkList = FileChunkManifest.resolveChunkManifest(filerClient, chunkList); - FilerProto.FileChunk[] chunks = chunkList.toArray(new FilerProto.FileChunk[0]); - Arrays.sort(chunks, new Comparator() { - @Override - public int compare(FilerProto.FileChunk a, FilerProto.FileChunk b) { - // if just a.getMtime() - b.getMtime(), it will overflow! - if (a.getMtime() < b.getMtime()) { - return -1; - } else if (a.getMtime() > b.getMtime()) { - return 1; - } - return 0; - } - }); - - List visibles = new ArrayList<>(); - for (FilerProto.FileChunk chunk : chunks) { - List newVisibles = new ArrayList<>(); - visibles = mergeIntoVisibles(visibles, newVisibles, chunk); - } - - return visibles; - } - - private static List mergeIntoVisibles(List visibles, - List newVisibles, - FilerProto.FileChunk chunk) { - VisibleInterval newV = new VisibleInterval( - chunk.getOffset(), - chunk.getOffset() + chunk.getSize(), - chunk.getFileId(), - chunk.getMtime(), - 0, - true, - chunk.getCipherKey().toByteArray(), - chunk.getIsCompressed() - ); - - // easy cases to speed up - if (visibles.size() == 0) { - visibles.add(newV); - return visibles; - } - if (visibles.get(visibles.size() - 1).stop <= chunk.getOffset()) { - visibles.add(newV); - return visibles; - } - - for (VisibleInterval v : visibles) { - if (v.start < chunk.getOffset() && chunk.getOffset() < v.stop) { - newVisibles.add(new VisibleInterval( - v.start, - chunk.getOffset(), - v.fileId, - v.modifiedTime, - v.chunkOffset, - false, - v.cipherKey, - v.isCompressed - )); - } - long chunkStop = chunk.getOffset() + chunk.getSize(); - if (v.start < chunkStop && chunkStop < v.stop) { - newVisibles.add(new VisibleInterval( - chunkStop, - v.stop, - v.fileId, - v.modifiedTime, - v.chunkOffset + (chunkStop - v.start), - false, - v.cipherKey, - v.isCompressed - )); - } - if (chunkStop <= v.start || v.stop <= chunk.getOffset()) { - newVisibles.add(v); - } - } - newVisibles.add(newV); - - // keep everything sorted - for (int i = newVisibles.size() - 1; i >= 0; i--) { - if (i > 0 && newV.start < newVisibles.get(i - 1).start) { - newVisibles.set(i, newVisibles.get(i - 1)); - } else { - newVisibles.set(i, newV); - break; - } - } + return ReadChunks.readResolvedChunks(chunkList); - return newVisibles; } public static String parseVolumeId(String fileId) { diff --git a/other/java/client/src/test/java/seaweedfs/client/SeaweedReadTest.java b/other/java/client/src/test/java/seaweedfs/client/SeaweedReadTest.java index 44b833c90..6ad9edb2c 100644 --- a/other/java/client/src/test/java/seaweedfs/client/SeaweedReadTest.java +++ b/other/java/client/src/test/java/seaweedfs/client/SeaweedReadTest.java @@ -6,6 +6,7 @@ import org.junit.Test; import java.io.IOException; import java.util.ArrayList; import java.util.List; +import java.util.Random; public class SeaweedReadTest { @@ -13,17 +14,17 @@ public class SeaweedReadTest { public void testNonOverlappingVisibleIntervals() throws IOException { List chunks = new ArrayList<>(); chunks.add(FilerProto.FileChunk.newBuilder() - .setFileId("aaa") - .setOffset(0) - .setSize(100) - .setMtime(1000) - .build()); + .setFileId("aaa") + .setOffset(0) + .setSize(100) + .setMtime(1000) + .build()); chunks.add(FilerProto.FileChunk.newBuilder() - .setFileId("bbb") - .setOffset(100) - .setSize(133) - .setMtime(2000) - .build()); + .setFileId("bbb") + .setOffset(100) + .setSize(133) + .setMtime(2000) + .build()); List visibleIntervals = SeaweedRead.nonOverlappingVisibleIntervals(null, chunks); for (SeaweedRead.VisibleInterval visibleInterval : visibleIntervals) { @@ -61,4 +62,106 @@ public class SeaweedReadTest { } + + @Test + public void testReadResolvedChunks() throws IOException { + List chunks = new ArrayList<>(); + chunks.add(FilerProto.FileChunk.newBuilder() + .setFileId("a") + .setOffset(0) + .setSize(100) + .setMtime(1) + .build()); + chunks.add(FilerProto.FileChunk.newBuilder() + .setFileId("b") + .setOffset(50) + .setSize(100) + .setMtime(2) + .build()); + chunks.add(FilerProto.FileChunk.newBuilder() + .setFileId("c") + .setOffset(200) + .setSize(50) + .setMtime(3) + .build()); + chunks.add(FilerProto.FileChunk.newBuilder() + .setFileId("d") + .setOffset(250) + .setSize(50) + .setMtime(4) + .build()); + chunks.add(FilerProto.FileChunk.newBuilder() + .setFileId("e") + .setOffset(175) + .setSize(100) + .setMtime(5) + .build()); + + List visibleIntervals = ReadChunks.readResolvedChunks(chunks); + for (SeaweedRead.VisibleInterval visibleInterval : visibleIntervals) { + System.out.println("visible:" + visibleInterval); + } + + Assert.assertEquals(4, visibleIntervals.size()); + + SeaweedRead.VisibleInterval visibleInterval = visibleIntervals.get(0); + Assert.assertEquals(visibleInterval.start, 0); + Assert.assertEquals(visibleInterval.stop, 50); + Assert.assertEquals(visibleInterval.modifiedTime, 1); + Assert.assertEquals(visibleInterval.fileId, "a"); + + visibleInterval = visibleIntervals.get(1); + Assert.assertEquals(visibleInterval.start, 50); + Assert.assertEquals(visibleInterval.stop, 150); + Assert.assertEquals(visibleInterval.modifiedTime, 2); + Assert.assertEquals(visibleInterval.fileId, "b"); + + visibleInterval = visibleIntervals.get(2); + Assert.assertEquals(visibleInterval.start, 175); + Assert.assertEquals(visibleInterval.stop, 275); + Assert.assertEquals(visibleInterval.modifiedTime, 5); + Assert.assertEquals(visibleInterval.fileId, "e"); + + visibleInterval = visibleIntervals.get(3); + Assert.assertEquals(visibleInterval.start, 275); + Assert.assertEquals(visibleInterval.stop, 300); + Assert.assertEquals(visibleInterval.modifiedTime, 4); + Assert.assertEquals(visibleInterval.fileId, "d"); + + } + + + @Test + public void testRandomizedReadResolvedChunks() throws IOException { + Random random = new Random(); + int limit = 1024*1024; + long[] array = new long[limit]; + List chunks = new ArrayList<>(); + for (long ts=0;ts<1024;ts++){ + int x = random.nextInt(limit); + int y = random.nextInt(limit); + int size = Math.min(Math.abs(x-y), 1024); + chunks.add(randomWrite(array, Math.min(x,y), size, ts)); + } + + List visibleIntervals = ReadChunks.readResolvedChunks(chunks); + for (SeaweedRead.VisibleInterval visibleInterval : visibleIntervals) { + System.out.println("visible:" + visibleInterval); + for (int i = (int) visibleInterval.start; i Date: Sun, 17 Oct 2021 04:22:42 -0700 Subject: mount: streaming renaming folders --- other/java/client/src/main/proto/filer.proto | 14 ++++++++++++++ 1 file changed, 14 insertions(+) (limited to 'other/java/client/src') diff --git a/other/java/client/src/main/proto/filer.proto b/other/java/client/src/main/proto/filer.proto index bb4b6cc15..24b651fe9 100644 --- a/other/java/client/src/main/proto/filer.proto +++ b/other/java/client/src/main/proto/filer.proto @@ -30,6 +30,8 @@ service SeaweedFiler { rpc AtomicRenameEntry (AtomicRenameEntryRequest) returns (AtomicRenameEntryResponse) { } + rpc StreamRenameEntry (StreamRenameEntryRequest) returns (stream StreamRenameEntryResponse) { + } rpc AssignVolume (AssignVolumeRequest) returns (AssignVolumeResponse) { } @@ -225,6 +227,18 @@ message AtomicRenameEntryRequest { message AtomicRenameEntryResponse { } +message StreamRenameEntryRequest { + string old_directory = 1; + string old_name = 2; + string new_directory = 3; + string new_name = 4; + repeated int32 signatures = 5; +} +message StreamRenameEntryResponse { + string directory = 1; + EventNotification event_notification = 2; + int64 ts_ns = 3; +} message AssignVolumeRequest { int32 count = 1; string collection = 2; -- cgit v1.2.3 From 24858507cca1429bb499c2bdb13236f9ef98d599 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Sat, 30 Oct 2021 19:27:25 -0700 Subject: rename API to avoid confusion --- other/java/client/src/main/java/seaweedfs/client/RemoteUtil.java | 4 ++-- other/java/client/src/main/proto/filer.proto | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) (limited to 'other/java/client/src') diff --git a/other/java/client/src/main/java/seaweedfs/client/RemoteUtil.java b/other/java/client/src/main/java/seaweedfs/client/RemoteUtil.java index 39c17644b..a84d5aa2b 100644 --- a/other/java/client/src/main/java/seaweedfs/client/RemoteUtil.java +++ b/other/java/client/src/main/java/seaweedfs/client/RemoteUtil.java @@ -14,8 +14,8 @@ public class RemoteUtil { String dir = SeaweedOutputStream.getParentDirectory(fullpath); String name = SeaweedOutputStream.getFileName(fullpath); - final FilerProto.DownloadToLocalResponse downloadToLocalResponse = filerClient.getBlockingStub() - .downloadToLocal(FilerProto.DownloadToLocalRequest.newBuilder() + final FilerProto.CacheRemoteObjectToLocalClusterResponse downloadToLocalResponse = filerClient.getBlockingStub() + .downloadToLocal(FilerProto.CacheRemoteObjectToLocalClusterRequest.newBuilder() .setDirectory(dir).setName(name).build()); return downloadToLocalResponse.getEntry(); diff --git a/other/java/client/src/main/proto/filer.proto b/other/java/client/src/main/proto/filer.proto index 24b651fe9..921ffc674 100644 --- a/other/java/client/src/main/proto/filer.proto +++ b/other/java/client/src/main/proto/filer.proto @@ -69,7 +69,7 @@ service SeaweedFiler { rpc KvPut (KvPutRequest) returns (KvPutResponse) { } - rpc DownloadToLocal (DownloadToLocalRequest) returns (DownloadToLocalResponse) { + rpc CacheRemoteObjectToLocalCluster (CacheRemoteObjectToLocalClusterRequest) returns (CacheRemoteObjectToLocalClusterResponse) { } } @@ -403,10 +403,10 @@ message FilerConf { ///////////////////////// // Remote Storage related ///////////////////////// -message DownloadToLocalRequest { +message CacheRemoteObjectToLocalClusterRequest { string directory = 1; string name = 2; } -message DownloadToLocalResponse { +message CacheRemoteObjectToLocalClusterResponse { Entry entry = 1; } -- cgit v1.2.3 From af71ae11aa29350a60ed7d3b9a16276a06ba9dcc Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Wed, 3 Nov 2021 01:09:48 -0700 Subject: master: rename grpc function KeepConnected() to SubscribeVolumeLocationUpdates() --- other/java/client/src/main/proto/filer.proto | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'other/java/client/src') diff --git a/other/java/client/src/main/proto/filer.proto b/other/java/client/src/main/proto/filer.proto index 921ffc674..94d522643 100644 --- a/other/java/client/src/main/proto/filer.proto +++ b/other/java/client/src/main/proto/filer.proto @@ -57,7 +57,7 @@ service SeaweedFiler { rpc SubscribeLocalMetadata (SubscribeMetadataRequest) returns (stream SubscribeMetadataResponse) { } - rpc KeepConnected (stream KeepConnectedRequest) returns (stream KeepConnectedResponse) { + rpc SubscribeVolumeLocationUpdates (stream SubscribeVolumeLocationUpdatesRequest) returns (stream SubscribeVolumeLocationUpdatesResponse) { } rpc LocateBroker (LocateBrokerRequest) returns (LocateBrokerResponse) { @@ -341,12 +341,12 @@ message LogEntry { bytes data = 3; } -message KeepConnectedRequest { +message SubscribeVolumeLocationUpdatesRequest { string name = 1; uint32 grpc_port = 2; repeated string resources = 3; } -message KeepConnectedResponse { +message SubscribeVolumeLocationUpdatesResponse { } message LocateBrokerRequest { -- cgit v1.2.3 From 6fb6480a3beef4cc23ec01a6e8f0b77a004513de Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Wed, 3 Nov 2021 01:36:26 -0700 Subject: Java: 1.7.0 update org.apache.httpcomponents to 4.5.13 update grpc API to use cacheRemoteObjectToLocalCluster --- other/java/client/src/main/java/seaweedfs/client/RemoteUtil.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'other/java/client/src') diff --git a/other/java/client/src/main/java/seaweedfs/client/RemoteUtil.java b/other/java/client/src/main/java/seaweedfs/client/RemoteUtil.java index a84d5aa2b..0d912272b 100644 --- a/other/java/client/src/main/java/seaweedfs/client/RemoteUtil.java +++ b/other/java/client/src/main/java/seaweedfs/client/RemoteUtil.java @@ -14,10 +14,10 @@ public class RemoteUtil { String dir = SeaweedOutputStream.getParentDirectory(fullpath); String name = SeaweedOutputStream.getFileName(fullpath); - final FilerProto.CacheRemoteObjectToLocalClusterResponse downloadToLocalResponse = filerClient.getBlockingStub() - .downloadToLocal(FilerProto.CacheRemoteObjectToLocalClusterRequest.newBuilder() + final FilerProto.CacheRemoteObjectToLocalClusterResponse response = filerClient.getBlockingStub() + .cacheRemoteObjectToLocalCluster(FilerProto.CacheRemoteObjectToLocalClusterRequest.newBuilder() .setDirectory(dir).setName(name).build()); - return downloadToLocalResponse.getEntry(); + return response.getEntry(); } } -- cgit v1.2.3 From 5ea86ef1dabfa6267eb68c1ef82939910f2d3505 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Fri, 5 Nov 2021 17:52:15 -0700 Subject: Revert "master: rename grpc function KeepConnected() to SubscribeVolumeLocationUpdates()" This reverts commit af71ae11aa29350a60ed7d3b9a16276a06ba9dcc. --- other/java/client/src/main/proto/filer.proto | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'other/java/client/src') diff --git a/other/java/client/src/main/proto/filer.proto b/other/java/client/src/main/proto/filer.proto index 94d522643..921ffc674 100644 --- a/other/java/client/src/main/proto/filer.proto +++ b/other/java/client/src/main/proto/filer.proto @@ -57,7 +57,7 @@ service SeaweedFiler { rpc SubscribeLocalMetadata (SubscribeMetadataRequest) returns (stream SubscribeMetadataResponse) { } - rpc SubscribeVolumeLocationUpdates (stream SubscribeVolumeLocationUpdatesRequest) returns (stream SubscribeVolumeLocationUpdatesResponse) { + rpc KeepConnected (stream KeepConnectedRequest) returns (stream KeepConnectedResponse) { } rpc LocateBroker (LocateBrokerRequest) returns (LocateBrokerResponse) { @@ -341,12 +341,12 @@ message LogEntry { bytes data = 3; } -message SubscribeVolumeLocationUpdatesRequest { +message KeepConnectedRequest { string name = 1; uint32 grpc_port = 2; repeated string resources = 3; } -message SubscribeVolumeLocationUpdatesResponse { +message KeepConnectedResponse { } message LocateBrokerRequest { -- cgit v1.2.3 From c7c60d1f8a4b22798bc96f2a7ce646ddecab3237 Mon Sep 17 00:00:00 2001 From: chrislu Date: Sun, 5 Dec 2021 17:15:01 -0800 Subject: Java: add retry on write --- .../main/java/seaweedfs/client/SeaweedWrite.java | 26 ++++++++++++++++++---- 1 file changed, 22 insertions(+), 4 deletions(-) (limited to 'other/java/client/src') diff --git a/other/java/client/src/main/java/seaweedfs/client/SeaweedWrite.java b/other/java/client/src/main/java/seaweedfs/client/SeaweedWrite.java index db7f1a19c..81c9ccb6c 100644 --- a/other/java/client/src/main/java/seaweedfs/client/SeaweedWrite.java +++ b/other/java/client/src/main/java/seaweedfs/client/SeaweedWrite.java @@ -29,11 +29,29 @@ public class SeaweedWrite { final byte[] bytes, final long bytesOffset, final long bytesLength, final String path) throws IOException { - FilerProto.FileChunk.Builder chunkBuilder = writeChunk( - replication, filerClient, offset, bytes, bytesOffset, bytesLength, path); - synchronized (entry) { - entry.addChunks(chunkBuilder); + for (long waitTime = 1000L; waitTime < 10 * 1000; waitTime += waitTime / 2) { + try { + FilerProto.FileChunk.Builder chunkBuilder = writeChunk( + replication, filerClient, offset, bytes, bytesOffset, bytesLength, path); + lastException = null; + synchronized (entry) { + entry.addChunks(chunkBuilder); + } + break; + } catch (IOException ioe) { + LOG.debug("writeData:{}", ioe); + lastException = ioe; + } + try { + Thread.sleep(waitTime); + } catch (InterruptedException e) { + } + } + + if (lastException != null) { + throw lastException; } + } public static FilerProto.FileChunk.Builder writeChunk(final String replication, -- cgit v1.2.3 From df9a72b87f574d21e0556c08c8f07d624ff4fdf9 Mon Sep 17 00:00:00 2001 From: chrislu Date: Sun, 5 Dec 2021 17:24:13 -0800 Subject: fix compilation --- other/java/client/src/main/java/seaweedfs/client/SeaweedWrite.java | 2 ++ 1 file changed, 2 insertions(+) (limited to 'other/java/client/src') diff --git a/other/java/client/src/main/java/seaweedfs/client/SeaweedWrite.java b/other/java/client/src/main/java/seaweedfs/client/SeaweedWrite.java index 81c9ccb6c..1ee745ed0 100644 --- a/other/java/client/src/main/java/seaweedfs/client/SeaweedWrite.java +++ b/other/java/client/src/main/java/seaweedfs/client/SeaweedWrite.java @@ -29,6 +29,8 @@ public class SeaweedWrite { final byte[] bytes, final long bytesOffset, final long bytesLength, final String path) throws IOException { + + IOException lastException = null; for (long waitTime = 1000L; waitTime < 10 * 1000; waitTime += waitTime / 2) { try { FilerProto.FileChunk.Builder chunkBuilder = writeChunk( -- cgit v1.2.3 From 08336be92e6b01ad5ec290c7f395b77dcb9f3174 Mon Sep 17 00:00:00 2001 From: banjiaojuhao Date: Wed, 22 Dec 2021 21:57:26 +0800 Subject: filer server: allow upload file to specific dataNode --- other/java/client/src/main/proto/filer.proto | 1 + 1 file changed, 1 insertion(+) (limited to 'other/java/client/src') diff --git a/other/java/client/src/main/proto/filer.proto b/other/java/client/src/main/proto/filer.proto index 921ffc674..42a19aa49 100644 --- a/other/java/client/src/main/proto/filer.proto +++ b/other/java/client/src/main/proto/filer.proto @@ -247,6 +247,7 @@ message AssignVolumeRequest { string data_center = 5; string path = 6; string rack = 7; + string data_node = 9; string disk_type = 8; } -- cgit v1.2.3 From 083bf3a1370c3c9b119fa1c1126e481b6b7ce750 Mon Sep 17 00:00:00 2001 From: banjiaojuhao Date: Thu, 23 Dec 2021 23:24:10 +0800 Subject: filer server: add "datacenter, rack and datanode" for path specific configuration --- other/java/client/src/main/proto/filer.proto | 3 +++ 1 file changed, 3 insertions(+) (limited to 'other/java/client/src') diff --git a/other/java/client/src/main/proto/filer.proto b/other/java/client/src/main/proto/filer.proto index 42a19aa49..ba5c1c694 100644 --- a/other/java/client/src/main/proto/filer.proto +++ b/other/java/client/src/main/proto/filer.proto @@ -397,6 +397,9 @@ message FilerConf { bool fsync = 6; uint32 volume_growth_count = 7; bool read_only = 8; + string data_center = 9; + string rack = 10; + string data_node = 11; } repeated PathConf locations = 2; } -- cgit v1.2.3 From 5c87fcc6d28b230154db35cbe7735a5f1b84024f Mon Sep 17 00:00:00 2001 From: chrislu Date: Thu, 30 Dec 2021 00:23:57 -0800 Subject: add client id for all metadata listening clients --- other/java/client/src/main/java/seaweedfs/client/FilerClient.java | 1 + other/java/client/src/main/java/seaweedfs/client/FilerGrpcClient.java | 3 +++ other/java/client/src/main/proto/filer.proto | 1 + 3 files changed, 5 insertions(+) (limited to 'other/java/client/src') diff --git a/other/java/client/src/main/java/seaweedfs/client/FilerClient.java b/other/java/client/src/main/java/seaweedfs/client/FilerClient.java index 84ee66e7d..10d263968 100644 --- a/other/java/client/src/main/java/seaweedfs/client/FilerClient.java +++ b/other/java/client/src/main/java/seaweedfs/client/FilerClient.java @@ -368,6 +368,7 @@ public class FilerClient extends FilerGrpcClient { .setPathPrefix(prefix) .setClientName(clientName) .setSinceNs(sinceNs) + .setClientId(this.randomClientId) .build() ); } diff --git a/other/java/client/src/main/java/seaweedfs/client/FilerGrpcClient.java b/other/java/client/src/main/java/seaweedfs/client/FilerGrpcClient.java index 54e7ccb68..0a2e6332e 100644 --- a/other/java/client/src/main/java/seaweedfs/client/FilerGrpcClient.java +++ b/other/java/client/src/main/java/seaweedfs/client/FilerGrpcClient.java @@ -11,6 +11,7 @@ import org.slf4j.LoggerFactory; import javax.net.ssl.SSLException; import java.util.HashMap; import java.util.Map; +import java.util.Random; import java.util.concurrent.TimeUnit; public class FilerGrpcClient { @@ -30,6 +31,7 @@ public class FilerGrpcClient { public final int VOLUME_SERVER_ACCESS_PUBLIC_URL = 1; public final int VOLUME_SERVER_ACCESS_FILER_PROXY = 2; public final Map vidLocations = new HashMap<>(); + protected int randomClientId; private final ManagedChannel channel; private final SeaweedFilerGrpc.SeaweedFilerBlockingStub blockingStub; private final SeaweedFilerGrpc.SeaweedFilerStub asyncStub; @@ -62,6 +64,7 @@ public class FilerGrpcClient { cipher = filerConfigurationResponse.getCipher(); collection = filerConfigurationResponse.getCollection(); replication = filerConfigurationResponse.getReplication(); + randomClientId = new Random().nextInt(); } diff --git a/other/java/client/src/main/proto/filer.proto b/other/java/client/src/main/proto/filer.proto index ba5c1c694..3303dc444 100644 --- a/other/java/client/src/main/proto/filer.proto +++ b/other/java/client/src/main/proto/filer.proto @@ -329,6 +329,7 @@ message SubscribeMetadataRequest { int64 since_ns = 3; int32 signature = 4; repeated string path_prefixes = 6; + int32 client_id = 7; } message SubscribeMetadataResponse { string directory = 1; -- cgit v1.2.3 From b1063162b68ff2e84d3038c13c151e479475b6ef Mon Sep 17 00:00:00 2001 From: chrislu Date: Fri, 21 Jan 2022 00:55:04 -0800 Subject: display bucket quota --- other/java/client/src/main/proto/filer.proto | 1 + 1 file changed, 1 insertion(+) (limited to 'other/java/client/src') diff --git a/other/java/client/src/main/proto/filer.proto b/other/java/client/src/main/proto/filer.proto index 3303dc444..3db2b53c9 100644 --- a/other/java/client/src/main/proto/filer.proto +++ b/other/java/client/src/main/proto/filer.proto @@ -114,6 +114,7 @@ message Entry { bytes content = 9; // if not empty, the file content RemoteEntry remote_entry = 10; + int64 quota = 11; // for bucket only. Positive/Negative means enabled/disabled. } message FullEntry { -- cgit v1.2.3 From ceaf993a271dde56c3ce2d8c9f6c448acb946773 Mon Sep 17 00:00:00 2001 From: chrislu Date: Thu, 24 Feb 2022 14:51:25 -0800 Subject: mount2: add rdev --- other/java/client/src/main/proto/filer.proto | 1 + 1 file changed, 1 insertion(+) (limited to 'other/java/client/src') diff --git a/other/java/client/src/main/proto/filer.proto b/other/java/client/src/main/proto/filer.proto index 3db2b53c9..4740637db 100644 --- a/other/java/client/src/main/proto/filer.proto +++ b/other/java/client/src/main/proto/filer.proto @@ -171,6 +171,7 @@ message FuseAttributes { string symlink_target = 13; bytes md5 = 14; string disk_type = 15; + uint32 rdev = 16; } message CreateEntryRequest { -- cgit v1.2.3 From be3fc7739161a29e57ae1481358656af385b1002 Mon Sep 17 00:00:00 2001 From: chrislu Date: Fri, 25 Feb 2022 00:53:27 -0800 Subject: mount2: use consistent inode --- other/java/client/src/main/proto/filer.proto | 1 + 1 file changed, 1 insertion(+) (limited to 'other/java/client/src') diff --git a/other/java/client/src/main/proto/filer.proto b/other/java/client/src/main/proto/filer.proto index 4740637db..36b253eec 100644 --- a/other/java/client/src/main/proto/filer.proto +++ b/other/java/client/src/main/proto/filer.proto @@ -172,6 +172,7 @@ message FuseAttributes { bytes md5 = 14; string disk_type = 15; uint32 rdev = 16; + uint64 inode = 17; } message CreateEntryRequest { -- cgit v1.2.3 From 4042fdf3bb5dc9cab2f6df3911819678bbf03e01 Mon Sep 17 00:00:00 2001 From: chrislu Date: Wed, 16 Mar 2022 23:55:31 -0700 Subject: rename to skipCheckParentDir related to https://github.com/chrislusf/seaweedfs/pull/2761 It's better to default to false. --- other/java/client/src/main/proto/filer.proto | 1 + 1 file changed, 1 insertion(+) (limited to 'other/java/client/src') diff --git a/other/java/client/src/main/proto/filer.proto b/other/java/client/src/main/proto/filer.proto index 36b253eec..389332114 100644 --- a/other/java/client/src/main/proto/filer.proto +++ b/other/java/client/src/main/proto/filer.proto @@ -181,6 +181,7 @@ message CreateEntryRequest { bool o_excl = 3; bool is_from_other_cluster = 4; repeated int32 signatures = 5; + bool skip_check_parent_directory = 6; } message CreateEntryResponse { -- cgit v1.2.3 From 743ad690b69d01081975bdb435364d8509dbe01b Mon Sep 17 00:00:00 2001 From: chrislu Date: Fri, 1 Apr 2022 16:44:58 -0700 Subject: filer supports grpc ping --- other/java/client/src/main/proto/filer.proto | 10 ++++++++++ 1 file changed, 10 insertions(+) (limited to 'other/java/client/src') diff --git a/other/java/client/src/main/proto/filer.proto b/other/java/client/src/main/proto/filer.proto index 389332114..35086e245 100644 --- a/other/java/client/src/main/proto/filer.proto +++ b/other/java/client/src/main/proto/filer.proto @@ -48,6 +48,9 @@ service SeaweedFiler { rpc Statistics (StatisticsRequest) returns (StatisticsResponse) { } + rpc Ping (PingRequest) returns (PingResponse) { + } + rpc GetFilerConfiguration (GetFilerConfigurationRequest) returns (GetFilerConfigurationResponse) { } @@ -311,6 +314,13 @@ message StatisticsResponse { uint64 file_count = 6; } +message PingRequest { + string target = 1; // default to ping itself + string target_type = 2; +} +message PingResponse { +} + message GetFilerConfigurationRequest { } message GetFilerConfigurationResponse { -- cgit v1.2.3 From b4be56bb3b97e979221ba3a123186479ba90cc57 Mon Sep 17 00:00:00 2001 From: chrislu Date: Sat, 16 Apr 2022 12:45:49 -0700 Subject: add timing info during ping operation --- other/java/client/src/main/proto/filer.proto | 3 +++ 1 file changed, 3 insertions(+) (limited to 'other/java/client/src') diff --git a/other/java/client/src/main/proto/filer.proto b/other/java/client/src/main/proto/filer.proto index 35086e245..11be522e7 100644 --- a/other/java/client/src/main/proto/filer.proto +++ b/other/java/client/src/main/proto/filer.proto @@ -319,6 +319,9 @@ message PingRequest { string target_type = 2; } message PingResponse { + int64 start_time_ns = 1; + int64 remote_time_ns = 2; + int64 stop_time_ns = 3; } message GetFilerConfigurationRequest { -- cgit v1.2.3 From ca16fbf0ef7acfae48a4dedb17bd4d9f2491052e Mon Sep 17 00:00:00 2001 From: chrislu Date: Sun, 29 May 2022 16:37:14 -0700 Subject: shell: cluster.ps display filer group --- other/java/client/src/main/proto/filer.proto | 1 + 1 file changed, 1 insertion(+) (limited to 'other/java/client/src') diff --git a/other/java/client/src/main/proto/filer.proto b/other/java/client/src/main/proto/filer.proto index 11be522e7..e5c7f21e7 100644 --- a/other/java/client/src/main/proto/filer.proto +++ b/other/java/client/src/main/proto/filer.proto @@ -338,6 +338,7 @@ message GetFilerConfigurationResponse { int32 metrics_interval_sec = 10; string version = 11; string cluster_id = 12; + string filer_group = 13; } message SubscribeMetadataRequest { -- cgit v1.2.3 From a2b101a737de0a4085f560971f6f25cb8f4e6050 Mon Sep 17 00:00:00 2001 From: chrislu Date: Mon, 30 May 2022 15:04:19 -0700 Subject: subscribe metadata between a range --- other/java/client/src/main/proto/filer.proto | 1 + 1 file changed, 1 insertion(+) (limited to 'other/java/client/src') diff --git a/other/java/client/src/main/proto/filer.proto b/other/java/client/src/main/proto/filer.proto index e5c7f21e7..e52bbcf89 100644 --- a/other/java/client/src/main/proto/filer.proto +++ b/other/java/client/src/main/proto/filer.proto @@ -348,6 +348,7 @@ message SubscribeMetadataRequest { int32 signature = 4; repeated string path_prefixes = 6; int32 client_id = 7; + int64 until_ns = 8; } message SubscribeMetadataResponse { string directory = 1; -- cgit v1.2.3 From 4fd5f9659836390b9949838d8ca0e9feec35f66d Mon Sep 17 00:00:00 2001 From: chrislu Date: Mon, 6 Jun 2022 00:39:35 -0700 Subject: filer: remove replication, collection, disk_type info from entry metadata these metadata can change and are not used --- other/java/client/src/main/proto/filer.proto | 3 --- 1 file changed, 3 deletions(-) (limited to 'other/java/client/src') diff --git a/other/java/client/src/main/proto/filer.proto b/other/java/client/src/main/proto/filer.proto index e52bbcf89..bd0932cb8 100644 --- a/other/java/client/src/main/proto/filer.proto +++ b/other/java/client/src/main/proto/filer.proto @@ -166,14 +166,11 @@ message FuseAttributes { uint32 gid = 5; int64 crtime = 6; // unix time in seconds string mime = 7; - string replication = 8; - string collection = 9; int32 ttl_sec = 10; string user_name = 11; // for hdfs repeated string group_name = 12; // for hdfs string symlink_target = 13; bytes md5 = 14; - string disk_type = 15; uint32 rdev = 16; uint64 inode = 17; } -- cgit v1.2.3 From 28f615dcae3972abb1cd25ce66de8c0b4c31c752 Mon Sep 17 00:00:00 2001 From: chrislu Date: Thu, 23 Jun 2022 03:05:59 -0700 Subject: java: fix data encryption fix https://github.com/chrislusf/seaweedfs/issues/3211 --- other/java/client/src/main/java/seaweedfs/client/SeaweedCipher.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'other/java/client/src') diff --git a/other/java/client/src/main/java/seaweedfs/client/SeaweedCipher.java b/other/java/client/src/main/java/seaweedfs/client/SeaweedCipher.java index 8d0ebd755..979decb8d 100644 --- a/other/java/client/src/main/java/seaweedfs/client/SeaweedCipher.java +++ b/other/java/client/src/main/java/seaweedfs/client/SeaweedCipher.java @@ -36,7 +36,7 @@ public class SeaweedCipher { byte[] encryptedText = AES_cipherInstance.doFinal(clearTextbytes, offset, length); byte[] iv = AES_cipherInstance.getIV(); - byte[] message = new byte[GCM_NONCE_LENGTH + clearTextbytes.length + GCM_TAG_LENGTH]; + byte[] message = new byte[GCM_NONCE_LENGTH + length + GCM_TAG_LENGTH]; System.arraycopy(iv, 0, message, 0, GCM_NONCE_LENGTH); System.arraycopy(encryptedText, 0, message, GCM_NONCE_LENGTH, encryptedText.length); -- cgit v1.2.3