From 0fa1269bc77abe30f4d108a88a97e29e1bca3124 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Fri, 13 Dec 2019 00:22:37 -0800 Subject: filer: streaming file listing --- other/java/client/src/main/proto/filer.proto | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'other/java/client/src') diff --git a/other/java/client/src/main/proto/filer.proto b/other/java/client/src/main/proto/filer.proto index 18ccca44f..ef847cbe7 100644 --- a/other/java/client/src/main/proto/filer.proto +++ b/other/java/client/src/main/proto/filer.proto @@ -12,7 +12,7 @@ service SeaweedFiler { rpc LookupDirectoryEntry (LookupDirectoryEntryRequest) returns (LookupDirectoryEntryResponse) { } - rpc ListEntries (ListEntriesRequest) returns (ListEntriesResponse) { + rpc ListEntries (ListEntriesRequest) returns (stream ListEntriesResponse) { } rpc CreateEntry (CreateEntryRequest) returns (CreateEntryResponse) { @@ -64,7 +64,7 @@ message ListEntriesRequest { } message ListEntriesResponse { - repeated Entry entries = 1; + Entry entry = 1; } message Entry { -- cgit v1.2.3 From 987108a2b16e61595d0c7768ca47937d0652d5c4 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Fri, 13 Dec 2019 10:04:53 -0800 Subject: HCFS: streaming list files --- .../client/src/main/java/seaweedfs/client/FilerClient.java | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) (limited to 'other/java/client/src') diff --git a/other/java/client/src/main/java/seaweedfs/client/FilerClient.java b/other/java/client/src/main/java/seaweedfs/client/FilerClient.java index a1e3cdb89..84aa26ad9 100644 --- a/other/java/client/src/main/java/seaweedfs/client/FilerClient.java +++ b/other/java/client/src/main/java/seaweedfs/client/FilerClient.java @@ -7,6 +7,7 @@ import java.nio.file.Path; import java.nio.file.Paths; import java.util.ArrayList; import java.util.Arrays; +import java.util.Iterator; import java.util.List; public class FilerClient { @@ -173,17 +174,18 @@ public class FilerClient { } public List listEntries(String path, String entryPrefix, String lastEntryName, int limit) { - List entries = filerGrpcClient.getBlockingStub().listEntries(FilerProto.ListEntriesRequest.newBuilder() + Iterator iter = filerGrpcClient.getBlockingStub().listEntries(FilerProto.ListEntriesRequest.newBuilder() .setDirectory(path) .setPrefix(entryPrefix) .setStartFromFileName(lastEntryName) .setLimit(limit) - .build()).getEntriesList(); - List fixedEntries = new ArrayList<>(entries.size()); - for (FilerProto.Entry entry : entries) { - fixedEntries.add(fixEntryAfterReading(entry)); + .build()); + List entries = new ArrayList<>(); + while (iter.hasNext()){ + FilerProto.ListEntriesResponse resp = iter.next(); + entries.add(fixEntryAfterReading(resp.getEntry())); } - return fixedEntries; + return entries; } public FilerProto.Entry lookupEntry(String directory, String entryName) { -- cgit v1.2.3 From d4e75a0d183b57180b2ff0be2531db540c0c9aa6 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Wed, 22 Jan 2020 11:42:40 -0800 Subject: filer: option to create file only if it is new, O_EXCL --- other/java/client/src/main/proto/filer.proto | 1 + 1 file changed, 1 insertion(+) (limited to 'other/java/client/src') diff --git a/other/java/client/src/main/proto/filer.proto b/other/java/client/src/main/proto/filer.proto index ef847cbe7..41c1650d4 100644 --- a/other/java/client/src/main/proto/filer.proto +++ b/other/java/client/src/main/proto/filer.proto @@ -123,6 +123,7 @@ message FuseAttributes { message CreateEntryRequest { string directory = 1; Entry entry = 2; + bool o_excl = 3; } message CreateEntryResponse { -- cgit v1.2.3 From 6a5c0370995653621fa8b576ea149e91875938d6 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Wed, 22 Jan 2020 22:59:12 -0800 Subject: fix http range requests --- other/java/client/src/main/java/seaweedfs/client/SeaweedRead.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'other/java/client/src') diff --git a/other/java/client/src/main/java/seaweedfs/client/SeaweedRead.java b/other/java/client/src/main/java/seaweedfs/client/SeaweedRead.java index 2efa64580..b08c14467 100644 --- a/other/java/client/src/main/java/seaweedfs/client/SeaweedRead.java +++ b/other/java/client/src/main/java/seaweedfs/client/SeaweedRead.java @@ -63,7 +63,7 @@ public class SeaweedRead { if (!chunkView.isFullChunk) { request.setHeader(HttpHeaders.ACCEPT_ENCODING, ""); request.setHeader(HttpHeaders.RANGE, - String.format("bytes=%d-%d", chunkView.offset, chunkView.offset + chunkView.size)); + String.format("bytes=%d-%d", chunkView.offset, chunkView.offset + chunkView.size - 1)); } try { -- cgit v1.2.3 From c48fc8b4de5922c44d22da306699f789353ecdd4 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Sat, 25 Jan 2020 09:17:19 -0800 Subject: grpc send error via response instead of grpc error --- other/java/client/src/main/proto/filer.proto | 1 + 1 file changed, 1 insertion(+) (limited to 'other/java/client/src') diff --git a/other/java/client/src/main/proto/filer.proto b/other/java/client/src/main/proto/filer.proto index 41c1650d4..6357d971f 100644 --- a/other/java/client/src/main/proto/filer.proto +++ b/other/java/client/src/main/proto/filer.proto @@ -127,6 +127,7 @@ message CreateEntryRequest { } message CreateEntryResponse { + string error = 1; } message UpdateEntryRequest { -- cgit v1.2.3 From d8dec2323bde1a5ab787b719a240969852004456 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Mon, 24 Feb 2020 14:34:14 -0800 Subject: s3: move buckets folder configuration to filer --- other/java/client/src/main/proto/filer.proto | 1 + 1 file changed, 1 insertion(+) (limited to 'other/java/client/src') diff --git a/other/java/client/src/main/proto/filer.proto b/other/java/client/src/main/proto/filer.proto index 6357d971f..909458daf 100644 --- a/other/java/client/src/main/proto/filer.proto +++ b/other/java/client/src/main/proto/filer.proto @@ -219,4 +219,5 @@ message GetFilerConfigurationResponse { string replication = 2; string collection = 3; uint32 max_mb = 4; + string dir_buckets = 5; } -- cgit v1.2.3 From 6ab7368ef2556ef086d13c6d0d4454f1e98a5cd8 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Mon, 24 Feb 2020 22:28:45 -0800 Subject: filer: dynamically create bucket under /buckets folder --- other/java/client/src/main/proto/filer.proto | 3 +++ 1 file changed, 3 insertions(+) (limited to 'other/java/client/src') diff --git a/other/java/client/src/main/proto/filer.proto b/other/java/client/src/main/proto/filer.proto index 909458daf..6892effe8 100644 --- a/other/java/client/src/main/proto/filer.proto +++ b/other/java/client/src/main/proto/filer.proto @@ -165,6 +165,7 @@ message AssignVolumeRequest { string replication = 3; int32 ttl_sec = 4; string data_center = 5; + string parent_path = 6; } message AssignVolumeResponse { @@ -173,6 +174,8 @@ message AssignVolumeResponse { string public_url = 3; int32 count = 4; string auth = 5; + string collection = 6; + string replication = 7; } message LookupVolumeRequest { -- cgit v1.2.3 From bc38b72a20bd79bf67ee1770e20dcd538285cedf Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Tue, 25 Feb 2020 14:38:36 -0800 Subject: s3: implemented DeleteMultipleObjects --- other/java/client/src/main/proto/filer.proto | 4 ++++ 1 file changed, 4 insertions(+) (limited to 'other/java/client/src') diff --git a/other/java/client/src/main/proto/filer.proto b/other/java/client/src/main/proto/filer.proto index 6892effe8..d26c5595f 100644 --- a/other/java/client/src/main/proto/filer.proto +++ b/other/java/client/src/main/proto/filer.proto @@ -24,6 +24,9 @@ service SeaweedFiler { rpc DeleteEntry (DeleteEntryRequest) returns (DeleteEntryResponse) { } + rpc StreamDeleteEntries (stream DeleteEntryRequest) returns (stream DeleteEntryResponse) { + } + rpc AtomicRenameEntry (AtomicRenameEntryRequest) returns (AtomicRenameEntryResponse) { } @@ -147,6 +150,7 @@ message DeleteEntryRequest { } message DeleteEntryResponse { + string error = 1; } message AtomicRenameEntryRequest { -- cgit v1.2.3 From 0841bedb150fb7d4a96c237961474310942c2454 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Tue, 25 Feb 2020 17:15:09 -0800 Subject: move filer assign volume grpc errror to response --- other/java/client/src/main/proto/filer.proto | 1 + 1 file changed, 1 insertion(+) (limited to 'other/java/client/src') diff --git a/other/java/client/src/main/proto/filer.proto b/other/java/client/src/main/proto/filer.proto index d26c5595f..04901770a 100644 --- a/other/java/client/src/main/proto/filer.proto +++ b/other/java/client/src/main/proto/filer.proto @@ -180,6 +180,7 @@ message AssignVolumeResponse { string auth = 5; string collection = 6; string replication = 7; + string error = 8; } message LookupVolumeRequest { -- cgit v1.2.3 From 555413d9fc4837302ef1c5b2b921b406c9de6777 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Thu, 27 Feb 2020 00:07:13 -0800 Subject: weed queue starts --- other/java/client/src/main/proto/filer.proto | 1 + 1 file changed, 1 insertion(+) (limited to 'other/java/client/src') diff --git a/other/java/client/src/main/proto/filer.proto b/other/java/client/src/main/proto/filer.proto index 04901770a..9ee552561 100644 --- a/other/java/client/src/main/proto/filer.proto +++ b/other/java/client/src/main/proto/filer.proto @@ -228,4 +228,5 @@ message GetFilerConfigurationResponse { string collection = 3; uint32 max_mb = 4; string dir_buckets = 5; + string dir_queues = 6; } -- cgit v1.2.3 From 13e215ee5cb5f4c2873f89c263d8c970e9978b19 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Fri, 6 Mar 2020 00:49:47 -0800 Subject: filer: option to encrypt data on volume server --- other/java/client/src/main/proto/filer.proto | 2 ++ 1 file changed, 2 insertions(+) (limited to 'other/java/client/src') diff --git a/other/java/client/src/main/proto/filer.proto b/other/java/client/src/main/proto/filer.proto index 9ee552561..5983c84d8 100644 --- a/other/java/client/src/main/proto/filer.proto +++ b/other/java/client/src/main/proto/filer.proto @@ -99,6 +99,7 @@ message FileChunk { string source_file_id = 6; // to be deprecated FileId fid = 7; FileId source_fid = 8; + bytes cipher_key = 9; } message FileId { @@ -229,4 +230,5 @@ message GetFilerConfigurationResponse { uint32 max_mb = 4; string dir_buckets = 5; string dir_queues = 6; + bool cipher = 7; } -- cgit v1.2.3 From 2e3f6ad3a97bc7fad349e63289695547f92c1f8b Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Sun, 8 Mar 2020 21:39:33 -0700 Subject: filer: remember content is gzipped or not --- other/java/client/src/main/proto/filer.proto | 1 + 1 file changed, 1 insertion(+) (limited to 'other/java/client/src') diff --git a/other/java/client/src/main/proto/filer.proto b/other/java/client/src/main/proto/filer.proto index 5983c84d8..8df46e917 100644 --- a/other/java/client/src/main/proto/filer.proto +++ b/other/java/client/src/main/proto/filer.proto @@ -100,6 +100,7 @@ message FileChunk { FileId fid = 7; FileId source_fid = 8; bytes cipher_key = 9; + bool is_gzipped = 10; } message FileId { -- cgit v1.2.3