From c08ac536ed83ef2159a13ce826a249223272818f Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Wed, 25 Aug 2021 17:34:29 -0700 Subject: cloud drive: add support for Wasabi * disable md5, sha256 checking to avoid reading one chunk twice * single threaded upload to avoid chunk swapping (to be enhanced later) --- other/java/client/src/main/proto/filer.proto | 5 +++++ 1 file changed, 5 insertions(+) (limited to 'other/java/client/src') diff --git a/other/java/client/src/main/proto/filer.proto b/other/java/client/src/main/proto/filer.proto index 2da575733..a5894a475 100644 --- a/other/java/client/src/main/proto/filer.proto +++ b/other/java/client/src/main/proto/filer.proto @@ -413,12 +413,17 @@ message RemoteConf { string tencent_secret_id = 30; string tencent_secret_key = 31; string tencent_endpoint = 32; + string tencent_region = 33; string baidu_access_key = 35; string baidu_secret_key = 36; string baidu_endpoint = 37; string baidu_region = 38; + string wasabi_access_key = 40; + string wasabi_secret_key = 41; + string wasabi_endpoint = 42; + string wasabi_region = 43; } message RemoteStorageMapping { -- cgit v1.2.3 From 05a648bb96df1be5a9261737d8f6fd01600c6a63 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Thu, 26 Aug 2021 15:18:34 -0700 Subject: refactor: separating out remote.proto --- other/java/client/src/main/proto/filer.proto | 49 +--------------------------- 1 file changed, 1 insertion(+), 48 deletions(-) (limited to 'other/java/client/src') diff --git a/other/java/client/src/main/proto/filer.proto b/other/java/client/src/main/proto/filer.proto index a5894a475..b7e4b0e6e 100644 --- a/other/java/client/src/main/proto/filer.proto +++ b/other/java/client/src/main/proto/filer.proto @@ -336,6 +336,7 @@ message KeepConnectedResponse { message LocateBrokerRequest { string resource = 1; } + message LocateBrokerResponse { bool found = 1; // if found, send the exact address @@ -386,54 +387,6 @@ message FilerConf { ///////////////////////// // Remote Storage related ///////////////////////// -message RemoteConf { - string type = 1; - string name = 2; - string s3_access_key = 4; - string s3_secret_key = 5; - string s3_region = 6; - string s3_endpoint = 7; - string s3_storage_class = 8; - bool s3_force_path_style = 9; - - string gcs_google_application_credentials = 10; - - string azure_account_name = 15; - string azure_account_key = 16; - - string backblaze_key_id = 20; - string backblaze_application_key = 21; - string backblaze_endpoint = 22; - - string aliyun_access_key = 25; - string aliyun_secret_key = 26; - string aliyun_endpoint = 27; - string aliyun_region = 28; - - string tencent_secret_id = 30; - string tencent_secret_key = 31; - string tencent_endpoint = 32; - string tencent_region = 33; - - string baidu_access_key = 35; - string baidu_secret_key = 36; - string baidu_endpoint = 37; - string baidu_region = 38; - - string wasabi_access_key = 40; - string wasabi_secret_key = 41; - string wasabi_endpoint = 42; - string wasabi_region = 43; -} - -message RemoteStorageMapping { - map mappings = 1; -} -message RemoteStorageLocation { - string name = 1; - string bucket = 2; - string path = 3; -} message DownloadToLocalRequest { string directory = 1; string name = 2; -- cgit v1.2.3 From dc936921aa027369d166467c9cb568b756957d1a Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Thu, 26 Aug 2021 18:20:14 -0700 Subject: ensure the following logic are executed --- other/java/client/src/main/java/seaweedfs/client/FilerClient.java | 4 ---- 1 file changed, 4 deletions(-) (limited to 'other/java/client/src') diff --git a/other/java/client/src/main/java/seaweedfs/client/FilerClient.java b/other/java/client/src/main/java/seaweedfs/client/FilerClient.java index dae7753a4..e70f6befa 100644 --- a/other/java/client/src/main/java/seaweedfs/client/FilerClient.java +++ b/other/java/client/src/main/java/seaweedfs/client/FilerClient.java @@ -72,10 +72,6 @@ public class FilerClient extends FilerGrpcClient { } return entry; } else { - String fileId = entry.getChunks(0).getFileId(); - if (fileId != null && fileId.length() != 0) { - return entry; - } FilerProto.Entry.Builder entryBuilder = entry.toBuilder(); entryBuilder.clearChunks(); long fileSize = 0; -- cgit v1.2.3 From 268de45aa521635a377950d7752d8f13316e4cad Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Sun, 29 Aug 2021 20:27:11 -0700 Subject: reset volume id cache if not found --- other/java/client/src/main/java/seaweedfs/client/SeaweedRead.java | 5 +++-- .../java/client/src/main/java/seaweedfs/client/VolumeIdCache.java | 7 +++++++ 2 files changed, 10 insertions(+), 2 deletions(-) (limited to 'other/java/client/src') diff --git a/other/java/client/src/main/java/seaweedfs/client/SeaweedRead.java b/other/java/client/src/main/java/seaweedfs/client/SeaweedRead.java index 384636601..011462a17 100644 --- a/other/java/client/src/main/java/seaweedfs/client/SeaweedRead.java +++ b/other/java/client/src/main/java/seaweedfs/client/SeaweedRead.java @@ -64,10 +64,11 @@ public class SeaweedRead { startOffset += gap; } - FilerProto.Locations locations = knownLocations.get(parseVolumeId(chunkView.fileId)); + String volumeId = parseVolumeId(chunkView.fileId); + FilerProto.Locations locations = knownLocations.get(volumeId); if (locations == null || locations.getLocationsCount() == 0) { LOG.error("failed to locate {}", chunkView.fileId); - // log here! + volumeIdCache.clearLocations(volumeId); return 0; } diff --git a/other/java/client/src/main/java/seaweedfs/client/VolumeIdCache.java b/other/java/client/src/main/java/seaweedfs/client/VolumeIdCache.java index fd2649cc2..86263fff9 100644 --- a/other/java/client/src/main/java/seaweedfs/client/VolumeIdCache.java +++ b/other/java/client/src/main/java/seaweedfs/client/VolumeIdCache.java @@ -26,6 +26,13 @@ public class VolumeIdCache { return this.cache.getIfPresent(volumeId); } + public void clearLocations(String volumeId) { + if (this.cache == null) { + return; + } + this.cache.invalidate(volumeId); + } + public void setLocations(String volumeId, FilerProto.Locations locations) { if (this.cache == null) { return; -- cgit v1.2.3 From 05f32376eb8e215fe69a5942ef741d406b6e93eb Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Sun, 29 Aug 2021 21:02:10 -0700 Subject: add cluster id in filer configuration response --- other/java/client/src/main/proto/filer.proto | 1 + 1 file changed, 1 insertion(+) (limited to 'other/java/client/src') diff --git a/other/java/client/src/main/proto/filer.proto b/other/java/client/src/main/proto/filer.proto index b7e4b0e6e..bb461936c 100644 --- a/other/java/client/src/main/proto/filer.proto +++ b/other/java/client/src/main/proto/filer.proto @@ -305,6 +305,7 @@ message GetFilerConfigurationResponse { string metrics_address = 9; int32 metrics_interval_sec = 10; string version = 11; + string cluster_id = 12; } message SubscribeMetadataRequest { -- cgit v1.2.3