From c32b6f4d9d99d175863e778dcc0da8ef3da27d96 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Wed, 11 Nov 2020 23:46:42 -0800 Subject: prepare for path based configuration --- other/java/client/src/main/proto/filer.proto | 8 ++++++++ 1 file changed, 8 insertions(+) (limited to 'other/java/client/src') diff --git a/other/java/client/src/main/proto/filer.proto b/other/java/client/src/main/proto/filer.proto index 11c29e6ec..8829062f4 100644 --- a/other/java/client/src/main/proto/filer.proto +++ b/other/java/client/src/main/proto/filer.proto @@ -348,3 +348,11 @@ message KvPutRequest { message KvPutResponse { string error = 1; } + +// path-based configurations +message FilerPathConfiguration { + string location = 1; + string collection = 2; + string replication = 3; + string ttl = 4; +} -- cgit v1.2.3 From a4dfa00030b7c7e0aeeaee7c00da203dc9cd0ed0 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Thu, 12 Nov 2020 00:39:59 -0800 Subject: adjust filer conf --- other/java/client/src/main/proto/filer.proto | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) (limited to 'other/java/client/src') diff --git a/other/java/client/src/main/proto/filer.proto b/other/java/client/src/main/proto/filer.proto index 8829062f4..f75caec4e 100644 --- a/other/java/client/src/main/proto/filer.proto +++ b/other/java/client/src/main/proto/filer.proto @@ -350,9 +350,18 @@ message KvPutResponse { } // path-based configurations -message FilerPathConfiguration { - string location = 1; - string collection = 2; - string replication = 3; - string ttl = 4; +message FilerConf { + int32 version = 1; + message PathConf { + string location_prefix = 1; + string collection = 2; + string replication = 3; + string ttl = 4; + enum DiskType { + HDD = 0; + SSD = 1; + } + DiskType disk_type = 5; + } + repeated PathConf locations = 2; } -- cgit v1.2.3 From a2962604ad0bf83b26c692c72ddd40e648fb804d Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Thu, 12 Nov 2020 23:25:11 -0800 Subject: add volume id cache --- .../src/main/java/seaweedfs/client/ChunkCache.java | 1 - .../main/java/seaweedfs/client/SeaweedRead.java | 19 ++++++++---- .../main/java/seaweedfs/client/VolumeIdCache.java | 36 ++++++++++++++++++++++ 3 files changed, 49 insertions(+), 7 deletions(-) create mode 100644 other/java/client/src/main/java/seaweedfs/client/VolumeIdCache.java (limited to 'other/java/client/src') diff --git a/other/java/client/src/main/java/seaweedfs/client/ChunkCache.java b/other/java/client/src/main/java/seaweedfs/client/ChunkCache.java index 7afa2dca0..58870d742 100644 --- a/other/java/client/src/main/java/seaweedfs/client/ChunkCache.java +++ b/other/java/client/src/main/java/seaweedfs/client/ChunkCache.java @@ -15,7 +15,6 @@ public class ChunkCache { } this.cache = CacheBuilder.newBuilder() .maximumSize(maxEntries) - .weakValues() .expireAfterAccess(1, TimeUnit.HOURS) .build(); } diff --git a/other/java/client/src/main/java/seaweedfs/client/SeaweedRead.java b/other/java/client/src/main/java/seaweedfs/client/SeaweedRead.java index ab2407dec..a9ddd51db 100644 --- a/other/java/client/src/main/java/seaweedfs/client/SeaweedRead.java +++ b/other/java/client/src/main/java/seaweedfs/client/SeaweedRead.java @@ -19,6 +19,7 @@ public class SeaweedRead { private static final Logger LOG = LoggerFactory.getLogger(SeaweedRead.class); static ChunkCache chunkCache = new ChunkCache(4); + static VolumeIdCache volumeIdCache = new VolumeIdCache(4 * 1024); // returns bytesRead public static long read(FilerGrpcClient filerGrpcClient, List visibleIntervals, @@ -30,13 +31,19 @@ public class SeaweedRead { FilerProto.LookupVolumeRequest.Builder lookupRequest = FilerProto.LookupVolumeRequest.newBuilder(); for (ChunkView chunkView : chunkViews) { String vid = parseVolumeId(chunkView.fileId); - lookupRequest.addVolumeIds(vid); + if (volumeIdCache.getLocations(vid)==null){ + lookupRequest.addVolumeIds(vid); + } } - FilerProto.LookupVolumeResponse lookupResponse = filerGrpcClient - .getBlockingStub().lookupVolume(lookupRequest.build()); - - Map vid2Locations = lookupResponse.getLocationsMapMap(); + if (lookupRequest.getVolumeIdsCount()>0){ + FilerProto.LookupVolumeResponse lookupResponse = filerGrpcClient + .getBlockingStub().lookupVolume(lookupRequest.build()); + Map vid2Locations = lookupResponse.getLocationsMapMap(); + for (Map.Entry entry : vid2Locations.entrySet()) { + volumeIdCache.setLocations(entry.getKey(), entry.getValue()); + } + } //TODO parallel this long readCount = 0; @@ -50,7 +57,7 @@ public class SeaweedRead { startOffset += gap; } - FilerProto.Locations locations = vid2Locations.get(parseVolumeId(chunkView.fileId)); + FilerProto.Locations locations = volumeIdCache.getLocations(parseVolumeId(chunkView.fileId)); if (locations == null || locations.getLocationsCount() == 0) { LOG.error("failed to locate {}", chunkView.fileId); // log here! diff --git a/other/java/client/src/main/java/seaweedfs/client/VolumeIdCache.java b/other/java/client/src/main/java/seaweedfs/client/VolumeIdCache.java new file mode 100644 index 000000000..38daa14ac --- /dev/null +++ b/other/java/client/src/main/java/seaweedfs/client/VolumeIdCache.java @@ -0,0 +1,36 @@ +package seaweedfs.client; + +import com.google.common.cache.Cache; +import com.google.common.cache.CacheBuilder; + +import java.util.concurrent.TimeUnit; + +public class VolumeIdCache { + + private Cache cache = null; + + public VolumeIdCache(int maxEntries) { + if (maxEntries == 0) { + return; + } + this.cache = CacheBuilder.newBuilder() + .maximumSize(maxEntries) + .expireAfterAccess(1, TimeUnit.HOURS) + .build(); + } + + public FilerProto.Locations getLocations(String volumeId) { + if (this.cache == null) { + return null; + } + return this.cache.getIfPresent(volumeId); + } + + public void setLocations(String volumeId, FilerProto.Locations locations) { + if (this.cache == null) { + return; + } + this.cache.put(volumeId, locations); + } + +} -- cgit v1.2.3 From 3e362451d226d9e19b4b652a02926dedc02f6cf9 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Fri, 13 Nov 2020 12:10:55 -0800 Subject: add example of watch files --- .../src/main/java/seaweedfs/client/FilerClient.java | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) (limited to 'other/java/client/src') diff --git a/other/java/client/src/main/java/seaweedfs/client/FilerClient.java b/other/java/client/src/main/java/seaweedfs/client/FilerClient.java index 035b2c852..7338d5bee 100644 --- a/other/java/client/src/main/java/seaweedfs/client/FilerClient.java +++ b/other/java/client/src/main/java/seaweedfs/client/FilerClient.java @@ -275,9 +275,9 @@ public class FilerClient { try { FilerProto.CreateEntryResponse createEntryResponse = filerGrpcClient.getBlockingStub().createEntry(FilerProto.CreateEntryRequest.newBuilder() - .setDirectory(parent) - .setEntry(entry) - .build()); + .setDirectory(parent) + .setEntry(entry) + .build()); if (Strings.isNullOrEmpty(createEntryResponse.getError())) { return true; } @@ -333,4 +333,13 @@ public class FilerClient { return true; } + public Iterator watch(String prefix, String clientName, long sinceNs) { + return filerGrpcClient.getBlockingStub().subscribeMetadata(FilerProto.SubscribeMetadataRequest.newBuilder() + .setPathPrefix(prefix) + .setClientName(clientName) + .setSinceNs(sinceNs) + .build() + ); + } + } -- cgit v1.2.3 From 95c0de285d907cbd826ba6ce97f7c4994c16ffd5 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Sun, 15 Nov 2020 16:58:48 -0800 Subject: refactoring --- other/java/client/src/main/proto/filer.proto | 1 + 1 file changed, 1 insertion(+) (limited to 'other/java/client/src') diff --git a/other/java/client/src/main/proto/filer.proto b/other/java/client/src/main/proto/filer.proto index f75caec4e..4d9398897 100644 --- a/other/java/client/src/main/proto/filer.proto +++ b/other/java/client/src/main/proto/filer.proto @@ -362,6 +362,7 @@ message FilerConf { SSD = 1; } DiskType disk_type = 5; + bool fsync = 6; } repeated PathConf locations = 2; } -- cgit v1.2.3 From 61d96fde011f81fb1addfd1a6599a4081c14b5e2 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Sun, 15 Nov 2020 21:26:04 -0800 Subject: protect against edge cases when locations expires --- .../client/src/main/java/seaweedfs/client/SeaweedRead.java | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) (limited to 'other/java/client/src') diff --git a/other/java/client/src/main/java/seaweedfs/client/SeaweedRead.java b/other/java/client/src/main/java/seaweedfs/client/SeaweedRead.java index a9ddd51db..2b530d2dd 100644 --- a/other/java/client/src/main/java/seaweedfs/client/SeaweedRead.java +++ b/other/java/client/src/main/java/seaweedfs/client/SeaweedRead.java @@ -28,20 +28,26 @@ public class SeaweedRead { List chunkViews = viewFromVisibles(visibleIntervals, position, bufferLength); + Map knownLocations = new HashMap<>(); + FilerProto.LookupVolumeRequest.Builder lookupRequest = FilerProto.LookupVolumeRequest.newBuilder(); for (ChunkView chunkView : chunkViews) { String vid = parseVolumeId(chunkView.fileId); - if (volumeIdCache.getLocations(vid)==null){ + FilerProto.Locations locations = volumeIdCache.getLocations(vid); + if (locations == null) { lookupRequest.addVolumeIds(vid); + } else { + knownLocations.put(vid, locations); } } - if (lookupRequest.getVolumeIdsCount()>0){ + if (lookupRequest.getVolumeIdsCount() > 0) { FilerProto.LookupVolumeResponse lookupResponse = filerGrpcClient .getBlockingStub().lookupVolume(lookupRequest.build()); Map vid2Locations = lookupResponse.getLocationsMapMap(); - for (Map.Entry entry : vid2Locations.entrySet()) { + for (Map.Entry entry : vid2Locations.entrySet()) { volumeIdCache.setLocations(entry.getKey(), entry.getValue()); + knownLocations.put(entry.getKey(), entry.getValue()); } } @@ -57,7 +63,7 @@ public class SeaweedRead { startOffset += gap; } - FilerProto.Locations locations = volumeIdCache.getLocations(parseVolumeId(chunkView.fileId)); + FilerProto.Locations locations = knownLocations.get(parseVolumeId(chunkView.fileId)); if (locations == null || locations.getLocationsCount() == 0) { LOG.error("failed to locate {}", chunkView.fileId); // log here! -- cgit v1.2.3 From fe0105967520ea9072fc19b0a94216fddb28f4f5 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Sun, 15 Nov 2020 21:27:21 -0800 Subject: volume id cache: reduce cache time --- other/java/client/src/main/java/seaweedfs/client/VolumeIdCache.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'other/java/client/src') diff --git a/other/java/client/src/main/java/seaweedfs/client/VolumeIdCache.java b/other/java/client/src/main/java/seaweedfs/client/VolumeIdCache.java index 38daa14ac..fd2649cc2 100644 --- a/other/java/client/src/main/java/seaweedfs/client/VolumeIdCache.java +++ b/other/java/client/src/main/java/seaweedfs/client/VolumeIdCache.java @@ -15,7 +15,7 @@ public class VolumeIdCache { } this.cache = CacheBuilder.newBuilder() .maximumSize(maxEntries) - .expireAfterAccess(1, TimeUnit.HOURS) + .expireAfterAccess(5, TimeUnit.MINUTES) .build(); } -- cgit v1.2.3 From 5f19e81dabf7e7bc4f7e9d5506139d36b981501c Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Mon, 16 Nov 2020 16:50:12 -0800 Subject: filer confi: support hierachical configuration --- other/java/client/src/main/proto/filer.proto | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) (limited to 'other/java/client/src') diff --git a/other/java/client/src/main/proto/filer.proto b/other/java/client/src/main/proto/filer.proto index 4d9398897..42787df79 100644 --- a/other/java/client/src/main/proto/filer.proto +++ b/other/java/client/src/main/proto/filer.proto @@ -358,8 +358,9 @@ message FilerConf { string replication = 3; string ttl = 4; enum DiskType { - HDD = 0; - SSD = 1; + NONE = 0; + HDD = 1; + SSD = 2; } DiskType disk_type = 5; bool fsync = 6; -- cgit v1.2.3 From dc304342b2a5113d402833b024c2123ac70fb162 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Tue, 17 Nov 2020 01:00:02 -0800 Subject: fs.configure: configurable volume growth --- other/java/client/src/main/proto/filer.proto | 1 + 1 file changed, 1 insertion(+) (limited to 'other/java/client/src') diff --git a/other/java/client/src/main/proto/filer.proto b/other/java/client/src/main/proto/filer.proto index 42787df79..f61b62fa3 100644 --- a/other/java/client/src/main/proto/filer.proto +++ b/other/java/client/src/main/proto/filer.proto @@ -364,6 +364,7 @@ message FilerConf { } DiskType disk_type = 5; bool fsync = 6; + uint32 volume_growth_count = 7; } repeated PathConf locations = 2; } -- cgit v1.2.3