From 735912e0f5de0d1d287fa6683504624a52e842e9 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Wed, 29 Jul 2020 17:39:18 -0700 Subject: enable read cache --- other/java/client/src/main/java/seaweedfs/client/SeaweedRead.java | 2 +- other/java/client/src/main/java/seaweedfs/client/SeaweedWrite.java | 3 --- 2 files changed, 1 insertion(+), 4 deletions(-) (limited to 'other/java/client/src') diff --git a/other/java/client/src/main/java/seaweedfs/client/SeaweedRead.java b/other/java/client/src/main/java/seaweedfs/client/SeaweedRead.java index f0490540d..1d8ded426 100644 --- a/other/java/client/src/main/java/seaweedfs/client/SeaweedRead.java +++ b/other/java/client/src/main/java/seaweedfs/client/SeaweedRead.java @@ -15,7 +15,7 @@ public class SeaweedRead { private static final Logger LOG = LoggerFactory.getLogger(SeaweedRead.class); - static ChunkCache chunkCache = new ChunkCache(0); + static ChunkCache chunkCache = new ChunkCache(4); // returns bytesRead public static long read(FilerGrpcClient filerGrpcClient, List visibleIntervals, diff --git a/other/java/client/src/main/java/seaweedfs/client/SeaweedWrite.java b/other/java/client/src/main/java/seaweedfs/client/SeaweedWrite.java index 5f4d888bd..d3cecab75 100644 --- a/other/java/client/src/main/java/seaweedfs/client/SeaweedWrite.java +++ b/other/java/client/src/main/java/seaweedfs/client/SeaweedWrite.java @@ -61,9 +61,6 @@ public class SeaweedWrite { String etag = multipartUpload(targetUrl, auth, bytes, bytesOffset, bytesLength, cipherKey); - // cache fileId ~ bytes - SeaweedRead.chunkCache.setChunk(fileId, bytes); - LOG.debug("write file chunk {} size {}", targetUrl, bytesLength); return FilerProto.FileChunk.newBuilder() -- cgit v1.2.3 From b684c312d232f4b8ffa3bb276033fd69c469d354 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Wed, 29 Jul 2020 18:25:35 -0700 Subject: minor --- other/java/client/src/main/java/seaweedfs/client/SeaweedRead.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) (limited to 'other/java/client/src') diff --git a/other/java/client/src/main/java/seaweedfs/client/SeaweedRead.java b/other/java/client/src/main/java/seaweedfs/client/SeaweedRead.java index 1d8ded426..af3aac0f5 100644 --- a/other/java/client/src/main/java/seaweedfs/client/SeaweedRead.java +++ b/other/java/client/src/main/java/seaweedfs/client/SeaweedRead.java @@ -62,6 +62,7 @@ public class SeaweedRead { if (chunkData == null) { chunkData = doFetchFullChunkData(chunkView, locations); + chunkCache.setChunk(chunkView.fileId, chunkData); } int len = (int) chunkView.size; @@ -69,8 +70,6 @@ public class SeaweedRead { chunkView.fileId, chunkData.length, chunkView.offset, buffer.length, startOffset, len); System.arraycopy(chunkData, (int) chunkView.offset, buffer, startOffset, len); - chunkCache.setChunk(chunkView.fileId, chunkData); - return len; } -- cgit v1.2.3 From daf8c0c8ce4c507264a175912d4693415f40f606 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Wed, 29 Jul 2020 18:25:52 -0700 Subject: cache vid locations --- .../main/java/seaweedfs/client/FileChunkManifest.java | 17 ++++++++++------- .../src/main/java/seaweedfs/client/FilerGrpcClient.java | 3 +++ 2 files changed, 13 insertions(+), 7 deletions(-) (limited to 'other/java/client/src') diff --git a/other/java/client/src/main/java/seaweedfs/client/FileChunkManifest.java b/other/java/client/src/main/java/seaweedfs/client/FileChunkManifest.java index d8d29ede8..a15671f46 100644 --- a/other/java/client/src/main/java/seaweedfs/client/FileChunkManifest.java +++ b/other/java/client/src/main/java/seaweedfs/client/FileChunkManifest.java @@ -6,7 +6,6 @@ import org.slf4j.LoggerFactory; import java.io.IOException; import java.util.ArrayList; import java.util.List; -import java.util.Map; public class FileChunkManifest { @@ -51,13 +50,17 @@ public class FileChunkManifest { private static byte[] fetchChunk(final FilerGrpcClient filerGrpcClient, FilerProto.FileChunk chunk) throws IOException { - FilerProto.LookupVolumeRequest.Builder lookupRequest = FilerProto.LookupVolumeRequest.newBuilder(); String vid = "" + chunk.getFid().getVolumeId(); - lookupRequest.addVolumeIds(vid); - FilerProto.LookupVolumeResponse lookupResponse = filerGrpcClient - .getBlockingStub().lookupVolume(lookupRequest.build()); - Map vid2Locations = lookupResponse.getLocationsMapMap(); - FilerProto.Locations locations = vid2Locations.get(vid); + FilerProto.Locations locations = filerGrpcClient.vidLocations.get(vid); + if (locations == null) { + FilerProto.LookupVolumeRequest.Builder lookupRequest = FilerProto.LookupVolumeRequest.newBuilder(); + lookupRequest.addVolumeIds(vid); + FilerProto.LookupVolumeResponse lookupResponse = filerGrpcClient + .getBlockingStub().lookupVolume(lookupRequest.build()); + locations = lookupResponse.getLocationsMapMap().get(vid); + filerGrpcClient.vidLocations.put(vid, locations); + LOG.warn("fetchChunk vid:{} locations:{}", vid, locations); + } SeaweedRead.ChunkView chunkView = new SeaweedRead.ChunkView( FilerClient.toFileId(chunk.getFid()), // avoid deprecated chunk.getFileId() diff --git a/other/java/client/src/main/java/seaweedfs/client/FilerGrpcClient.java b/other/java/client/src/main/java/seaweedfs/client/FilerGrpcClient.java index 57b67f6b0..1a719f3c0 100644 --- a/other/java/client/src/main/java/seaweedfs/client/FilerGrpcClient.java +++ b/other/java/client/src/main/java/seaweedfs/client/FilerGrpcClient.java @@ -9,6 +9,8 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import javax.net.ssl.SSLException; +import java.util.Map; +import java.util.HashMap; import java.util.concurrent.TimeUnit; public class FilerGrpcClient { @@ -24,6 +26,7 @@ public class FilerGrpcClient { } } + public final Map vidLocations = new HashMap<>(); private final ManagedChannel channel; private final SeaweedFilerGrpc.SeaweedFilerBlockingStub blockingStub; private final SeaweedFilerGrpc.SeaweedFilerStub asyncStub; -- cgit v1.2.3 From 5080bc1d6964cc71044333edc7ee36c1c1f06adb Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Wed, 29 Jul 2020 18:26:26 -0700 Subject: debug --- other/java/client/src/main/java/seaweedfs/client/FileChunkManifest.java | 2 +- other/java/client/src/main/java/seaweedfs/client/SeaweedRead.java | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) (limited to 'other/java/client/src') diff --git a/other/java/client/src/main/java/seaweedfs/client/FileChunkManifest.java b/other/java/client/src/main/java/seaweedfs/client/FileChunkManifest.java index a15671f46..28c2f47fc 100644 --- a/other/java/client/src/main/java/seaweedfs/client/FileChunkManifest.java +++ b/other/java/client/src/main/java/seaweedfs/client/FileChunkManifest.java @@ -59,7 +59,7 @@ public class FileChunkManifest { .getBlockingStub().lookupVolume(lookupRequest.build()); locations = lookupResponse.getLocationsMapMap().get(vid); filerGrpcClient.vidLocations.put(vid, locations); - LOG.warn("fetchChunk vid:{} locations:{}", vid, locations); + LOG.debug("fetchChunk vid:{} locations:{}", vid, locations); } SeaweedRead.ChunkView chunkView = new SeaweedRead.ChunkView( diff --git a/other/java/client/src/main/java/seaweedfs/client/SeaweedRead.java b/other/java/client/src/main/java/seaweedfs/client/SeaweedRead.java index af3aac0f5..05457ed48 100644 --- a/other/java/client/src/main/java/seaweedfs/client/SeaweedRead.java +++ b/other/java/client/src/main/java/seaweedfs/client/SeaweedRead.java @@ -108,6 +108,8 @@ public class SeaweedRead { } } + LOG.debug("doFetchFullChunkData fid:{} chunkData.length:{}", chunkView.fileId, data.length); + return data; } -- cgit v1.2.3