From 3abd74b1d7d1685ca4ddcd741ddf3ac0585b848a Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Fri, 17 Jul 2020 15:46:16 -0700 Subject: 1.3.8 --- .../java/seaweed/hdfs/SeaweedOutputStream.java | 35 ++++++++++++++++++---- 1 file changed, 29 insertions(+), 6 deletions(-) (limited to 'other/java/hdfs3/src') diff --git a/other/java/hdfs3/src/main/java/seaweed/hdfs/SeaweedOutputStream.java b/other/java/hdfs3/src/main/java/seaweed/hdfs/SeaweedOutputStream.java index 9ea26776b..d5c8ae521 100644 --- a/other/java/hdfs3/src/main/java/seaweed/hdfs/SeaweedOutputStream.java +++ b/other/java/hdfs3/src/main/java/seaweed/hdfs/SeaweedOutputStream.java @@ -216,12 +216,35 @@ public class SeaweedOutputStream extends OutputStream implements Syncable, Strea return; } - buffer.flip(); - int bytesLength = buffer.limit() - buffer.position(); - SeaweedWrite.writeData(entry, replication, filerGrpcClient, position, buffer.array(), buffer.position(), buffer.limit()); - // System.out.println(path + " saved [" + (position) + "," + ((position) + bytesLength) + ")"); - position += bytesLength; - buffer.clear(); + position += submitWriteBufferToService(buffer, position); + + buffer = ByteBufferPool.request(bufferSize); + + } + + private synchronized int submitWriteBufferToService(final ByteBuffer bufferToWrite, final long writePosition) throws IOException { + + bufferToWrite.flip(); + int bytesLength = bufferToWrite.limit() - bufferToWrite.position(); + + if (threadExecutor.getQueue().size() >= maxConcurrentRequestCount * 2) { + waitForTaskToComplete(); + } + final Future job = completionService.submit(() -> { + System.out.println(path + " is going to save [" + (writePosition) + "," + ((writePosition) + bytesLength) + ")"); + SeaweedWrite.writeData(entry, replication, filerGrpcClient, writePosition, bufferToWrite.array(), bufferToWrite.position(), bufferToWrite.limit()); + System.out.println(path + " saved [" + (writePosition) + "," + ((writePosition) + bytesLength) + ")"); + bufferToWrite.clear(); + ByteBufferPool.release(bufferToWrite); + return null; + }); + + writeOperations.add(new WriteOperation(job, writePosition, bytesLength)); + + // Try to shrink the queue + shrinkWriteOperationQueue(); + + return bytesLength; } -- cgit v1.2.3 From f90d2c93c9e34997a8e76aeefb438ec06b1cd093 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Fri, 17 Jul 2020 16:11:57 -0700 Subject: 1.3.9 remove logs --- other/java/hdfs3/src/main/java/seaweed/hdfs/SeaweedOutputStream.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'other/java/hdfs3/src') diff --git a/other/java/hdfs3/src/main/java/seaweed/hdfs/SeaweedOutputStream.java b/other/java/hdfs3/src/main/java/seaweed/hdfs/SeaweedOutputStream.java index d5c8ae521..c602a0d81 100644 --- a/other/java/hdfs3/src/main/java/seaweed/hdfs/SeaweedOutputStream.java +++ b/other/java/hdfs3/src/main/java/seaweed/hdfs/SeaweedOutputStream.java @@ -231,9 +231,9 @@ public class SeaweedOutputStream extends OutputStream implements Syncable, Strea waitForTaskToComplete(); } final Future job = completionService.submit(() -> { - System.out.println(path + " is going to save [" + (writePosition) + "," + ((writePosition) + bytesLength) + ")"); + // System.out.println(path + " is going to save [" + (writePosition) + "," + ((writePosition) + bytesLength) + ")"); SeaweedWrite.writeData(entry, replication, filerGrpcClient, writePosition, bufferToWrite.array(), bufferToWrite.position(), bufferToWrite.limit()); - System.out.println(path + " saved [" + (writePosition) + "," + ((writePosition) + bytesLength) + ")"); + // System.out.println(path + " saved [" + (writePosition) + "," + ((writePosition) + bytesLength) + ")"); bufferToWrite.clear(); ByteBufferPool.release(bufferToWrite); return null; -- cgit v1.2.3