diff options
Diffstat (limited to 'other/java/hdfs3')
| -rw-r--r-- | other/java/hdfs3/dependency-reduced-pom.xml | 2 | ||||
| -rw-r--r-- | other/java/hdfs3/pom.xml | 6 | ||||
| -rw-r--r-- | other/java/hdfs3/src/main/java/seaweed/hdfs/SeaweedOutputStream.java | 35 |
3 files changed, 33 insertions, 10 deletions
diff --git a/other/java/hdfs3/dependency-reduced-pom.xml b/other/java/hdfs3/dependency-reduced-pom.xml index 00e236aa2..51c4a4473 100644 --- a/other/java/hdfs3/dependency-reduced-pom.xml +++ b/other/java/hdfs3/dependency-reduced-pom.xml @@ -127,7 +127,7 @@ </snapshotRepository>
</distributionManagement>
<properties>
- <seaweedfs.client.version>1.3.6</seaweedfs.client.version>
+ <seaweedfs.client.version>1.3.8</seaweedfs.client.version>
<hadoop.version>3.1.1</hadoop.version>
</properties>
</project>
diff --git a/other/java/hdfs3/pom.xml b/other/java/hdfs3/pom.xml index a03068a48..cf7a6a632 100644 --- a/other/java/hdfs3/pom.xml +++ b/other/java/hdfs3/pom.xml @@ -5,7 +5,7 @@ <modelVersion>4.0.0</modelVersion> <properties> - <seaweedfs.client.version>1.3.6</seaweedfs.client.version> + <seaweedfs.client.version>1.3.8</seaweedfs.client.version> <hadoop.version>3.1.1</hadoop.version> </properties> @@ -31,8 +31,8 @@ <groupId>org.apache.maven.plugins</groupId> <artifactId>maven-compiler-plugin</artifactId> <configuration> - <source>7</source> - <target>7</target> + <source>8</source> + <target>8</target> </configuration> </plugin> <plugin> diff --git a/other/java/hdfs3/src/main/java/seaweed/hdfs/SeaweedOutputStream.java b/other/java/hdfs3/src/main/java/seaweed/hdfs/SeaweedOutputStream.java index 9ea26776b..d5c8ae521 100644 --- a/other/java/hdfs3/src/main/java/seaweed/hdfs/SeaweedOutputStream.java +++ b/other/java/hdfs3/src/main/java/seaweed/hdfs/SeaweedOutputStream.java @@ -216,12 +216,35 @@ public class SeaweedOutputStream extends OutputStream implements Syncable, Strea return; } - buffer.flip(); - int bytesLength = buffer.limit() - buffer.position(); - SeaweedWrite.writeData(entry, replication, filerGrpcClient, position, buffer.array(), buffer.position(), buffer.limit()); - // System.out.println(path + " saved [" + (position) + "," + ((position) + bytesLength) + ")"); - position += bytesLength; - buffer.clear(); + position += submitWriteBufferToService(buffer, position); + + buffer = ByteBufferPool.request(bufferSize); + + } + + private synchronized int submitWriteBufferToService(final ByteBuffer bufferToWrite, final long writePosition) throws IOException { + + bufferToWrite.flip(); + int bytesLength = bufferToWrite.limit() - bufferToWrite.position(); + + if (threadExecutor.getQueue().size() >= maxConcurrentRequestCount * 2) { + waitForTaskToComplete(); + } + final Future<Void> job = completionService.submit(() -> { + System.out.println(path + " is going to save [" + (writePosition) + "," + ((writePosition) + bytesLength) + ")"); + SeaweedWrite.writeData(entry, replication, filerGrpcClient, writePosition, bufferToWrite.array(), bufferToWrite.position(), bufferToWrite.limit()); + System.out.println(path + " saved [" + (writePosition) + "," + ((writePosition) + bytesLength) + ")"); + bufferToWrite.clear(); + ByteBufferPool.release(bufferToWrite); + return null; + }); + + writeOperations.add(new WriteOperation(job, writePosition, bytesLength)); + + // Try to shrink the queue + shrinkWriteOperationQueue(); + + return bytesLength; } |
