From 13bfe5deefcb83a5a89f9f15e7a15c29f97b1730 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Sun, 16 Aug 2020 21:14:39 -0700 Subject: same logic for reading random access files from Go --- .../java/hdfs3/src/main/java/seaweed/hdfs/SeaweedFileSystemStore.java | 4 ++-- other/java/hdfs3/src/main/java/seaweed/hdfs/SeaweedInputStream.java | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) (limited to 'other/java/hdfs3') diff --git a/other/java/hdfs3/src/main/java/seaweed/hdfs/SeaweedFileSystemStore.java b/other/java/hdfs3/src/main/java/seaweed/hdfs/SeaweedFileSystemStore.java index 0db6a1f49..53185367a 100644 --- a/other/java/hdfs3/src/main/java/seaweed/hdfs/SeaweedFileSystemStore.java +++ b/other/java/hdfs3/src/main/java/seaweed/hdfs/SeaweedFileSystemStore.java @@ -124,7 +124,7 @@ public class SeaweedFileSystemStore { private FileStatus doGetFileStatus(Path path, FilerProto.Entry entry) { FilerProto.FuseAttributes attributes = entry.getAttributes(); - long length = SeaweedRead.totalSize(entry.getChunksList()); + long length = SeaweedRead.fileSize(entry); boolean isDir = entry.getIsDirectory(); int block_replication = 1; int blocksize = 512; @@ -185,7 +185,7 @@ public class SeaweedFileSystemStore { entry.mergeFrom(existingEntry); entry.getAttributesBuilder().setMtime(now); LOG.debug("createFile merged entry path:{} entry:{} from:{}", path, entry, existingEntry); - writePosition = SeaweedRead.totalSize(existingEntry.getChunksList()); + writePosition = SeaweedRead.fileSize(existingEntry); replication = existingEntry.getAttributes().getReplication(); } } diff --git a/other/java/hdfs3/src/main/java/seaweed/hdfs/SeaweedInputStream.java b/other/java/hdfs3/src/main/java/seaweed/hdfs/SeaweedInputStream.java index 6b3c72f7d..36c0766a4 100644 --- a/other/java/hdfs3/src/main/java/seaweed/hdfs/SeaweedInputStream.java +++ b/other/java/hdfs3/src/main/java/seaweed/hdfs/SeaweedInputStream.java @@ -41,7 +41,7 @@ public class SeaweedInputStream extends FSInputStream { this.statistics = statistics; this.path = path; this.entry = entry; - this.contentLength = SeaweedRead.totalSize(entry.getChunksList()); + this.contentLength = SeaweedRead.fileSize(entry); this.bufferSize = bufferSize; this.visibleIntervalList = SeaweedRead.nonOverlappingVisibleIntervals(filerGrpcClient, entry.getChunksList()); @@ -87,7 +87,7 @@ public class SeaweedInputStream extends FSInputStream { throw new IllegalArgumentException("requested read length is more than will fit after requested offset in buffer"); } - long bytesRead = SeaweedRead.read(this.filerGrpcClient, this.visibleIntervalList, this.position, b, off, len); + long bytesRead = SeaweedRead.read(this.filerGrpcClient, this.visibleIntervalList, this.position, b, off, len, SeaweedRead.fileSize(entry)); if (bytesRead > Integer.MAX_VALUE) { throw new IOException("Unexpected Content-Length"); } -- cgit v1.2.3 From f5d6b8b777f8f416c4a33ea86a8be8c578c9f7a4 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Sun, 16 Aug 2020 21:17:03 -0700 Subject: Hadoop HCFS 1.4.6 random access file --- other/java/hdfs3/dependency-reduced-pom.xml | 2 +- other/java/hdfs3/pom.xml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) (limited to 'other/java/hdfs3') diff --git a/other/java/hdfs3/dependency-reduced-pom.xml b/other/java/hdfs3/dependency-reduced-pom.xml index 5f1e278f8..3564ef33d 100644 --- a/other/java/hdfs3/dependency-reduced-pom.xml +++ b/other/java/hdfs3/dependency-reduced-pom.xml @@ -127,7 +127,7 @@ - 1.4.5 + 1.4.6 3.1.1 diff --git a/other/java/hdfs3/pom.xml b/other/java/hdfs3/pom.xml index b1bd27f74..afb6ef309 100644 --- a/other/java/hdfs3/pom.xml +++ b/other/java/hdfs3/pom.xml @@ -5,7 +5,7 @@ 4.0.0 - 1.4.5 + 1.4.6 3.1.1 -- cgit v1.2.3 From 48ed7f8a328ecaa40a21cac0a72c3711ca196a19 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Sun, 16 Aug 2020 21:25:24 -0700 Subject: generated files --- other/java/hdfs3/dependency-reduced-pom.xml | 182 ++++++++++++++++++++++++++++ 1 file changed, 182 insertions(+) (limited to 'other/java/hdfs3') diff --git a/other/java/hdfs3/dependency-reduced-pom.xml b/other/java/hdfs3/dependency-reduced-pom.xml index 3564ef33d..25de9dfc0 100644 --- a/other/java/hdfs3/dependency-reduced-pom.xml +++ b/other/java/hdfs3/dependency-reduced-pom.xml @@ -120,6 +120,188 @@ + + + org.apache.hadoop + hadoop-client + 3.1.1 + provided + + + hadoop-hdfs-client + org.apache.hadoop + + + hadoop-yarn-api + org.apache.hadoop + + + hadoop-yarn-client + org.apache.hadoop + + + hadoop-mapreduce-client-core + org.apache.hadoop + + + hadoop-mapreduce-client-jobclient + org.apache.hadoop + + + hadoop-annotations + org.apache.hadoop + + + + + org.apache.hadoop + hadoop-common + 3.1.1 + provided + + + commons-cli + commons-cli + + + commons-math3 + org.apache.commons + + + commons-io + commons-io + + + commons-net + commons-net + + + commons-collections + commons-collections + + + javax.servlet-api + javax.servlet + + + jetty-server + org.eclipse.jetty + + + jetty-util + org.eclipse.jetty + + + jetty-servlet + org.eclipse.jetty + + + jetty-webapp + org.eclipse.jetty + + + jsp-api + javax.servlet.jsp + + + jersey-core + com.sun.jersey + + + jersey-servlet + com.sun.jersey + + + jersey-json + com.sun.jersey + + + jersey-server + com.sun.jersey + + + log4j + log4j + + + commons-lang + commons-lang + + + commons-beanutils + commons-beanutils + + + commons-configuration2 + org.apache.commons + + + commons-lang3 + org.apache.commons + + + slf4j-log4j12 + org.slf4j + + + avro + org.apache.avro + + + re2j + com.google.re2j + + + hadoop-auth + org.apache.hadoop + + + jsch + com.jcraft + + + curator-client + org.apache.curator + + + curator-recipes + org.apache.curator + + + htrace-core4 + org.apache.htrace + + + zookeeper + org.apache.zookeeper + + + commons-compress + org.apache.commons + + + kerb-simplekdc + org.apache.kerby + + + jackson-databind + com.fasterxml.jackson.core + + + stax2-api + org.codehaus.woodstox + + + woodstox-core + com.fasterxml.woodstox + + + hadoop-annotations + org.apache.hadoop + + + + ossrh -- cgit v1.2.3