Merge pull request #1 from chrislusf/master

catch up

5 files changed, 208 insertions, 22 deletions

diff --git a/other/java/hdfs3/dependency-reduced-pom.xml b/other/java/hdfs3/dependency-reduced-pom.xml
index 0b01074c3..691be547e 100644
--- a/other/java/hdfs3/dependency-reduced-pom.xml
+++ b/other/java/hdfs3/dependency-reduced-pom.xml
@@ -120,6 +120,188 @@
       </plugin>

     </plugins>

   </build>

+  <dependencies>

+    <dependency>

+      <groupId>org.apache.hadoop</groupId>

+      <artifactId>hadoop-client</artifactId>

+      <version>3.1.1</version>

+      <scope>provided</scope>

+      <exclusions>

+        <exclusion>

+          <artifactId>hadoop-hdfs-client</artifactId>

+          <groupId>org.apache.hadoop</groupId>

+        </exclusion>

+        <exclusion>

+          <artifactId>hadoop-yarn-api</artifactId>

+          <groupId>org.apache.hadoop</groupId>

+        </exclusion>

+        <exclusion>

+          <artifactId>hadoop-yarn-client</artifactId>

+          <groupId>org.apache.hadoop</groupId>

+        </exclusion>

+        <exclusion>

+          <artifactId>hadoop-mapreduce-client-core</artifactId>

+          <groupId>org.apache.hadoop</groupId>

+        </exclusion>

+        <exclusion>

+          <artifactId>hadoop-mapreduce-client-jobclient</artifactId>

+          <groupId>org.apache.hadoop</groupId>

+        </exclusion>

+        <exclusion>

+          <artifactId>hadoop-annotations</artifactId>

+          <groupId>org.apache.hadoop</groupId>

+        </exclusion>

+      </exclusions>

+    </dependency>

+    <dependency>

+      <groupId>org.apache.hadoop</groupId>

+      <artifactId>hadoop-common</artifactId>

+      <version>3.1.1</version>

+      <scope>provided</scope>

+      <exclusions>

+        <exclusion>

+          <artifactId>commons-cli</artifactId>

+          <groupId>commons-cli</groupId>

+        </exclusion>

+        <exclusion>

+          <artifactId>commons-math3</artifactId>

+          <groupId>org.apache.commons</groupId>

+        </exclusion>

+        <exclusion>

+          <artifactId>commons-io</artifactId>

+          <groupId>commons-io</groupId>

+        </exclusion>

+        <exclusion>

+          <artifactId>commons-net</artifactId>

+          <groupId>commons-net</groupId>

+        </exclusion>

+        <exclusion>

+          <artifactId>commons-collections</artifactId>

+          <groupId>commons-collections</groupId>

+        </exclusion>

+        <exclusion>

+          <artifactId>javax.servlet-api</artifactId>

+          <groupId>javax.servlet</groupId>

+        </exclusion>

+        <exclusion>

+          <artifactId>jetty-server</artifactId>

+          <groupId>org.eclipse.jetty</groupId>

+        </exclusion>

+        <exclusion>

+          <artifactId>jetty-util</artifactId>

+          <groupId>org.eclipse.jetty</groupId>

+        </exclusion>

+        <exclusion>

+          <artifactId>jetty-servlet</artifactId>

+          <groupId>org.eclipse.jetty</groupId>

+        </exclusion>

+        <exclusion>

+          <artifactId>jetty-webapp</artifactId>

+          <groupId>org.eclipse.jetty</groupId>

+        </exclusion>

+        <exclusion>

+          <artifactId>jsp-api</artifactId>

+          <groupId>javax.servlet.jsp</groupId>

+        </exclusion>

+        <exclusion>

+          <artifactId>jersey-core</artifactId>

+          <groupId>com.sun.jersey</groupId>

+        </exclusion>

+        <exclusion>

+          <artifactId>jersey-servlet</artifactId>

+          <groupId>com.sun.jersey</groupId>

+        </exclusion>

+        <exclusion>

+          <artifactId>jersey-json</artifactId>

+          <groupId>com.sun.jersey</groupId>

+        </exclusion>

+        <exclusion>

+          <artifactId>jersey-server</artifactId>

+          <groupId>com.sun.jersey</groupId>

+        </exclusion>

+        <exclusion>

+          <artifactId>log4j</artifactId>

+          <groupId>log4j</groupId>

+        </exclusion>

+        <exclusion>

+          <artifactId>commons-lang</artifactId>

+          <groupId>commons-lang</groupId>

+        </exclusion>

+        <exclusion>

+          <artifactId>commons-beanutils</artifactId>

+          <groupId>commons-beanutils</groupId>

+        </exclusion>

+        <exclusion>

+          <artifactId>commons-configuration2</artifactId>

+          <groupId>org.apache.commons</groupId>

+        </exclusion>

+        <exclusion>

+          <artifactId>commons-lang3</artifactId>

+          <groupId>org.apache.commons</groupId>

+        </exclusion>

+        <exclusion>

+          <artifactId>slf4j-log4j12</artifactId>

+          <groupId>org.slf4j</groupId>

+        </exclusion>

+        <exclusion>

+          <artifactId>avro</artifactId>

+          <groupId>org.apache.avro</groupId>

+        </exclusion>

+        <exclusion>

+          <artifactId>re2j</artifactId>

+          <groupId>com.google.re2j</groupId>

+        </exclusion>

+        <exclusion>

+          <artifactId>hadoop-auth</artifactId>

+          <groupId>org.apache.hadoop</groupId>

+        </exclusion>

+        <exclusion>

+          <artifactId>jsch</artifactId>

+          <groupId>com.jcraft</groupId>

+        </exclusion>

+        <exclusion>

+          <artifactId>curator-client</artifactId>

+          <groupId>org.apache.curator</groupId>

+        </exclusion>

+        <exclusion>

+          <artifactId>curator-recipes</artifactId>

+          <groupId>org.apache.curator</groupId>

+        </exclusion>

+        <exclusion>

+          <artifactId>htrace-core4</artifactId>

+          <groupId>org.apache.htrace</groupId>

+        </exclusion>

+        <exclusion>

+          <artifactId>zookeeper</artifactId>

+          <groupId>org.apache.zookeeper</groupId>

+        </exclusion>

+        <exclusion>

+          <artifactId>commons-compress</artifactId>

+          <groupId>org.apache.commons</groupId>

+        </exclusion>

+        <exclusion>

+          <artifactId>kerb-simplekdc</artifactId>

+          <groupId>org.apache.kerby</groupId>

+        </exclusion>

+        <exclusion>

+          <artifactId>jackson-databind</artifactId>

+          <groupId>com.fasterxml.jackson.core</groupId>

+        </exclusion>

+        <exclusion>

+          <artifactId>stax2-api</artifactId>

+          <groupId>org.codehaus.woodstox</groupId>

+        </exclusion>

+        <exclusion>

+          <artifactId>woodstox-core</artifactId>

+          <groupId>com.fasterxml.woodstox</groupId>

+        </exclusion>

+        <exclusion>

+          <artifactId>hadoop-annotations</artifactId>

+          <groupId>org.apache.hadoop</groupId>

+        </exclusion>

+      </exclusions>

+    </dependency>

+  </dependencies>

   <distributionManagement>

     <snapshotRepository>

       <id>ossrh</id>

@@ -127,7 +309,7 @@
     </snapshotRepository>

   </distributionManagement>

   <properties>

-    <seaweedfs.client.version>1.4.1</seaweedfs.client.version>

+    <seaweedfs.client.version>1.4.7</seaweedfs.client.version>

     <hadoop.version>3.1.1</hadoop.version>

   </properties>

 </project>

diff --git a/other/java/hdfs3/pom.xml b/other/java/hdfs3/pom.xml
index 025843d3c..8e8ec1958 100644
--- a/other/java/hdfs3/pom.xml
+++ b/other/java/hdfs3/pom.xml
@@ -5,7 +5,7 @@
     <modelVersion>4.0.0</modelVersion>
 
     <properties>
-        <seaweedfs.client.version>1.4.1</seaweedfs.client.version>
+        <seaweedfs.client.version>1.4.7</seaweedfs.client.version>
         <hadoop.version>3.1.1</hadoop.version>
     </properties>
 
@@ -147,6 +147,7 @@
             <groupId>org.apache.hadoop</groupId>
             <artifactId>hadoop-client</artifactId>
             <version>${hadoop.version}</version>
+            <scope>provided</scope>
         </dependency>
         <dependency>
             <groupId>com.github.chrislusf</groupId>
@@ -157,6 +158,7 @@
             <groupId>org.apache.hadoop</groupId>
             <artifactId>hadoop-common</artifactId>
             <version>${hadoop.version}</version>
+            <scope>provided</scope>
         </dependency>
     </dependencies>
 
diff --git a/other/java/hdfs3/src/main/java/seaweed/hdfs/SeaweedFileSystem.java b/other/java/hdfs3/src/main/java/seaweed/hdfs/SeaweedFileSystem.java
index 2341d335d..6551548fa 100644
--- a/other/java/hdfs3/src/main/java/seaweed/hdfs/SeaweedFileSystem.java
+++ b/other/java/hdfs3/src/main/java/seaweed/hdfs/SeaweedFileSystem.java
@@ -5,7 +5,6 @@ import org.apache.hadoop.fs.*;
 import org.apache.hadoop.fs.permission.AclEntry;
 import org.apache.hadoop.fs.permission.AclStatus;
 import org.apache.hadoop.fs.permission.FsPermission;
-import org.apache.hadoop.security.AccessControlException;
 import org.apache.hadoop.security.UserGroupInformation;
 import org.apache.hadoop.util.Progressable;
 import org.slf4j.Logger;
@@ -14,20 +13,19 @@ import seaweedfs.client.FilerProto;
 
 import java.io.FileNotFoundException;
 import java.io.IOException;
-import java.io.InputStream;
 import java.io.OutputStream;
 import java.net.URI;
 import java.util.EnumSet;
 import java.util.List;
 import java.util.Map;
 
-import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.IO_FILE_BUFFER_SIZE_KEY;
-
 public class SeaweedFileSystem extends FileSystem {
 
-    public static final int FS_SEAWEED_DEFAULT_PORT = 8888;
     public static final String FS_SEAWEED_FILER_HOST = "fs.seaweed.filer.host";
     public static final String FS_SEAWEED_FILER_PORT = "fs.seaweed.filer.port";
+    public static final int FS_SEAWEED_DEFAULT_PORT = 8888;
+    public static final String FS_SEAWEED_BUFFER_SIZE = "fs.seaweed.buffer.size";
+    public static final int FS_SEAWEED_DEFAULT_BUFFER_SIZE = 4 * 1024 * 1024;
 
     private static final Logger LOG = LoggerFactory.getLogger(SeaweedFileSystem.class);
 
@@ -75,8 +73,9 @@ public class SeaweedFileSystem extends FileSystem {
         path = qualify(path);
 
         try {
-            InputStream inputStream = seaweedFileSystemStore.openFileForRead(path, statistics, bufferSize);
-            return new FSDataInputStream(inputStream);
+            int seaweedBufferSize = this.getConf().getInt(FS_SEAWEED_BUFFER_SIZE, FS_SEAWEED_DEFAULT_BUFFER_SIZE);
+            FSInputStream inputStream = seaweedFileSystemStore.openFileForRead(path, statistics, seaweedBufferSize);
+            return new FSDataInputStream(new BufferedFSInputStream(inputStream, 4 * seaweedBufferSize));
         } catch (Exception ex) {
             LOG.warn("open path: {} bufferSize:{}", path, bufferSize, ex);
             return null;
@@ -93,7 +92,8 @@ public class SeaweedFileSystem extends FileSystem {
 
         try {
             String replicaPlacement = String.format("%03d", replication - 1);
-            OutputStream outputStream = seaweedFileSystemStore.createFile(path, overwrite, permission, bufferSize, replicaPlacement);
+            int seaweedBufferSize = this.getConf().getInt(FS_SEAWEED_BUFFER_SIZE, FS_SEAWEED_DEFAULT_BUFFER_SIZE);
+            OutputStream outputStream = seaweedFileSystemStore.createFile(path, overwrite, permission, seaweedBufferSize, replicaPlacement);
             return new FSDataOutputStream(outputStream, statistics);
         } catch (Exception ex) {
             LOG.warn("create path: {} bufferSize:{} blockSize:{}", path, bufferSize, blockSize, ex);
@@ -103,8 +103,9 @@ public class SeaweedFileSystem extends FileSystem {
 
     /**
      * {@inheritDoc}
+     *
      * @throws FileNotFoundException if the parent directory is not present -or
-     * is not a directory.
+     *                               is not a directory.
      */
     @Override
     public FSDataOutputStream createNonRecursive(Path path,
@@ -121,9 +122,10 @@ public class SeaweedFileSystem extends FileSystem {
                 throw new FileAlreadyExistsException("Not a directory: " + parent);
             }
         }
+        int seaweedBufferSize = this.getConf().getInt(FS_SEAWEED_BUFFER_SIZE, FS_SEAWEED_DEFAULT_BUFFER_SIZE);
         return create(path, permission,
                 flags.contains(CreateFlag.OVERWRITE), bufferSize,
-                replication, blockSize, progress);
+                replication, seaweedBufferSize, progress);
     }
 
     @Override
@@ -133,7 +135,8 @@ public class SeaweedFileSystem extends FileSystem {
 
         path = qualify(path);
         try {
-            OutputStream outputStream = seaweedFileSystemStore.createFile(path, false, null, bufferSize, "");
+            int seaweedBufferSize = this.getConf().getInt(FS_SEAWEED_BUFFER_SIZE, FS_SEAWEED_DEFAULT_BUFFER_SIZE);
+            OutputStream outputStream = seaweedFileSystemStore.createFile(path, false, null, seaweedBufferSize, "");
             return new FSDataOutputStream(outputStream, statistics);
         } catch (Exception ex) {
             LOG.warn("append path: {} bufferSize:{}", path, bufferSize, ex);
@@ -338,9 +341,7 @@ public class SeaweedFileSystem extends FileSystem {
 
     @Override
     public void createSymlink(final Path target, final Path link,
-                              final boolean createParent) throws AccessControlException,
-            FileAlreadyExistsException, FileNotFoundException,
-            ParentNotDirectoryException, UnsupportedFileSystemException,
+                              final boolean createParent) throws
             IOException {
         // Supporting filesystems should override this method
         throw new UnsupportedOperationException(
diff --git a/other/java/hdfs3/src/main/java/seaweed/hdfs/SeaweedFileSystemStore.java b/other/java/hdfs3/src/main/java/seaweed/hdfs/SeaweedFileSystemStore.java
index d9c6d6f0d..53185367a 100644
--- a/other/java/hdfs3/src/main/java/seaweed/hdfs/SeaweedFileSystemStore.java
+++ b/other/java/hdfs3/src/main/java/seaweed/hdfs/SeaweedFileSystemStore.java
@@ -1,5 +1,6 @@
 package seaweed.hdfs;
 
+import org.apache.hadoop.fs.FSInputStream;
 import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
@@ -123,7 +124,7 @@ public class SeaweedFileSystemStore {
 
     private FileStatus doGetFileStatus(Path path, FilerProto.Entry entry) {
         FilerProto.FuseAttributes attributes = entry.getAttributes();
-        long length = SeaweedRead.totalSize(entry.getChunksList());
+        long length = SeaweedRead.fileSize(entry);
         boolean isDir = entry.getIsDirectory();
         int block_replication = 1;
         int blocksize = 512;
@@ -184,7 +185,7 @@ public class SeaweedFileSystemStore {
                 entry.mergeFrom(existingEntry);
                 entry.getAttributesBuilder().setMtime(now);
                 LOG.debug("createFile merged entry path:{} entry:{} from:{}", path, entry, existingEntry);
-                writePosition = SeaweedRead.totalSize(existingEntry.getChunksList());
+                writePosition = SeaweedRead.fileSize(existingEntry);
                 replication = existingEntry.getAttributes().getReplication();
             }
         }
@@ -207,8 +208,8 @@ public class SeaweedFileSystemStore {
 
     }
 
-    public InputStream openFileForRead(final Path path, FileSystem.Statistics statistics,
-                                       int bufferSize) throws IOException {
+    public FSInputStream openFileForRead(final Path path, FileSystem.Statistics statistics,
+                                         int bufferSize) throws IOException {
 
         LOG.debug("openFileForRead path:{} bufferSize:{}", path, bufferSize);
 
diff --git a/other/java/hdfs3/src/main/java/seaweed/hdfs/SeaweedInputStream.java b/other/java/hdfs3/src/main/java/seaweed/hdfs/SeaweedInputStream.java
index 6b3c72f7d..36c0766a4 100644
--- a/other/java/hdfs3/src/main/java/seaweed/hdfs/SeaweedInputStream.java
+++ b/other/java/hdfs3/src/main/java/seaweed/hdfs/SeaweedInputStream.java
@@ -41,7 +41,7 @@ public class SeaweedInputStream extends FSInputStream {
         this.statistics = statistics;
         this.path = path;
         this.entry = entry;
-        this.contentLength = SeaweedRead.totalSize(entry.getChunksList());
+        this.contentLength = SeaweedRead.fileSize(entry);
         this.bufferSize = bufferSize;
 
         this.visibleIntervalList = SeaweedRead.nonOverlappingVisibleIntervals(filerGrpcClient, entry.getChunksList());
@@ -87,7 +87,7 @@ public class SeaweedInputStream extends FSInputStream {
             throw new IllegalArgumentException("requested read length is more than will fit after requested offset in buffer");
         }
 
-        long bytesRead = SeaweedRead.read(this.filerGrpcClient, this.visibleIntervalList, this.position, b, off, len);
+        long bytesRead = SeaweedRead.read(this.filerGrpcClient, this.visibleIntervalList, this.position, b, off, len, SeaweedRead.fileSize(entry));
         if (bytesRead > Integer.MAX_VALUE) {
             throw new IOException("Unexpected Content-Length");
         }