HCFS: use latest grpc versions, separate hadoop2 and hadoop3

12 files changed, 126 insertions, 85 deletions

diff --git a/other/java/client/pom.xml b/other/java/client/pom.xml
index 5882c726d..c37b9aab8 100644
--- a/other/java/client/pom.xml
+++ b/other/java/client/pom.xml
@@ -4,7 +4,7 @@
 
     <groupId>com.github.chrislusf</groupId>
     <artifactId>seaweedfs-client</artifactId>
-    <version>1.1.0</version>
+    <version>1.1.3</version>
 
     <parent>
         <groupId>org.sonatype.oss</groupId>
@@ -13,12 +13,18 @@
     </parent>
 
     <properties>
-        <protobuf.version>3.5.1</protobuf.version>
-        <grpc.version>1.16.1</grpc.version>
-        <guava.version>26.0-jre</guava.version>
+        <protobuf.version>3.9.1</protobuf.version>
+        <!-- follow https://github.com/grpc/grpc-java -->
+        <grpc.version>1.23.0</grpc.version>
+        <guava.version>28.0-jre</guava.version>
     </properties>
 
     <dependencies>
+        <dependency>
+            <groupId>com.moandjiezana.toml</groupId>
+            <artifactId>toml4j</artifactId>
+            <version>0.7.2</version>
+        </dependency>
         <!-- https://mvnrepository.com/artifact/com.google.protobuf/protobuf-java -->
         <dependency>
             <groupId>com.google.protobuf</groupId>
@@ -74,7 +80,7 @@
             <extension>
                 <groupId>kr.motd.maven</groupId>
                 <artifactId>os-maven-plugin</artifactId>
-                <version>1.5.0.Final</version>
+                <version>1.6.2</version>
             </extension>
         </extensions>
         <plugins>
@@ -89,9 +95,9 @@
             <plugin>
                 <groupId>org.xolstice.maven.plugins</groupId>
                 <artifactId>protobuf-maven-plugin</artifactId>
-                <version>0.5.1</version>
+                <version>0.6.1</version>
                 <configuration>
-                    <protocArtifact>com.google.protobuf:protoc:${protobuf.version}-1:exe:${os.detected.classifier}</protocArtifact>
+                    <protocArtifact>com.google.protobuf:protoc:${protobuf.version}:exe:${os.detected.classifier}</protocArtifact>
                     <pluginId>grpc-java</pluginId>
                     <pluginArtifact>io.grpc:protoc-gen-grpc-java:${grpc.version}:exe:${os.detected.classifier}</pluginArtifact>
                 </configuration>
diff --git a/other/java/client/src/main/java/seaweedfs/client/FilerGrpcClient.java b/other/java/client/src/main/java/seaweedfs/client/FilerGrpcClient.java
index c28c1dcf2..3626c76de 100644
--- a/other/java/client/src/main/java/seaweedfs/client/FilerGrpcClient.java
+++ b/other/java/client/src/main/java/seaweedfs/client/FilerGrpcClient.java
@@ -2,39 +2,46 @@ package seaweedfs.client;
 
 import io.grpc.ManagedChannel;
 import io.grpc.ManagedChannelBuilder;
-import io.grpc.netty.shaded.io.grpc.netty.GrpcSslContexts;
 import io.grpc.netty.shaded.io.grpc.netty.NegotiationType;
 import io.grpc.netty.shaded.io.grpc.netty.NettyChannelBuilder;
 import io.grpc.netty.shaded.io.netty.handler.ssl.SslContext;
-import io.grpc.netty.shaded.io.netty.handler.ssl.SslContextBuilder;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 import javax.net.ssl.SSLException;
-import java.io.File;
 import java.util.concurrent.TimeUnit;
-import java.util.logging.Logger;
 
 public class FilerGrpcClient {
 
-    private static final Logger logger = Logger.getLogger(FilerGrpcClient.class.getName());
+    private static final Logger logger = LoggerFactory.getLogger(FilerGrpcClient.class);
 
     private final ManagedChannel channel;
     private final SeaweedFilerGrpc.SeaweedFilerBlockingStub blockingStub;
     private final SeaweedFilerGrpc.SeaweedFilerStub asyncStub;
     private final SeaweedFilerGrpc.SeaweedFilerFutureStub futureStub;
 
+    static SslContext sslContext;
+
+    static {
+        try {
+            sslContext = FilerSslContext.loadSslContext();
+        } catch (SSLException e) {
+            logger.warn("failed to load ssl context", e);
+        }
+    }
 
     public FilerGrpcClient(String host, int grpcPort) {
-        this(ManagedChannelBuilder.forAddress(host, grpcPort).usePlaintext());
+        this(host, grpcPort, sslContext);
     }
 
-    public FilerGrpcClient(String host, int grpcPort,
-                           String caFilePath,
-                           String clientCertFilePath,
-                           String clientPrivateKeyFilePath) throws SSLException {
+    public FilerGrpcClient(String host, int grpcPort, SslContext sslContext) {
+
+        this(sslContext == null ?
+                ManagedChannelBuilder.forAddress(host, grpcPort).usePlaintext() :
+                NettyChannelBuilder.forAddress(host, grpcPort)
+                        .negotiationType(NegotiationType.TLS)
+                        .sslContext(sslContext));
 
-        this(NettyChannelBuilder.forAddress(host, grpcPort)
-                .negotiationType(NegotiationType.TLS)
-                .sslContext(buildSslContext(caFilePath,clientCertFilePath,clientPrivateKeyFilePath)));
     }
 
     public FilerGrpcClient(ManagedChannelBuilder<?> channelBuilder) {
@@ -60,17 +67,4 @@ public class FilerGrpcClient {
         return futureStub;
     }
 
-    private static SslContext buildSslContext(String trustCertCollectionFilePath,
-                                              String clientCertChainFilePath,
-                                              String clientPrivateKeyFilePath) throws SSLException {
-        SslContextBuilder builder = GrpcSslContexts.forClient();
-        if (trustCertCollectionFilePath != null) {
-            builder.trustManager(new File(trustCertCollectionFilePath));
-        }
-        if (clientCertChainFilePath != null && clientPrivateKeyFilePath != null) {
-            builder.keyManager(new File(clientCertChainFilePath), new File(clientPrivateKeyFilePath));
-        }
-        return builder.build();
-    }
-
 }
diff --git a/other/java/client/src/main/java/seaweedfs/client/FilerSslContext.java b/other/java/client/src/main/java/seaweedfs/client/FilerSslContext.java
new file mode 100644
index 000000000..5a88c1da3
--- /dev/null
+++ b/other/java/client/src/main/java/seaweedfs/client/FilerSslContext.java
@@ -0,0 +1,64 @@
+package seaweedfs.client;
+
+import com.google.common.base.Strings;
+import com.moandjiezana.toml.Toml;
+import io.grpc.netty.shaded.io.grpc.netty.GrpcSslContexts;
+import io.grpc.netty.shaded.io.netty.handler.ssl.SslContext;
+import io.grpc.netty.shaded.io.netty.handler.ssl.SslContextBuilder;
+import io.grpc.netty.shaded.io.netty.handler.ssl.util.InsecureTrustManagerFactory;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import javax.net.ssl.SSLException;
+import java.io.File;
+
+public class FilerSslContext {
+
+    private static final Logger logger = LoggerFactory.getLogger(FilerSslContext.class);
+
+    public static SslContext loadSslContext() throws SSLException {
+        String securityFileName = "security.toml";
+        String home = System.getProperty("user.home");
+        File f1 = new File("./"+securityFileName);
+        File f2 = new File(home + "/.seaweedfs/"+securityFileName);
+        File f3 = new File(home + "/etc/seaweedfs/"+securityFileName);
+
+        File securityFile = f1.exists()? f1 : f2.exists() ? f2 : f3.exists()? f3 : null;
+
+        if (securityFile==null){
+            return null;
+        }
+
+        Toml toml = new Toml().read(securityFile);
+        logger.debug("reading ssl setup from {}", securityFile);
+
+        String trustCertCollectionFilePath = toml.getString("grpc.ca");
+        logger.debug("loading ca from {}", trustCertCollectionFilePath);
+        String clientCertChainFilePath = toml.getString("grpc.client.cert");
+        logger.debug("loading client ca from {}", clientCertChainFilePath);
+        String clientPrivateKeyFilePath = toml.getString("grpc.client.key");
+        logger.debug("loading client key from {}", clientPrivateKeyFilePath);
+
+        if (Strings.isNullOrEmpty(clientPrivateKeyFilePath) && Strings.isNullOrEmpty(clientPrivateKeyFilePath)){
+            return null;
+        }
+
+        // possibly fix the format https://netty.io/wiki/sslcontextbuilder-and-private-key.html
+
+        return buildSslContext(trustCertCollectionFilePath, clientCertChainFilePath, clientPrivateKeyFilePath);
+    }
+
+
+    private static SslContext buildSslContext(String trustCertCollectionFilePath,
+                                             String clientCertChainFilePath,
+                                             String clientPrivateKeyFilePath) throws SSLException {
+        SslContextBuilder builder = GrpcSslContexts.forClient();
+        if (trustCertCollectionFilePath != null) {
+            builder.trustManager(new File(trustCertCollectionFilePath));
+        }
+        if (clientCertChainFilePath != null && clientPrivateKeyFilePath != null) {
+            builder.keyManager(new File(clientCertChainFilePath), new File(clientPrivateKeyFilePath));
+        }
+        return builder.trustManager(InsecureTrustManagerFactory.INSTANCE).build();
+    }
+}
diff --git a/other/java/client/src/test/java/seaweedfs/client/SeaweedFilerTest.java b/other/java/client/src/test/java/seaweedfs/client/SeaweedFilerTest.java
new file mode 100644
index 000000000..dde23ee87
--- /dev/null
+++ b/other/java/client/src/test/java/seaweedfs/client/SeaweedFilerTest.java
@@ -0,0 +1,17 @@
+package seaweedfs.client;
+
+import java.util.List;
+
+public class SeaweedFilerTest {
+    public static void main(String[] args){
+
+        FilerClient filerClient = new FilerClient("localhost", 18888);
+
+        List<FilerProto.Entry> entries = filerClient.listEntries("/");
+
+        for (FilerProto.Entry entry : entries) {
+            System.out.println(entry.toString());
+        }
+
+    }
+}
diff --git a/other/java/hdfs2/dependency-reduced-pom.xml b/other/java/hdfs2/dependency-reduced-pom.xml
index cfc869312..7003f5317 100644
--- a/other/java/hdfs2/dependency-reduced-pom.xml
+++ b/other/java/hdfs2/dependency-reduced-pom.xml
@@ -8,7 +8,7 @@
   </parent>

   <modelVersion>4.0.0</modelVersion>

   <groupId>com.github.chrislusf</groupId>

-  <artifactId>seaweedfs-hadoop-client</artifactId>

+  <artifactId>seaweedfs-hadoop2-client</artifactId>

   <version>${seaweedfs.client.version}</version>

   <build>

     <plugins>

@@ -123,7 +123,7 @@
     </snapshotRepository>

   </distributionManagement>

   <properties>

-    <seaweedfs.client.version>1.1.0</seaweedfs.client.version>

-    <hadoop.version>2.7.4</hadoop.version>

+    <seaweedfs.client.version>1.1.3</seaweedfs.client.version>

+    <hadoop.version>2.9.2</hadoop.version>

   </properties>

 </project>

diff --git a/other/java/hdfs2/pom.xml b/other/java/hdfs2/pom.xml
index 5efb46757..b3fa911c1 100644
--- a/other/java/hdfs2/pom.xml
+++ b/other/java/hdfs2/pom.xml
@@ -5,12 +5,12 @@
     <modelVersion>4.0.0</modelVersion>
 
     <properties>
-        <seaweedfs.client.version>1.1.0</seaweedfs.client.version>
-        <hadoop.version>2.7.4</hadoop.version>
+        <seaweedfs.client.version>1.1.3</seaweedfs.client.version>
+        <hadoop.version>2.9.2</hadoop.version>
     </properties>
 
     <groupId>com.github.chrislusf</groupId>
-    <artifactId>seaweedfs-hadoop-client</artifactId>
+    <artifactId>seaweedfs-hadoop2-client</artifactId>
     <version>${seaweedfs.client.version}</version>
 
     <parent>
diff --git a/other/java/hdfs2/src/main/java/seaweed/hdfs/SeaweedFileSystem.java b/other/java/hdfs2/src/main/java/seaweed/hdfs/SeaweedFileSystem.java
index 453924cf7..7cf76e5e8 100644
--- a/other/java/hdfs2/src/main/java/seaweed/hdfs/SeaweedFileSystem.java
+++ b/other/java/hdfs2/src/main/java/seaweed/hdfs/SeaweedFileSystem.java
@@ -34,9 +34,6 @@ public class SeaweedFileSystem extends org.apache.hadoop.fs.FileSystem {
     public static final int FS_SEAWEED_DEFAULT_PORT = 8888;
     public static final String FS_SEAWEED_FILER_HOST = "fs.seaweed.filer.host";
     public static final String FS_SEAWEED_FILER_PORT = "fs.seaweed.filer.port";
-    public static final String FS_SEAWEED_GRPC_CA = "fs.seaweed.ca";
-    public static final String FS_SEAWEED_GRPC_CLIENT_KEY = "fs.seaweed.client.key";
-    public static final String FS_SEAWEED_GRPC_CLIENT_CERT = "fs.seaweed.client.cert";
 
     private static final Logger LOG = LoggerFactory.getLogger(SeaweedFileSystem.class);
     private static int BUFFER_SIZE = 16 * 1024 * 1024;
@@ -75,16 +72,7 @@ public class SeaweedFileSystem extends org.apache.hadoop.fs.FileSystem {
         setConf(conf);
         this.uri = uri;
 
-        if (conf.get(FS_SEAWEED_GRPC_CA) != null && conf.getTrimmed(FS_SEAWEED_GRPC_CA).length() != 0
-                && conf.get(FS_SEAWEED_GRPC_CLIENT_CERT) != null && conf.getTrimmed(FS_SEAWEED_GRPC_CLIENT_CERT).length() != 0
-                && conf.get(FS_SEAWEED_GRPC_CLIENT_KEY) != null && conf.getTrimmed(FS_SEAWEED_GRPC_CLIENT_KEY).length() != 0) {
-            seaweedFileSystemStore = new SeaweedFileSystemStore(host, port,
-                    conf.get(FS_SEAWEED_GRPC_CA),
-                    conf.get(FS_SEAWEED_GRPC_CLIENT_CERT),
-                    conf.get(FS_SEAWEED_GRPC_CLIENT_KEY));
-        } else {
-            seaweedFileSystemStore = new SeaweedFileSystemStore(host, port);
-        }
+        seaweedFileSystemStore = new SeaweedFileSystemStore(host, port);
 
     }
 
diff --git a/other/java/hdfs2/src/main/java/seaweed/hdfs/SeaweedFileSystemStore.java b/other/java/hdfs2/src/main/java/seaweed/hdfs/SeaweedFileSystemStore.java
index 643467898..2ddcd41e8 100644
--- a/other/java/hdfs2/src/main/java/seaweed/hdfs/SeaweedFileSystemStore.java
+++ b/other/java/hdfs2/src/main/java/seaweed/hdfs/SeaweedFileSystemStore.java
@@ -12,7 +12,6 @@ import seaweedfs.client.FilerGrpcClient;
 import seaweedfs.client.FilerProto;
 import seaweedfs.client.SeaweedRead;
 
-import javax.net.ssl.SSLException;
 import java.io.FileNotFoundException;
 import java.io.IOException;
 import java.io.InputStream;
@@ -34,13 +33,6 @@ public class SeaweedFileSystemStore {
         filerClient = new FilerClient(filerGrpcClient);
     }
 
-    public SeaweedFileSystemStore(String host, int port,
-                                  String caFile, String clientCertFile, String clientKeyFile) throws SSLException {
-        int grpcPort = 10000 + port;
-        filerGrpcClient = new FilerGrpcClient(host, grpcPort, caFile, clientCertFile, clientKeyFile);
-        filerClient = new FilerClient(filerGrpcClient);
-    }
-
     public static String getParentDirectory(Path path) {
         return path.isRoot() ? "/" : path.getParent().toUri().getPath();
     }
diff --git a/other/java/hdfs3/dependency-reduced-pom.xml b/other/java/hdfs3/dependency-reduced-pom.xml
index 867a81caf..8916a8b5a 100644
--- a/other/java/hdfs3/dependency-reduced-pom.xml
+++ b/other/java/hdfs3/dependency-reduced-pom.xml
@@ -8,7 +8,7 @@
   </parent>

   <modelVersion>4.0.0</modelVersion>

   <groupId>com.github.chrislusf</groupId>

-  <artifactId>seaweedfs-hadoop-client</artifactId>

+  <artifactId>seaweedfs-hadoop3-client</artifactId>

   <version>${seaweedfs.client.version}</version>

   <build>

     <plugins>

@@ -123,7 +123,7 @@
     </snapshotRepository>

   </distributionManagement>

   <properties>

-    <seaweedfs.client.version>1.1.0</seaweedfs.client.version>

+    <seaweedfs.client.version>1.1.3</seaweedfs.client.version>

     <hadoop.version>3.1.1</hadoop.version>

   </properties>

 </project>

diff --git a/other/java/hdfs3/pom.xml b/other/java/hdfs3/pom.xml
index 6a1cd897f..ad1930587 100644
--- a/other/java/hdfs3/pom.xml
+++ b/other/java/hdfs3/pom.xml
@@ -5,12 +5,12 @@
     <modelVersion>4.0.0</modelVersion>
 
     <properties>
-        <seaweedfs.client.version>1.1.0</seaweedfs.client.version>
+        <seaweedfs.client.version>1.1.3</seaweedfs.client.version>
         <hadoop.version>3.1.1</hadoop.version>
     </properties>
 
     <groupId>com.github.chrislusf</groupId>
-    <artifactId>seaweedfs-hadoop-client</artifactId>
+    <artifactId>seaweedfs-hadoop3-client</artifactId>
     <version>${seaweedfs.client.version}</version>
 
     <parent>
diff --git a/other/java/hdfs3/src/main/java/seaweed/hdfs/SeaweedFileSystem.java b/other/java/hdfs3/src/main/java/seaweed/hdfs/SeaweedFileSystem.java
index 453924cf7..7cf76e5e8 100644
--- a/other/java/hdfs3/src/main/java/seaweed/hdfs/SeaweedFileSystem.java
+++ b/other/java/hdfs3/src/main/java/seaweed/hdfs/SeaweedFileSystem.java
@@ -34,9 +34,6 @@ public class SeaweedFileSystem extends org.apache.hadoop.fs.FileSystem {
     public static final int FS_SEAWEED_DEFAULT_PORT = 8888;
     public static final String FS_SEAWEED_FILER_HOST = "fs.seaweed.filer.host";
     public static final String FS_SEAWEED_FILER_PORT = "fs.seaweed.filer.port";
-    public static final String FS_SEAWEED_GRPC_CA = "fs.seaweed.ca";
-    public static final String FS_SEAWEED_GRPC_CLIENT_KEY = "fs.seaweed.client.key";
-    public static final String FS_SEAWEED_GRPC_CLIENT_CERT = "fs.seaweed.client.cert";
 
     private static final Logger LOG = LoggerFactory.getLogger(SeaweedFileSystem.class);
     private static int BUFFER_SIZE = 16 * 1024 * 1024;
@@ -75,16 +72,7 @@ public class SeaweedFileSystem extends org.apache.hadoop.fs.FileSystem {
         setConf(conf);
         this.uri = uri;
 
-        if (conf.get(FS_SEAWEED_GRPC_CA) != null && conf.getTrimmed(FS_SEAWEED_GRPC_CA).length() != 0
-                && conf.get(FS_SEAWEED_GRPC_CLIENT_CERT) != null && conf.getTrimmed(FS_SEAWEED_GRPC_CLIENT_CERT).length() != 0
-                && conf.get(FS_SEAWEED_GRPC_CLIENT_KEY) != null && conf.getTrimmed(FS_SEAWEED_GRPC_CLIENT_KEY).length() != 0) {
-            seaweedFileSystemStore = new SeaweedFileSystemStore(host, port,
-                    conf.get(FS_SEAWEED_GRPC_CA),
-                    conf.get(FS_SEAWEED_GRPC_CLIENT_CERT),
-                    conf.get(FS_SEAWEED_GRPC_CLIENT_KEY));
-        } else {
-            seaweedFileSystemStore = new SeaweedFileSystemStore(host, port);
-        }
+        seaweedFileSystemStore = new SeaweedFileSystemStore(host, port);
 
     }
 
diff --git a/other/java/hdfs3/src/main/java/seaweed/hdfs/SeaweedFileSystemStore.java b/other/java/hdfs3/src/main/java/seaweed/hdfs/SeaweedFileSystemStore.java
index 643467898..2ddcd41e8 100644
--- a/other/java/hdfs3/src/main/java/seaweed/hdfs/SeaweedFileSystemStore.java
+++ b/other/java/hdfs3/src/main/java/seaweed/hdfs/SeaweedFileSystemStore.java
@@ -12,7 +12,6 @@ import seaweedfs.client.FilerGrpcClient;
 import seaweedfs.client.FilerProto;
 import seaweedfs.client.SeaweedRead;
 
-import javax.net.ssl.SSLException;
 import java.io.FileNotFoundException;
 import java.io.IOException;
 import java.io.InputStream;
@@ -34,13 +33,6 @@ public class SeaweedFileSystemStore {
         filerClient = new FilerClient(filerGrpcClient);
     }
 
-    public SeaweedFileSystemStore(String host, int port,
-                                  String caFile, String clientCertFile, String clientKeyFile) throws SSLException {
-        int grpcPort = 10000 + port;
-        filerGrpcClient = new FilerGrpcClient(host, grpcPort, caFile, clientCertFile, clientKeyFile);
-        filerClient = new FilerClient(filerGrpcClient);
-    }
-
     public static String getParentDirectory(Path path) {
         return path.isRoot() ? "/" : path.getParent().toUri().getPath();
     }