merge master, resolve conflicts

13 files changed, 94 insertions, 16 deletions

diff --git a/other/java/client/pom.xml b/other/java/client/pom.xml
index d2c91b121..70c5dbd31 100644
--- a/other/java/client/pom.xml
+++ b/other/java/client/pom.xml
@@ -5,7 +5,7 @@
 
     <groupId>com.github.chrislusf</groupId>
     <artifactId>seaweedfs-client</artifactId>
-    <version>1.6.6</version>
+    <version>1.6.7</version>
 
     <parent>
         <groupId>org.sonatype.oss</groupId>
diff --git a/other/java/client/pom.xml.deploy b/other/java/client/pom.xml.deploy
index 7910e2491..82cf5e82b 100644
--- a/other/java/client/pom.xml.deploy
+++ b/other/java/client/pom.xml.deploy
@@ -5,7 +5,7 @@
 
     <groupId>com.github.chrislusf</groupId>
     <artifactId>seaweedfs-client</artifactId>
-    <version>1.6.6</version>
+    <version>1.6.7</version>
 
     <parent>
         <groupId>org.sonatype.oss</groupId>
diff --git a/other/java/client/pom_debug.xml b/other/java/client/pom_debug.xml
index c3cf904c0..c72c81ab7 100644
--- a/other/java/client/pom_debug.xml
+++ b/other/java/client/pom_debug.xml
@@ -5,7 +5,7 @@
 
     <groupId>com.github.chrislusf</groupId>
     <artifactId>seaweedfs-client</artifactId>
-    <version>1.6.6</version>
+    <version>1.6.7</version>
 
     <parent>
         <groupId>org.sonatype.oss</groupId>
diff --git a/other/java/client/src/main/java/seaweedfs/client/ByteBufferPool.java b/other/java/client/src/main/java/seaweedfs/client/ByteBufferPool.java
index 51053becd..19ae78277 100644
--- a/other/java/client/src/main/java/seaweedfs/client/ByteBufferPool.java
+++ b/other/java/client/src/main/java/seaweedfs/client/ByteBufferPool.java
@@ -10,7 +10,7 @@ import java.util.List;
 
 public class ByteBufferPool {
 
-    private static final int MIN_BUFFER_SIZE = 8 * 1024 * 1024;
+    private static final int MIN_BUFFER_SIZE = 1 * 1024 * 1024;
     private static final Logger LOG = LoggerFactory.getLogger(ByteBufferPool.class);
 
     private static final List<ByteBuffer> bufferList = new ArrayList<>();
diff --git a/other/java/client/src/main/java/seaweedfs/client/RemoteUtil.java b/other/java/client/src/main/java/seaweedfs/client/RemoteUtil.java
new file mode 100644
index 000000000..39c17644b
--- /dev/null
+++ b/other/java/client/src/main/java/seaweedfs/client/RemoteUtil.java
@@ -0,0 +1,23 @@
+package seaweedfs.client;
+
+import java.io.IOException;
+
+public class RemoteUtil {
+    public static boolean isInRemoteOnly(FilerProto.Entry entry) {
+        if (entry.getChunksList() == null || entry.getChunksList().isEmpty()) {
+            return entry.getRemoteEntry() != null && entry.getRemoteEntry().getRemoteSize() > 0;
+        }
+        return false;
+    }
+
+    public static FilerProto.Entry downloadRemoteEntry(FilerClient filerClient, String fullpath, FilerProto.Entry entry) throws IOException {
+        String dir = SeaweedOutputStream.getParentDirectory(fullpath);
+        String name = SeaweedOutputStream.getFileName(fullpath);
+
+        final FilerProto.DownloadToLocalResponse downloadToLocalResponse = filerClient.getBlockingStub()
+                .downloadToLocal(FilerProto.DownloadToLocalRequest.newBuilder()
+                .setDirectory(dir).setName(name).build());
+
+        return downloadToLocalResponse.getEntry();
+    }
+}
diff --git a/other/java/client/src/main/java/seaweedfs/client/SeaweedInputStream.java b/other/java/client/src/main/java/seaweedfs/client/SeaweedInputStream.java
index 6097b8d56..9d1fb3417 100644
--- a/other/java/client/src/main/java/seaweedfs/client/SeaweedInputStream.java
+++ b/other/java/client/src/main/java/seaweedfs/client/SeaweedInputStream.java
@@ -19,9 +19,9 @@ public class SeaweedInputStream extends InputStream {
 
     private final FilerClient filerClient;
     private final String path;
-    private final FilerProto.Entry entry;
     private final List<SeaweedRead.VisibleInterval> visibleIntervalList;
     private final long contentLength;
+    private FilerProto.Entry entry;
 
     private long position = 0;  // cursor of the file
 
@@ -35,10 +35,14 @@ public class SeaweedInputStream extends InputStream {
         this.entry = filerClient.lookupEntry(
                 SeaweedOutputStream.getParentDirectory(fullpath),
                 SeaweedOutputStream.getFileName(fullpath));
-        if(entry == null){
+        if (entry == null) {
             throw new FileNotFoundException();
         }
 
+        if (RemoteUtil.isInRemoteOnly(entry)) {
+            entry = RemoteUtil.downloadRemoteEntry(filerClient, fullpath, entry);
+        }
+
         this.contentLength = SeaweedRead.fileSize(entry);
 
         this.visibleIntervalList = SeaweedRead.nonOverlappingVisibleIntervals(filerClient, entry.getChunksList());
@@ -54,6 +58,11 @@ public class SeaweedInputStream extends InputStream {
         this.filerClient = filerClient;
         this.path = path;
         this.entry = entry;
+
+        if (RemoteUtil.isInRemoteOnly(entry)) {
+            this.entry = RemoteUtil.downloadRemoteEntry(filerClient, path, entry);
+        }
+
         this.contentLength = SeaweedRead.fileSize(entry);
 
         this.visibleIntervalList = SeaweedRead.nonOverlappingVisibleIntervals(filerClient, entry.getChunksList());
@@ -111,8 +120,8 @@ public class SeaweedInputStream extends InputStream {
         long bytesRead = 0;
         int len = buf.remaining();
         int start = (int) this.position;
-        if (start+len <= entry.getContent().size()) {
-            entry.getContent().substring(start, start+len).copyTo(buf);
+        if (start + len <= entry.getContent().size()) {
+            entry.getContent().substring(start, start + len).copyTo(buf);
         } else {
             bytesRead = SeaweedRead.read(this.filerClient, this.visibleIntervalList, this.position, buf, SeaweedRead.fileSize(entry));
         }
diff --git a/other/java/client/src/main/java/seaweedfs/client/SeaweedOutputStream.java b/other/java/client/src/main/java/seaweedfs/client/SeaweedOutputStream.java
index 3bec05796..7a94acbd0 100644
--- a/other/java/client/src/main/java/seaweedfs/client/SeaweedOutputStream.java
+++ b/other/java/client/src/main/java/seaweedfs/client/SeaweedOutputStream.java
@@ -32,10 +32,10 @@ public class SeaweedOutputStream extends OutputStream {
     private long lastTotalAppendOffset = 0;
     private ByteBuffer buffer;
     private long outputIndex;
-    private String replication = "000";
+    private String replication = "";
 
     public SeaweedOutputStream(FilerClient filerClient, final String fullpath) {
-        this(filerClient, fullpath, "000");
+        this(filerClient, fullpath, "");
     }
 
     public SeaweedOutputStream(FilerClient filerClient, final String fullpath, final String replication) {
diff --git a/other/java/client/src/main/proto/filer.proto b/other/java/client/src/main/proto/filer.proto
index ac4c9a0e7..c9257ddca 100644
--- a/other/java/client/src/main/proto/filer.proto
+++ b/other/java/client/src/main/proto/filer.proto
@@ -67,6 +67,8 @@ service SeaweedFiler {
     rpc KvPut (KvPutRequest) returns (KvPutResponse) {
     }
 
+    rpc DownloadToLocal (DownloadToLocalRequest) returns (DownloadToLocalResponse) {
+    }
 }
 
 //////////////////////////////////////////////////
@@ -92,6 +94,13 @@ message ListEntriesResponse {
     Entry entry = 1;
 }
 
+message RemoteEntry {
+    string storage_name = 1;
+    int64 local_mtime = 2;
+    string remote_e_tag = 3;
+    int64 remote_mtime = 4;
+    int64 remote_size = 5;
+}
 message Entry {
     string name = 1;
     bool is_directory = 2;
@@ -101,6 +110,8 @@ message Entry {
     bytes hard_link_id = 7;
     int32 hard_link_counter = 8; // only exists in hard link meta data
     bytes content = 9; // if not empty, the file content
+
+    RemoteEntry remote_entry = 10;
 }
 
 message FullEntry {
@@ -208,6 +219,7 @@ message AtomicRenameEntryRequest {
     string old_name = 2;
     string new_directory = 3;
     string new_name = 4;
+    repeated int32 signatures = 5;
 }
 
 message AtomicRenameEntryResponse {
@@ -292,6 +304,7 @@ message GetFilerConfigurationResponse {
     int32 signature = 8;
     string metrics_address = 9;
     int32 metrics_interval_sec = 10;
+    string version = 11;
 }
 
 message SubscribeMetadataRequest {
@@ -334,7 +347,9 @@ message LocateBrokerResponse {
     repeated Resource resources = 2;
 }
 
+/////////////////////////
 // Key-Value operations
+/////////////////////////
 message KvGetRequest {
     bytes key = 1;
 }
@@ -350,7 +365,9 @@ message KvPutResponse {
     string error = 1;
 }
 
+/////////////////////////
 // path-based configurations
+/////////////////////////
 message FilerConf {
     int32 version = 1;
     message PathConf {
@@ -361,6 +378,35 @@ message FilerConf {
         string disk_type = 5;
         bool fsync = 6;
         uint32 volume_growth_count = 7;
+        bool read_only = 8;
     }
     repeated PathConf locations = 2;
 }
+
+/////////////////////////
+// Remote Storage related
+/////////////////////////
+message RemoteConf {
+    string type = 1;
+    string name = 2;
+    string s3_access_key = 4;
+    string s3_secret_key = 5;
+    string s3_region = 6;
+    string s3_endpoint = 7;
+}
+
+message RemoteStorageMapping {
+    map<string,RemoteStorageLocation> mappings = 1;
+}
+message RemoteStorageLocation {
+    string name = 1;
+    string bucket = 2;
+    string path = 3;
+}
+message DownloadToLocalRequest {
+    string directory = 1;
+    string name = 2;
+}
+message DownloadToLocalResponse {
+    Entry entry = 1;
+}
diff --git a/other/java/examples/pom.xml b/other/java/examples/pom.xml
index 9a42a0191..26c9bdfdc 100644
--- a/other/java/examples/pom.xml
+++ b/other/java/examples/pom.xml
@@ -11,13 +11,13 @@
         <dependency>
             <groupId>com.github.chrislusf</groupId>
             <artifactId>seaweedfs-client</artifactId>
-            <version>1.6.6</version>
+            <version>1.6.7</version>
             <scope>compile</scope>
         </dependency>
         <dependency>
             <groupId>com.github.chrislusf</groupId>
             <artifactId>seaweedfs-hadoop2-client</artifactId>
-            <version>1.6.6</version>
+            <version>1.6.7</version>
             <scope>compile</scope>
         </dependency>
         <dependency>
diff --git a/other/java/hdfs2/dependency-reduced-pom.xml b/other/java/hdfs2/dependency-reduced-pom.xml
index 1b5a5c3fc..bd31637ce 100644
--- a/other/java/hdfs2/dependency-reduced-pom.xml
+++ b/other/java/hdfs2/dependency-reduced-pom.xml
@@ -301,7 +301,7 @@
     </snapshotRepository>

   </distributionManagement>

   <properties>

-    <seaweedfs.client.version>1.6.6</seaweedfs.client.version>

+    <seaweedfs.client.version>1.6.7</seaweedfs.client.version>

     <hadoop.version>2.9.2</hadoop.version>

   </properties>

 </project>

diff --git a/other/java/hdfs2/pom.xml b/other/java/hdfs2/pom.xml
index 58e51a2a1..f15d24faa 100644
--- a/other/java/hdfs2/pom.xml
+++ b/other/java/hdfs2/pom.xml
@@ -5,7 +5,7 @@
     <modelVersion>4.0.0</modelVersion>
 
     <properties>
-        <seaweedfs.client.version>1.6.6</seaweedfs.client.version>
+        <seaweedfs.client.version>1.6.7</seaweedfs.client.version>
         <hadoop.version>2.9.2</hadoop.version>
     </properties>
 
diff --git a/other/java/hdfs3/dependency-reduced-pom.xml b/other/java/hdfs3/dependency-reduced-pom.xml
index 58556a9c7..4640b5a84 100644
--- a/other/java/hdfs3/dependency-reduced-pom.xml
+++ b/other/java/hdfs3/dependency-reduced-pom.xml
@@ -309,7 +309,7 @@
     </snapshotRepository>

   </distributionManagement>

   <properties>

-    <seaweedfs.client.version>1.6.6</seaweedfs.client.version>

+    <seaweedfs.client.version>1.6.7</seaweedfs.client.version>

     <hadoop.version>3.1.1</hadoop.version>

   </properties>

 </project>

diff --git a/other/java/hdfs3/pom.xml b/other/java/hdfs3/pom.xml
index bbcc1788d..efcc1e4c0 100644
--- a/other/java/hdfs3/pom.xml
+++ b/other/java/hdfs3/pom.xml
@@ -5,7 +5,7 @@
     <modelVersion>4.0.0</modelVersion>
 
     <properties>
-        <seaweedfs.client.version>1.6.6</seaweedfs.client.version>
+        <seaweedfs.client.version>1.6.7</seaweedfs.client.version>
         <hadoop.version>3.1.1</hadoop.version>
     </properties>