aboutsummaryrefslogtreecommitdiff
path: root/other
diff options
context:
space:
mode:
Diffstat (limited to 'other')
-rw-r--r--other/java/client/pom.xml2
-rw-r--r--other/java/client/pom.xml.deploy2
-rw-r--r--other/java/client/pom_debug.xml2
-rw-r--r--other/java/client/src/main/java/seaweedfs/client/ByteBufferPool.java2
-rw-r--r--other/java/client/src/main/java/seaweedfs/client/RemoteUtil.java23
-rw-r--r--other/java/client/src/main/java/seaweedfs/client/SeaweedInputStream.java17
-rw-r--r--other/java/client/src/main/java/seaweedfs/client/SeaweedOutputStream.java4
-rw-r--r--other/java/client/src/main/proto/filer.proto46
-rw-r--r--other/java/examples/pom.xml4
-rw-r--r--other/java/hdfs2/dependency-reduced-pom.xml2
-rw-r--r--other/java/hdfs2/pom.xml2
-rw-r--r--other/java/hdfs3/dependency-reduced-pom.xml2
-rw-r--r--other/java/hdfs3/pom.xml2
-rw-r--r--other/metrics/grafana_seaweedfs.json330
14 files changed, 424 insertions, 16 deletions
diff --git a/other/java/client/pom.xml b/other/java/client/pom.xml
index d2c91b121..70c5dbd31 100644
--- a/other/java/client/pom.xml
+++ b/other/java/client/pom.xml
@@ -5,7 +5,7 @@
<groupId>com.github.chrislusf</groupId>
<artifactId>seaweedfs-client</artifactId>
- <version>1.6.6</version>
+ <version>1.6.7</version>
<parent>
<groupId>org.sonatype.oss</groupId>
diff --git a/other/java/client/pom.xml.deploy b/other/java/client/pom.xml.deploy
index 7910e2491..82cf5e82b 100644
--- a/other/java/client/pom.xml.deploy
+++ b/other/java/client/pom.xml.deploy
@@ -5,7 +5,7 @@
<groupId>com.github.chrislusf</groupId>
<artifactId>seaweedfs-client</artifactId>
- <version>1.6.6</version>
+ <version>1.6.7</version>
<parent>
<groupId>org.sonatype.oss</groupId>
diff --git a/other/java/client/pom_debug.xml b/other/java/client/pom_debug.xml
index c3cf904c0..c72c81ab7 100644
--- a/other/java/client/pom_debug.xml
+++ b/other/java/client/pom_debug.xml
@@ -5,7 +5,7 @@
<groupId>com.github.chrislusf</groupId>
<artifactId>seaweedfs-client</artifactId>
- <version>1.6.6</version>
+ <version>1.6.7</version>
<parent>
<groupId>org.sonatype.oss</groupId>
diff --git a/other/java/client/src/main/java/seaweedfs/client/ByteBufferPool.java b/other/java/client/src/main/java/seaweedfs/client/ByteBufferPool.java
index 51053becd..19ae78277 100644
--- a/other/java/client/src/main/java/seaweedfs/client/ByteBufferPool.java
+++ b/other/java/client/src/main/java/seaweedfs/client/ByteBufferPool.java
@@ -10,7 +10,7 @@ import java.util.List;
public class ByteBufferPool {
- private static final int MIN_BUFFER_SIZE = 8 * 1024 * 1024;
+ private static final int MIN_BUFFER_SIZE = 1 * 1024 * 1024;
private static final Logger LOG = LoggerFactory.getLogger(ByteBufferPool.class);
private static final List<ByteBuffer> bufferList = new ArrayList<>();
diff --git a/other/java/client/src/main/java/seaweedfs/client/RemoteUtil.java b/other/java/client/src/main/java/seaweedfs/client/RemoteUtil.java
new file mode 100644
index 000000000..39c17644b
--- /dev/null
+++ b/other/java/client/src/main/java/seaweedfs/client/RemoteUtil.java
@@ -0,0 +1,23 @@
+package seaweedfs.client;
+
+import java.io.IOException;
+
+public class RemoteUtil {
+ public static boolean isInRemoteOnly(FilerProto.Entry entry) {
+ if (entry.getChunksList() == null || entry.getChunksList().isEmpty()) {
+ return entry.getRemoteEntry() != null && entry.getRemoteEntry().getRemoteSize() > 0;
+ }
+ return false;
+ }
+
+ public static FilerProto.Entry downloadRemoteEntry(FilerClient filerClient, String fullpath, FilerProto.Entry entry) throws IOException {
+ String dir = SeaweedOutputStream.getParentDirectory(fullpath);
+ String name = SeaweedOutputStream.getFileName(fullpath);
+
+ final FilerProto.DownloadToLocalResponse downloadToLocalResponse = filerClient.getBlockingStub()
+ .downloadToLocal(FilerProto.DownloadToLocalRequest.newBuilder()
+ .setDirectory(dir).setName(name).build());
+
+ return downloadToLocalResponse.getEntry();
+ }
+}
diff --git a/other/java/client/src/main/java/seaweedfs/client/SeaweedInputStream.java b/other/java/client/src/main/java/seaweedfs/client/SeaweedInputStream.java
index 6097b8d56..9d1fb3417 100644
--- a/other/java/client/src/main/java/seaweedfs/client/SeaweedInputStream.java
+++ b/other/java/client/src/main/java/seaweedfs/client/SeaweedInputStream.java
@@ -19,9 +19,9 @@ public class SeaweedInputStream extends InputStream {
private final FilerClient filerClient;
private final String path;
- private final FilerProto.Entry entry;
private final List<SeaweedRead.VisibleInterval> visibleIntervalList;
private final long contentLength;
+ private FilerProto.Entry entry;
private long position = 0; // cursor of the file
@@ -35,10 +35,14 @@ public class SeaweedInputStream extends InputStream {
this.entry = filerClient.lookupEntry(
SeaweedOutputStream.getParentDirectory(fullpath),
SeaweedOutputStream.getFileName(fullpath));
- if(entry == null){
+ if (entry == null) {
throw new FileNotFoundException();
}
+ if (RemoteUtil.isInRemoteOnly(entry)) {
+ entry = RemoteUtil.downloadRemoteEntry(filerClient, fullpath, entry);
+ }
+
this.contentLength = SeaweedRead.fileSize(entry);
this.visibleIntervalList = SeaweedRead.nonOverlappingVisibleIntervals(filerClient, entry.getChunksList());
@@ -54,6 +58,11 @@ public class SeaweedInputStream extends InputStream {
this.filerClient = filerClient;
this.path = path;
this.entry = entry;
+
+ if (RemoteUtil.isInRemoteOnly(entry)) {
+ this.entry = RemoteUtil.downloadRemoteEntry(filerClient, path, entry);
+ }
+
this.contentLength = SeaweedRead.fileSize(entry);
this.visibleIntervalList = SeaweedRead.nonOverlappingVisibleIntervals(filerClient, entry.getChunksList());
@@ -111,8 +120,8 @@ public class SeaweedInputStream extends InputStream {
long bytesRead = 0;
int len = buf.remaining();
int start = (int) this.position;
- if (start+len <= entry.getContent().size()) {
- entry.getContent().substring(start, start+len).copyTo(buf);
+ if (start + len <= entry.getContent().size()) {
+ entry.getContent().substring(start, start + len).copyTo(buf);
} else {
bytesRead = SeaweedRead.read(this.filerClient, this.visibleIntervalList, this.position, buf, SeaweedRead.fileSize(entry));
}
diff --git a/other/java/client/src/main/java/seaweedfs/client/SeaweedOutputStream.java b/other/java/client/src/main/java/seaweedfs/client/SeaweedOutputStream.java
index 3bec05796..7a94acbd0 100644
--- a/other/java/client/src/main/java/seaweedfs/client/SeaweedOutputStream.java
+++ b/other/java/client/src/main/java/seaweedfs/client/SeaweedOutputStream.java
@@ -32,10 +32,10 @@ public class SeaweedOutputStream extends OutputStream {
private long lastTotalAppendOffset = 0;
private ByteBuffer buffer;
private long outputIndex;
- private String replication = "000";
+ private String replication = "";
public SeaweedOutputStream(FilerClient filerClient, final String fullpath) {
- this(filerClient, fullpath, "000");
+ this(filerClient, fullpath, "");
}
public SeaweedOutputStream(FilerClient filerClient, final String fullpath, final String replication) {
diff --git a/other/java/client/src/main/proto/filer.proto b/other/java/client/src/main/proto/filer.proto
index ac4c9a0e7..c9257ddca 100644
--- a/other/java/client/src/main/proto/filer.proto
+++ b/other/java/client/src/main/proto/filer.proto
@@ -67,6 +67,8 @@ service SeaweedFiler {
rpc KvPut (KvPutRequest) returns (KvPutResponse) {
}
+ rpc DownloadToLocal (DownloadToLocalRequest) returns (DownloadToLocalResponse) {
+ }
}
//////////////////////////////////////////////////
@@ -92,6 +94,13 @@ message ListEntriesResponse {
Entry entry = 1;
}
+message RemoteEntry {
+ string storage_name = 1;
+ int64 local_mtime = 2;
+ string remote_e_tag = 3;
+ int64 remote_mtime = 4;
+ int64 remote_size = 5;
+}
message Entry {
string name = 1;
bool is_directory = 2;
@@ -101,6 +110,8 @@ message Entry {
bytes hard_link_id = 7;
int32 hard_link_counter = 8; // only exists in hard link meta data
bytes content = 9; // if not empty, the file content
+
+ RemoteEntry remote_entry = 10;
}
message FullEntry {
@@ -208,6 +219,7 @@ message AtomicRenameEntryRequest {
string old_name = 2;
string new_directory = 3;
string new_name = 4;
+ repeated int32 signatures = 5;
}
message AtomicRenameEntryResponse {
@@ -292,6 +304,7 @@ message GetFilerConfigurationResponse {
int32 signature = 8;
string metrics_address = 9;
int32 metrics_interval_sec = 10;
+ string version = 11;
}
message SubscribeMetadataRequest {
@@ -334,7 +347,9 @@ message LocateBrokerResponse {
repeated Resource resources = 2;
}
+/////////////////////////
// Key-Value operations
+/////////////////////////
message KvGetRequest {
bytes key = 1;
}
@@ -350,7 +365,9 @@ message KvPutResponse {
string error = 1;
}
+/////////////////////////
// path-based configurations
+/////////////////////////
message FilerConf {
int32 version = 1;
message PathConf {
@@ -361,6 +378,35 @@ message FilerConf {
string disk_type = 5;
bool fsync = 6;
uint32 volume_growth_count = 7;
+ bool read_only = 8;
}
repeated PathConf locations = 2;
}
+
+/////////////////////////
+// Remote Storage related
+/////////////////////////
+message RemoteConf {
+ string type = 1;
+ string name = 2;
+ string s3_access_key = 4;
+ string s3_secret_key = 5;
+ string s3_region = 6;
+ string s3_endpoint = 7;
+}
+
+message RemoteStorageMapping {
+ map<string,RemoteStorageLocation> mappings = 1;
+}
+message RemoteStorageLocation {
+ string name = 1;
+ string bucket = 2;
+ string path = 3;
+}
+message DownloadToLocalRequest {
+ string directory = 1;
+ string name = 2;
+}
+message DownloadToLocalResponse {
+ Entry entry = 1;
+}
diff --git a/other/java/examples/pom.xml b/other/java/examples/pom.xml
index 9a42a0191..26c9bdfdc 100644
--- a/other/java/examples/pom.xml
+++ b/other/java/examples/pom.xml
@@ -11,13 +11,13 @@
<dependency>
<groupId>com.github.chrislusf</groupId>
<artifactId>seaweedfs-client</artifactId>
- <version>1.6.6</version>
+ <version>1.6.7</version>
<scope>compile</scope>
</dependency>
<dependency>
<groupId>com.github.chrislusf</groupId>
<artifactId>seaweedfs-hadoop2-client</artifactId>
- <version>1.6.6</version>
+ <version>1.6.7</version>
<scope>compile</scope>
</dependency>
<dependency>
diff --git a/other/java/hdfs2/dependency-reduced-pom.xml b/other/java/hdfs2/dependency-reduced-pom.xml
index 1b5a5c3fc..bd31637ce 100644
--- a/other/java/hdfs2/dependency-reduced-pom.xml
+++ b/other/java/hdfs2/dependency-reduced-pom.xml
@@ -301,7 +301,7 @@
</snapshotRepository>
</distributionManagement>
<properties>
- <seaweedfs.client.version>1.6.6</seaweedfs.client.version>
+ <seaweedfs.client.version>1.6.7</seaweedfs.client.version>
<hadoop.version>2.9.2</hadoop.version>
</properties>
</project>
diff --git a/other/java/hdfs2/pom.xml b/other/java/hdfs2/pom.xml
index 58e51a2a1..f15d24faa 100644
--- a/other/java/hdfs2/pom.xml
+++ b/other/java/hdfs2/pom.xml
@@ -5,7 +5,7 @@
<modelVersion>4.0.0</modelVersion>
<properties>
- <seaweedfs.client.version>1.6.6</seaweedfs.client.version>
+ <seaweedfs.client.version>1.6.7</seaweedfs.client.version>
<hadoop.version>2.9.2</hadoop.version>
</properties>
diff --git a/other/java/hdfs3/dependency-reduced-pom.xml b/other/java/hdfs3/dependency-reduced-pom.xml
index 58556a9c7..4640b5a84 100644
--- a/other/java/hdfs3/dependency-reduced-pom.xml
+++ b/other/java/hdfs3/dependency-reduced-pom.xml
@@ -309,7 +309,7 @@
</snapshotRepository>
</distributionManagement>
<properties>
- <seaweedfs.client.version>1.6.6</seaweedfs.client.version>
+ <seaweedfs.client.version>1.6.7</seaweedfs.client.version>
<hadoop.version>3.1.1</hadoop.version>
</properties>
</project>
diff --git a/other/java/hdfs3/pom.xml b/other/java/hdfs3/pom.xml
index bbcc1788d..efcc1e4c0 100644
--- a/other/java/hdfs3/pom.xml
+++ b/other/java/hdfs3/pom.xml
@@ -5,7 +5,7 @@
<modelVersion>4.0.0</modelVersion>
<properties>
- <seaweedfs.client.version>1.6.6</seaweedfs.client.version>
+ <seaweedfs.client.version>1.6.7</seaweedfs.client.version>
<hadoop.version>3.1.1</hadoop.version>
</properties>
diff --git a/other/metrics/grafana_seaweedfs.json b/other/metrics/grafana_seaweedfs.json
index 074a3531f..8fbd5989a 100644
--- a/other/metrics/grafana_seaweedfs.json
+++ b/other/metrics/grafana_seaweedfs.json
@@ -1805,6 +1805,336 @@
"show": true
}
]
+ },
+ {
+ "type": "graph",
+ "title": "Process memory",
+ "gridPos": {
+ "x": 0,
+ "y": 40,
+ "w": 24,
+ "h": 7
+ },
+ "aliasColors": {},
+ "dashLength": 10,
+ "datasource": "${DS_PROMETHEUS-DEV}",
+ "fieldConfig": {
+ "defaults": {
+ "custom": {},
+ "links": []
+ },
+ "overrides": []
+ },
+ "fill": 1,
+ "id": 66,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [],
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "paceLength": 10,
+ "pluginVersion": "7.4.5",
+ "pointradius": 5,
+ "renderer": "flot",
+ "seriesOverrides": [
+ {
+ "$$hashKey": "object:258",
+ "alias": "max limit",
+ "color": "#890f02"
+ }
+ ],
+ "spaceLength": 10,
+ "targets": [
+ {
+ "expr": "process_resident_memory_bytes{exported_job=\"filer\"}",
+ "legendFormat": "resident {{exported_instance}}",
+ "interval": "",
+ "format": "time_series",
+ "groupBy": [
+ {
+ "params": [
+ "$__interval"
+ ],
+ "type": "time"
+ },
+ {
+ "params": [
+ "null"
+ ],
+ "type": "fill"
+ }
+ ],
+ "intervalFactor": 2,
+ "orderByTime": "ASC",
+ "policy": "default",
+ "refId": "A",
+ "resultFormat": "time_series",
+ "select": [
+ [
+ {
+ "params": [
+ "value"
+ ],
+ "type": "field"
+ },
+ {
+ "params": [],
+ "type": "mean"
+ }
+ ]
+ ],
+ "step": 120,
+ "tags": []
+ },
+ {
+ "expr": "process_virtual_memory_bytes{exported_job=\"filer\"}",
+ "legendFormat": "virtual {{exported_instance}}",
+ "interval": "",
+ "format": "time_series",
+ "intervalFactor": 2,
+ "refId": "B",
+ "step": 120
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "tooltip": {
+ "shared": true,
+ "sort": 2,
+ "value_type": "individual"
+ },
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "$$hashKey": "object:265",
+ "format": "bytes",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ },
+ {
+ "$$hashKey": "object:266",
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false,
+ "alignLevel": null
+ },
+ "bars": false,
+ "dashes": false,
+ "fillGradient": 0,
+ "hiddenSeries": false,
+ "percentage": false,
+ "points": false,
+ "stack": false,
+ "steppedLine": false,
+ "timeFrom": null,
+ "timeShift": null
+ },
+ {
+ "type": "graph",
+ "title": "Open File Descriptor",
+ "gridPos": {
+ "x": 0,
+ "y": 47,
+ "w": 24,
+ "h": 7
+ },
+ "aliasColors": {},
+ "dashLength": 10,
+ "datasource": "${DS_PROMETHEUS-DEV}",
+ "editable": true,
+ "fieldConfig": {
+ "defaults": {
+ "custom": {},
+ "links": []
+ },
+ "overrides": []
+ },
+ "fill": 1,
+ "id": 68,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [],
+ "nullPointMode": "connected",
+ "options": {
+ "alertThreshold": true
+ },
+ "paceLength": 10,
+ "pluginVersion": "7.4.5",
+ "pointradius": 5,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "targets": [
+ {
+ "expr": "process_open_fds{exported_job=\"filer\"}",
+ "legendFormat": "{{exported_instance}}",
+ "interval": "",
+ "format": "time_series",
+ "groupBy": [
+ {
+ "params": [
+ "$interval"
+ ],
+ "type": "time"
+ },
+ {
+ "params": [
+ "null"
+ ],
+ "type": "fill"
+ }
+ ],
+ "intervalFactor": 2,
+ "metric": "",
+ "policy": "default",
+ "refId": "A",
+ "resultFormat": "time_series",
+ "select": [
+ [
+ {
+ "params": [
+ "value"
+ ],
+ "type": "field"
+ },
+ {
+ "params": [],
+ "type": "mean"
+ }
+ ]
+ ],
+ "step": 240,
+ "tags": []
+ },
+ {
+ "expr": "process_max_fds{exported_job=\"filer\"}",
+ "legendFormat": "{{exported_instance}}",
+ "interval": "",
+ "format": "time_series",
+ "groupBy": [
+ {
+ "params": [
+ "$interval"
+ ],
+ "type": "time"
+ },
+ {
+ "params": [
+ "null"
+ ],
+ "type": "fill"
+ }
+ ],
+ "intervalFactor": 2,
+ "metric": "",
+ "policy": "default",
+ "refId": "B",
+ "resultFormat": "time_series",
+ "select": [
+ [
+ {
+ "params": [
+ "value"
+ ],
+ "type": "field"
+ },
+ {
+ "params": [],
+ "type": "mean"
+ }
+ ]
+ ],
+ "step": 240,
+ "tags": [],
+ "hide": false
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "tooltip": {
+ "msResolution": false,
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "xaxis": {
+ "buckets": null,
+ "mode": "time",
+ "name": null,
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "$$hashKey": "object:1087",
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ },
+ {
+ "$$hashKey": "object:1088",
+ "format": "short",
+ "label": null,
+ "logBase": 1,
+ "max": null,
+ "min": null,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false,
+ "alignLevel": null
+ },
+ "bars": false,
+ "dashes": false,
+ "error": false,
+ "fillGradient": 0,
+ "hiddenSeries": false,
+ "percentage": false,
+ "points": false,
+ "stack": false,
+ "steppedLine": false,
+ "timeFrom": null,
+ "timeShift": null
}
],
"repeat": null,
generated by cgit v1.2.3 (git 2.25.1) at 2025-12-24 08:14:26 +0000