diff options
Diffstat (limited to 'other/java/client/src')
6 files changed, 92 insertions, 59 deletions
diff --git a/other/java/client/src/main/java/seaweedfs/client/FilerClient.java b/other/java/client/src/main/java/seaweedfs/client/FilerClient.java index 0a8356258..e962cbbcc 100644 --- a/other/java/client/src/main/java/seaweedfs/client/FilerClient.java +++ b/other/java/client/src/main/java/seaweedfs/client/FilerClient.java @@ -4,7 +4,6 @@ import com.google.common.base.Strings; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.File; import java.util.ArrayList; import java.util.Arrays; import java.util.Iterator; @@ -59,24 +58,39 @@ public class FilerClient extends FilerGrpcClient { public static FilerProto.Entry afterEntryDeserialization(FilerProto.Entry entry) { if (entry.getChunksList().size() <= 0) { + if (entry.getContent().isEmpty()) { + return entry; + } else { + if (entry.getAttributes().getFileSize() <= 0) { + FilerProto.Entry.Builder entryBuilder = entry.toBuilder(); + FilerProto.FuseAttributes.Builder attrBuilder = entry.getAttributes().toBuilder(); + attrBuilder.setFileSize(entry.getContent().size()); + entryBuilder.setAttributes(attrBuilder); + return entryBuilder.build(); + } + } return entry; - } - String fileId = entry.getChunks(0).getFileId(); - if (fileId != null && fileId.length() != 0) { - return entry; - } - FilerProto.Entry.Builder entryBuilder = entry.toBuilder(); - entryBuilder.clearChunks(); - for (FilerProto.FileChunk chunk : entry.getChunksList()) { - FilerProto.FileChunk.Builder chunkBuilder = chunk.toBuilder(); - chunkBuilder.setFileId(toFileId(chunk.getFid())); - String sourceFileId = toFileId(chunk.getSourceFid()); - if (sourceFileId != null) { - chunkBuilder.setSourceFileId(sourceFileId); + } else { + FilerProto.Entry.Builder entryBuilder = entry.toBuilder(); + entryBuilder.clearChunks(); + long fileSize = 0; + for (FilerProto.FileChunk chunk : entry.getChunksList()) { + fileSize = Math.max(fileSize, chunk.getOffset()+chunk.getSize()); + FilerProto.FileChunk.Builder chunkBuilder = chunk.toBuilder(); + chunkBuilder.setFileId(toFileId(chunk.getFid())); + String sourceFileId = toFileId(chunk.getSourceFid()); + if (sourceFileId != null) { + chunkBuilder.setSourceFileId(sourceFileId); + } + entryBuilder.addChunks(chunkBuilder); + } + if (entry.getAttributes().getFileSize() <= 0) { + FilerProto.FuseAttributes.Builder attrBuilder = entry.getAttributes().toBuilder(); + attrBuilder.setFileSize(fileSize); + entryBuilder.setAttributes(attrBuilder); } - entryBuilder.addChunks(chunkBuilder); + return entryBuilder.build(); } - return entryBuilder.build(); } public boolean mkdirs(String path, int mode) { @@ -93,9 +107,9 @@ public class FilerClient extends FilerGrpcClient { if ("/".equals(path)) { return true; } - File pathFile = new File(path); - String parent = pathFile.getParent().replace('\\','/'); - String name = pathFile.getName(); + String[] dirAndName = SeaweedUtil.toDirAndName(path); + String parent = dirAndName[0]; + String name = dirAndName[1]; mkdirs(parent, mode, uid, gid, userName, groupNames); @@ -114,35 +128,32 @@ public class FilerClient extends FilerGrpcClient { public boolean mv(String oldPath, String newPath) { - File oldPathFile = new File(oldPath); - String oldParent = oldPathFile.getParent().replace('\\','/'); - String oldName = oldPathFile.getName(); + String[] oldDirAndName = SeaweedUtil.toDirAndName(oldPath); + String oldParent = oldDirAndName[0]; + String oldName = oldDirAndName[1]; - File newPathFile = new File(newPath); - String newParent = newPathFile.getParent().replace('\\','/'); - String newName = newPathFile.getName(); + String[] newDirAndName = SeaweedUtil.toDirAndName(newPath); + String newParent = newDirAndName[0]; + String newName = newDirAndName[1]; return atomicRenameEntry(oldParent, oldName, newParent, newName); } public boolean exists(String path){ - File pathFile = new File(path); - String parent = pathFile.getParent(); - String entryName = pathFile.getName(); - if(parent == null) { - parent = path; - entryName =""; - } - return lookupEntry(parent, entryName) != null; + String[] dirAndName = SeaweedUtil.toDirAndName(path); + String parent = dirAndName[0]; + String entryName = dirAndName[1]; + + return lookupEntry(parent, entryName) != null; } public boolean rm(String path, boolean isRecursive, boolean ignoreRecusiveError) { - File pathFile = new File(path); - String parent = pathFile.getParent().replace('\\','/'); - String name = pathFile.getName(); + String[] dirAndName = SeaweedUtil.toDirAndName(path); + String parent = dirAndName[0]; + String name = dirAndName[1]; return deleteEntry( parent, @@ -153,17 +164,19 @@ public class FilerClient extends FilerGrpcClient { } public boolean touch(String path, int mode) { + String currentUser = System.getProperty("user.name"); long now = System.currentTimeMillis() / 1000L; return touch(path, now, mode, 0, 0, currentUser, new String[]{}); + } public boolean touch(String path, long modifiedTimeSecond, int mode, int uid, int gid, String userName, String[] groupNames) { - File pathFile = new File(path); - String parent = pathFile.getParent().replace('\\','/'); - String name = pathFile.getName(); + String[] dirAndName = SeaweedUtil.toDirAndName(path); + String parent = dirAndName[0]; + String name = dirAndName[1]; FilerProto.Entry entry = lookupEntry(parent, name); if (entry == null) { diff --git a/other/java/client/src/main/java/seaweedfs/client/FilerGrpcClient.java b/other/java/client/src/main/java/seaweedfs/client/FilerGrpcClient.java index 6c57e2e0d..56aa35876 100644 --- a/other/java/client/src/main/java/seaweedfs/client/FilerGrpcClient.java +++ b/other/java/client/src/main/java/seaweedfs/client/FilerGrpcClient.java @@ -54,7 +54,7 @@ public class FilerGrpcClient { .negotiationType(NegotiationType.TLS) .sslContext(sslContext)); - filerAddress = String.format("%s:%d", host, grpcPort - 10000); + filerAddress = SeaweedUtil.joinHostPort(host, grpcPort - 10000); FilerProto.GetFilerConfigurationResponse filerConfigurationResponse = this.getBlockingStub().getFilerConfiguration( diff --git a/other/java/client/src/main/java/seaweedfs/client/SeaweedRead.java b/other/java/client/src/main/java/seaweedfs/client/SeaweedRead.java index 384636601..011462a17 100644 --- a/other/java/client/src/main/java/seaweedfs/client/SeaweedRead.java +++ b/other/java/client/src/main/java/seaweedfs/client/SeaweedRead.java @@ -64,10 +64,11 @@ public class SeaweedRead { startOffset += gap; } - FilerProto.Locations locations = knownLocations.get(parseVolumeId(chunkView.fileId)); + String volumeId = parseVolumeId(chunkView.fileId); + FilerProto.Locations locations = knownLocations.get(volumeId); if (locations == null || locations.getLocationsCount() == 0) { LOG.error("failed to locate {}", chunkView.fileId); - // log here! + volumeIdCache.clearLocations(volumeId); return 0; } diff --git a/other/java/client/src/main/java/seaweedfs/client/SeaweedUtil.java b/other/java/client/src/main/java/seaweedfs/client/SeaweedUtil.java index c465d935f..027e49b96 100644 --- a/other/java/client/src/main/java/seaweedfs/client/SeaweedUtil.java +++ b/other/java/client/src/main/java/seaweedfs/client/SeaweedUtil.java @@ -27,4 +27,30 @@ public class SeaweedUtil { public static CloseableHttpClient getClosableHttpClient() { return httpClient; } + + public static String[] toDirAndName(String fullpath) { + if (fullpath == null) { + return new String[]{"/", ""}; + } + if (fullpath.endsWith("/")) { + fullpath = fullpath.substring(0, fullpath.length() - 1); + } + if (fullpath.length() == 0) { + return new String[]{"/", ""}; + } + int sep = fullpath.lastIndexOf("/"); + String parent = sep == 0 ? "/" : fullpath.substring(0, sep); + String name = fullpath.substring(sep + 1); + return new String[]{parent, name}; + } + + public static String joinHostPort(String host, int port) { + if (host.startsWith("[") && host.endsWith("]")) { + return host + ":" + port; + } + if (host.indexOf(':')>=0) { + return "[" + host + "]:" + port; + } + return host + ":" + port; + } } diff --git a/other/java/client/src/main/java/seaweedfs/client/VolumeIdCache.java b/other/java/client/src/main/java/seaweedfs/client/VolumeIdCache.java index fd2649cc2..86263fff9 100644 --- a/other/java/client/src/main/java/seaweedfs/client/VolumeIdCache.java +++ b/other/java/client/src/main/java/seaweedfs/client/VolumeIdCache.java @@ -26,6 +26,13 @@ public class VolumeIdCache { return this.cache.getIfPresent(volumeId); } + public void clearLocations(String volumeId) { + if (this.cache == null) { + return; + } + this.cache.invalidate(volumeId); + } + public void setLocations(String volumeId, FilerProto.Locations locations) { if (this.cache == null) { return; diff --git a/other/java/client/src/main/proto/filer.proto b/other/java/client/src/main/proto/filer.proto index 1c22276ae..7b1838565 100644 --- a/other/java/client/src/main/proto/filer.proto +++ b/other/java/client/src/main/proto/filer.proto @@ -305,6 +305,7 @@ message GetFilerConfigurationResponse { string metrics_address = 9; int32 metrics_interval_sec = 10; string version = 11; + string cluster_id = 12; } message SubscribeMetadataRequest { @@ -312,6 +313,7 @@ message SubscribeMetadataRequest { string path_prefix = 2; int64 since_ns = 3; int32 signature = 4; + repeated string path_prefixes = 6; } message SubscribeMetadataResponse { string directory = 1; @@ -336,6 +338,7 @@ message KeepConnectedResponse { message LocateBrokerRequest { string resource = 1; } + message LocateBrokerResponse { bool found = 1; // if found, send the exact address @@ -386,23 +389,6 @@ message FilerConf { ///////////////////////// // Remote Storage related ///////////////////////// -message RemoteConf { - string type = 1; - string name = 2; - string s3_access_key = 4; - string s3_secret_key = 5; - string s3_region = 6; - string s3_endpoint = 7; -} - -message RemoteStorageMapping { - map<string,RemoteStorageLocation> mappings = 1; -} -message RemoteStorageLocation { - string name = 1; - string bucket = 2; - string path = 3; -} message DownloadToLocalRequest { string directory = 1; string name = 2; |