aboutsummaryrefslogtreecommitdiff
path: root/go/filer
diff options
context:
space:
mode:
Diffstat (limited to 'go/filer')
-rw-r--r--go/filer/cassandra_store/cassandra_store.go87
-rw-r--r--go/filer/cassandra_store/schema.cql22
-rw-r--r--go/filer/client_operations.go70
-rw-r--r--go/filer/embedded_filer/design.txt26
-rw-r--r--go/filer/embedded_filer/directory.go15
-rw-r--r--go/filer/embedded_filer/directory_in_map.go310
-rw-r--r--go/filer/embedded_filer/directory_test.go86
-rw-r--r--go/filer/embedded_filer/filer_embedded.go141
-rw-r--r--go/filer/embedded_filer/files_in_leveldb.go85
-rw-r--r--go/filer/filer.go28
-rw-r--r--go/filer/flat_namespace/flat_namespace_filer.go50
-rw-r--r--go/filer/flat_namespace/flat_namespace_store.go9
-rw-r--r--go/filer/redis_store/redis_store.go48
13 files changed, 0 insertions, 977 deletions
diff --git a/go/filer/cassandra_store/cassandra_store.go b/go/filer/cassandra_store/cassandra_store.go
deleted file mode 100644
index 1c2be8a8e..000000000
--- a/go/filer/cassandra_store/cassandra_store.go
+++ /dev/null
@@ -1,87 +0,0 @@
-package cassandra_store
-
-import (
- "fmt"
-
- "github.com/chrislusf/seaweedfs/go/glog"
-
- "github.com/gocql/gocql"
-)
-
-/*
-
-Basically you need a table just like this:
-
-CREATE TABLE seaweed_files (
- path varchar,
- fids list<varchar>,
- PRIMARY KEY (path)
-);
-
-Need to match flat_namespace.FlatNamespaceStore interface
- Put(fullFileName string, fid string) (err error)
- Get(fullFileName string) (fid string, err error)
- Delete(fullFileName string) (fid string, err error)
-
-*/
-type CassandraStore struct {
- cluster *gocql.ClusterConfig
- session *gocql.Session
-}
-
-func NewCassandraStore(keyspace string, hosts ...string) (c *CassandraStore, err error) {
- c = &CassandraStore{}
- c.cluster = gocql.NewCluster(hosts...)
- c.cluster.Keyspace = keyspace
- c.cluster.Consistency = gocql.Quorum
- c.session, err = c.cluster.CreateSession()
- if err != nil {
- glog.V(0).Infof("Failed to open cassandra store, hosts %v, keyspace %s", hosts, keyspace)
- }
- return
-}
-
-func (c *CassandraStore) Put(fullFileName string, fid string) (err error) {
- var input []string
- input = append(input, fid)
- if err := c.session.Query(
- `INSERT INTO seaweed_files (path, fids) VALUES (?, ?)`,
- fullFileName, input).Exec(); err != nil {
- glog.V(0).Infof("Failed to save file %s with id %s: %v", fullFileName, fid, err)
- return err
- }
- return nil
-}
-func (c *CassandraStore) Get(fullFileName string) (fid string, err error) {
- var output []string
- if err := c.session.Query(
- `select fids FROM seaweed_files WHERE path = ? LIMIT 1`,
- fullFileName).Consistency(gocql.One).Scan(&output); err != nil {
- if err != gocql.ErrNotFound {
- glog.V(0).Infof("Failed to find file %s: %v", fullFileName, fid, err)
- }
- }
- if len(output) == 0 {
- return "", fmt.Errorf("No file id found for %s", fullFileName)
- }
- return output[0], nil
-}
-
-// Currently the fid is not returned
-func (c *CassandraStore) Delete(fullFileName string) (fid string, err error) {
- if err := c.session.Query(
- `DELETE FROM seaweed_files WHERE path = ?`,
- fullFileName).Exec(); err != nil {
- if err != gocql.ErrNotFound {
- glog.V(0).Infof("Failed to delete file %s: %v", fullFileName, err)
- }
- return "", err
- }
- return "", nil
-}
-
-func (c *CassandraStore) Close() {
- if c.session != nil {
- c.session.Close()
- }
-}
diff --git a/go/filer/cassandra_store/schema.cql b/go/filer/cassandra_store/schema.cql
deleted file mode 100644
index d6f2bb093..000000000
--- a/go/filer/cassandra_store/schema.cql
+++ /dev/null
@@ -1,22 +0,0 @@
-/*
-
-Here is the CQL to create the table.CassandraStore
-
-Optionally you can adjust the keyspace name and replication settings.
-
-For production server, very likely you want to set replication_factor to 3
-
-*/
-
-create keyspace seaweed WITH replication = {
- 'class':'SimpleStrategy',
- 'replication_factor':1
-};
-
-use seaweed;
-
-CREATE TABLE seaweed_files (
- path varchar,
- fids list<varchar>,
- PRIMARY KEY (path)
-);
diff --git a/go/filer/client_operations.go b/go/filer/client_operations.go
deleted file mode 100644
index 80ac51693..000000000
--- a/go/filer/client_operations.go
+++ /dev/null
@@ -1,70 +0,0 @@
-package filer
-
-import (
- "encoding/json"
- "errors"
- "fmt"
-
- "github.com/chrislusf/seaweedfs/go/util"
-
- "net/url"
-)
-
-type ApiRequest struct {
- Command string //"listFiles", "listDirectories"
- Directory string
- FileName string
-}
-
-type ListFilesResult struct {
- Files []FileEntry
- Error string `json:"error,omitempty"`
-}
-
-func ListFiles(server string, directory string, fileName string) (*ListFilesResult, error) {
- var ret ListFilesResult
- if err := call(server, ApiRequest{Command: "listFiles", Directory: directory, FileName: fileName}, &ret); err == nil {
- if ret.Error != "" {
- return nil, errors.New(ret.Error)
- }
- return &ret, nil
- } else {
- return nil, err
- }
-}
-
-type ListDirectoriesResult struct {
- Directories []DirectoryEntry
- Error string `json:"error,omitempty"`
-}
-
-func ListDirectories(server string, directory string) (*ListDirectoriesResult, error) {
- var ret ListDirectoriesResult
- if err := call(server, ApiRequest{Command: "listDirectories", Directory: directory}, &ret); err == nil {
- if ret.Error != "" {
- return nil, errors.New(ret.Error)
- }
- return &ret, nil
- } else {
- return nil, err
- }
-}
-
-func call(server string, request ApiRequest, ret interface{}) error {
- b, err := json.Marshal(request)
- if err != nil {
- fmt.Println("error:", err)
- return nil
- }
- values := make(url.Values)
- values.Add("request", string(b))
- jsonBlob, err := util.Post("http://"+server+"/__api__", values)
- if err != nil {
- return err
- }
- err = json.Unmarshal(jsonBlob, ret)
- if err != nil {
- return err
- }
- return nil
-}
diff --git a/go/filer/embedded_filer/design.txt b/go/filer/embedded_filer/design.txt
deleted file mode 100644
index 45fec8fbe..000000000
--- a/go/filer/embedded_filer/design.txt
+++ /dev/null
@@ -1,26 +0,0 @@
-Design Assumptions:
-1. the number of directories are magnitudely smaller than the number of files
-2. unlimited number of files under any directories
-Phylosophy:
- metadata for directories and files should be separated
-Design:
- Store directories in normal map
- all of directories hopefully all be in memory
- efficient to move/rename/list_directories
- Log directory changes to append only log file
- Store files in sorted string table in <dir_id/filename> format
- efficient to list_files, just simple iterator
- efficient to locate files, binary search
-
-Testing:
-1. starting server, "weed server -filer=true"
-2. posting files to different folders
-curl -F "filename=@design.txt" "http://localhost:8888/sources/"
-curl -F "filename=@design.txt" "http://localhost:8888/design/"
-curl -F "filename=@directory.go" "http://localhost:8888/sources/weed/go/"
-curl -F "filename=@directory.go" "http://localhost:8888/sources/testing/go/"
-curl -F "filename=@filer.go" "http://localhost:8888/sources/weed/go/"
-curl -F "filename=@filer_in_leveldb.go" "http://localhost:8888/sources/weed/go/"
-curl "http://localhost:8888/?pretty=y"
-curl "http://localhost:8888/sources/weed/go/?pretty=y"
-curl "http://localhost:8888/sources/weed/go/?pretty=y"
diff --git a/go/filer/embedded_filer/directory.go b/go/filer/embedded_filer/directory.go
deleted file mode 100644
index 7c50e7185..000000000
--- a/go/filer/embedded_filer/directory.go
+++ /dev/null
@@ -1,15 +0,0 @@
-package embedded_filer
-
-import (
- "github.com/chrislusf/seaweedfs/go/filer"
-)
-
-type DirectoryManager interface {
- FindDirectory(dirPath string) (filer.DirectoryId, error)
- ListDirectories(dirPath string) (dirs []filer.DirectoryEntry, err error)
- MakeDirectory(currentDirPath string, dirName string) (filer.DirectoryId, error)
- MoveUnderDirectory(oldDirPath string, newParentDirPath string) error
- DeleteDirectory(dirPath string) error
- //functions used by FUSE
- FindDirectoryById(filer.DirectoryId, error)
-}
diff --git a/go/filer/embedded_filer/directory_in_map.go b/go/filer/embedded_filer/directory_in_map.go
deleted file mode 100644
index c67922541..000000000
--- a/go/filer/embedded_filer/directory_in_map.go
+++ /dev/null
@@ -1,310 +0,0 @@
-package embedded_filer
-
-import (
- "bufio"
- "fmt"
- "io"
- "os"
- "path/filepath"
- "strconv"
- "strings"
- "sync"
-
- "github.com/chrislusf/seaweedfs/go/filer"
- "github.com/chrislusf/seaweedfs/go/util"
-)
-
-var writeLock sync.Mutex //serialize changes to dir.log
-
-type DirectoryEntryInMap struct {
- sync.Mutex
- Name string
- Parent *DirectoryEntryInMap
- subDirectories map[string]*DirectoryEntryInMap
- Id filer.DirectoryId
-}
-
-func (de *DirectoryEntryInMap) getChild(dirName string) (*DirectoryEntryInMap, bool) {
- de.Lock()
- defer de.Unlock()
- child, ok := de.subDirectories[dirName]
- return child, ok
-}
-func (de *DirectoryEntryInMap) addChild(dirName string, child *DirectoryEntryInMap) {
- de.Lock()
- defer de.Unlock()
- de.subDirectories[dirName] = child
-}
-func (de *DirectoryEntryInMap) removeChild(dirName string) {
- de.Lock()
- defer de.Unlock()
- delete(de.subDirectories, dirName)
-}
-func (de *DirectoryEntryInMap) hasChildren() bool {
- de.Lock()
- defer de.Unlock()
- return len(de.subDirectories) > 0
-}
-func (de *DirectoryEntryInMap) children() (dirNames []filer.DirectoryEntry) {
- de.Lock()
- defer de.Unlock()
- for k, v := range de.subDirectories {
- dirNames = append(dirNames, filer.DirectoryEntry{Name: k, Id: v.Id})
- }
- return dirNames
-}
-
-type DirectoryManagerInMap struct {
- Root *DirectoryEntryInMap
- max filer.DirectoryId
- logFile *os.File
- isLoading bool
-}
-
-func (dm *DirectoryManagerInMap) newDirectoryEntryInMap(parent *DirectoryEntryInMap, name string) (d *DirectoryEntryInMap, err error) {
- d = &DirectoryEntryInMap{Name: name, Parent: parent, subDirectories: make(map[string]*DirectoryEntryInMap)}
- var parts []string
- for p := d; p != nil && p.Name != ""; p = p.Parent {
- parts = append(parts, p.Name)
- }
- n := len(parts)
- if n <= 0 {
- return nil, fmt.Errorf("Failed to create folder %s/%s", parent.Name, name)
- }
- for i := 0; i < n/2; i++ {
- parts[i], parts[n-1-i] = parts[n-1-i], parts[i]
- }
- dm.max++
- d.Id = dm.max
- dm.log("add", "/"+strings.Join(parts, "/"), strconv.Itoa(int(d.Id)))
- return d, nil
-}
-
-func (dm *DirectoryManagerInMap) log(words ...string) {
- if !dm.isLoading {
- dm.logFile.WriteString(strings.Join(words, "\t") + "\n")
- }
-}
-
-func NewDirectoryManagerInMap(dirLogFile string) (dm *DirectoryManagerInMap, err error) {
- dm = &DirectoryManagerInMap{}
- //dm.Root do not use newDirectoryEntryInMap, since dm.max will be changed
- dm.Root = &DirectoryEntryInMap{subDirectories: make(map[string]*DirectoryEntryInMap)}
- if dm.logFile, err = os.OpenFile(dirLogFile, os.O_RDWR|os.O_CREATE, 0644); err != nil {
- return nil, fmt.Errorf("cannot write directory log file %s: %v", dirLogFile, err)
- }
- return dm, dm.load()
-}
-
-func (dm *DirectoryManagerInMap) processEachLine(line string) error {
- if strings.HasPrefix(line, "#") {
- return nil
- }
- if line == "" {
- return nil
- }
- parts := strings.Split(line, "\t")
- if len(parts) == 0 {
- return nil
- }
- switch parts[0] {
- case "add":
- v, pe := strconv.Atoi(parts[2])
- if pe != nil {
- return pe
- }
- if e := dm.loadDirectory(parts[1], filer.DirectoryId(v)); e != nil {
- return e
- }
- case "mov":
- newName := ""
- if len(parts) >= 4 {
- newName = parts[3]
- }
- if e := dm.MoveUnderDirectory(parts[1], parts[2], newName); e != nil {
- return e
- }
- case "del":
- if e := dm.DeleteDirectory(parts[1]); e != nil {
- return e
- }
- default:
- fmt.Printf("line %s has %s!\n", line, parts[0])
- return nil
- }
- return nil
-}
-func (dm *DirectoryManagerInMap) load() error {
- dm.max = 0
- lines := bufio.NewReader(dm.logFile)
- dm.isLoading = true
- defer func() { dm.isLoading = false }()
- for {
- line, err := util.Readln(lines)
- if err != nil && err != io.EOF {
- return err
- }
- if pe := dm.processEachLine(string(line)); pe != nil {
- return pe
- }
- if err == io.EOF {
- return nil
- }
- }
-}
-
-func (dm *DirectoryManagerInMap) findDirectory(dirPath string) (*DirectoryEntryInMap, error) {
- if dirPath == "" {
- return dm.Root, nil
- }
- dirPath = CleanFilePath(dirPath)
- if dirPath == "/" {
- return dm.Root, nil
- }
- parts := strings.Split(dirPath, "/")
- dir := dm.Root
- for i := 1; i < len(parts); i++ {
- if sub, ok := dir.getChild(parts[i]); ok {
- dir = sub
- } else {
- return dm.Root, fmt.Errorf("Directory %s Not Found", dirPath)
- }
- }
- return dir, nil
-}
-func (dm *DirectoryManagerInMap) FindDirectory(dirPath string) (filer.DirectoryId, error) {
- d, e := dm.findDirectory(dirPath)
- if e == nil {
- return d.Id, nil
- }
- return dm.Root.Id, e
-}
-
-func (dm *DirectoryManagerInMap) loadDirectory(dirPath string, dirId filer.DirectoryId) error {
- dirPath = CleanFilePath(dirPath)
- if dirPath == "/" {
- return nil
- }
- parts := strings.Split(dirPath, "/")
- dir := dm.Root
- for i := 1; i < len(parts); i++ {
- sub, ok := dir.getChild(parts[i])
- if !ok {
- writeLock.Lock()
- if sub2, createdByOtherThread := dir.getChild(parts[i]); createdByOtherThread {
- sub = sub2
- } else {
- if i != len(parts)-1 {
- writeLock.Unlock()
- return fmt.Errorf("%s should be created after parent %s", dirPath, parts[i])
- }
- var err error
- sub, err = dm.newDirectoryEntryInMap(dir, parts[i])
- if err != nil {
- writeLock.Unlock()
- return err
- }
- if sub.Id != dirId {
- writeLock.Unlock()
- // the dir.log should be the same order as in-memory directory id
- return fmt.Errorf("%s should be have id %v instead of %v", dirPath, sub.Id, dirId)
- }
- dir.addChild(parts[i], sub)
- }
- writeLock.Unlock()
- }
- dir = sub
- }
- return nil
-}
-
-func (dm *DirectoryManagerInMap) makeDirectory(dirPath string) (dir *DirectoryEntryInMap, created bool) {
- dirPath = CleanFilePath(dirPath)
- if dirPath == "/" {
- return dm.Root, false
- }
- parts := strings.Split(dirPath, "/")
- dir = dm.Root
- for i := 1; i < len(parts); i++ {
- sub, ok := dir.getChild(parts[i])
- if !ok {
- writeLock.Lock()
- if sub2, createdByOtherThread := dir.getChild(parts[i]); createdByOtherThread {
- sub = sub2
- } else {
- var err error
- sub, err = dm.newDirectoryEntryInMap(dir, parts[i])
- if err != nil {
- writeLock.Unlock()
- return nil, false
- }
- dir.addChild(parts[i], sub)
- created = true
- }
- writeLock.Unlock()
- }
- dir = sub
- }
- return dir, created
-}
-
-func (dm *DirectoryManagerInMap) MakeDirectory(dirPath string) (filer.DirectoryId, error) {
- dir, _ := dm.makeDirectory(dirPath)
- return dir.Id, nil
-}
-
-func (dm *DirectoryManagerInMap) MoveUnderDirectory(oldDirPath string, newParentDirPath string, newName string) error {
- writeLock.Lock()
- defer writeLock.Unlock()
- oldDir, oe := dm.findDirectory(oldDirPath)
- if oe != nil {
- return oe
- }
- parentDir, pe := dm.findDirectory(newParentDirPath)
- if pe != nil {
- return pe
- }
- dm.log("mov", oldDirPath, newParentDirPath, newName)
- oldDir.Parent.removeChild(oldDir.Name)
- if newName == "" {
- newName = oldDir.Name
- }
- parentDir.addChild(newName, oldDir)
- oldDir.Name = newName
- oldDir.Parent = parentDir
- return nil
-}
-
-func (dm *DirectoryManagerInMap) ListDirectories(dirPath string) (dirNames []filer.DirectoryEntry, err error) {
- d, e := dm.findDirectory(dirPath)
- if e != nil {
- return dirNames, e
- }
- return d.children(), nil
-}
-func (dm *DirectoryManagerInMap) DeleteDirectory(dirPath string) error {
- writeLock.Lock()
- defer writeLock.Unlock()
- if dirPath == "/" {
- return fmt.Errorf("Can not delete %s", dirPath)
- }
- d, e := dm.findDirectory(dirPath)
- if e != nil {
- return e
- }
- if d.hasChildren() {
- return fmt.Errorf("dir %s still has sub directories", dirPath)
- }
- d.Parent.removeChild(d.Name)
- d.Parent = nil
- dm.log("del", dirPath)
- return nil
-}
-
-func CleanFilePath(fp string) string {
- ret := filepath.Clean(fp)
- if os.PathSeparator == '\\' {
- return strings.Replace(ret, "\\", "/", -1)
- }
- return ret
-}
diff --git a/go/filer/embedded_filer/directory_test.go b/go/filer/embedded_filer/directory_test.go
deleted file mode 100644
index c8b3f1f30..000000000
--- a/go/filer/embedded_filer/directory_test.go
+++ /dev/null
@@ -1,86 +0,0 @@
-package embedded_filer
-
-import (
- "os"
- "strings"
- "testing"
-)
-
-func TestDirectory(t *testing.T) {
- dm, _ := NewDirectoryManagerInMap("/tmp/dir.log")
- defer func() {
- if true {
- os.Remove("/tmp/dir.log")
- }
- }()
- dm.MakeDirectory("/a/b/c")
- dm.MakeDirectory("/a/b/d")
- dm.MakeDirectory("/a/b/e")
- dm.MakeDirectory("/a/b/e/f")
- dm.MakeDirectory("/a/b/e/f/g")
- dm.MoveUnderDirectory("/a/b/e/f/g", "/a/b", "t")
- if _, err := dm.FindDirectory("/a/b/e/f/g"); err == nil {
- t.Fatal("/a/b/e/f/g should not exist any more after moving")
- }
- if _, err := dm.FindDirectory("/a/b/t"); err != nil {
- t.Fatal("/a/b/t should exist after moving")
- }
- if _, err := dm.FindDirectory("/a/b/g"); err == nil {
- t.Fatal("/a/b/g should not exist after moving")
- }
- dm.MoveUnderDirectory("/a/b/e/f", "/a/b", "")
- if _, err := dm.FindDirectory("/a/b/f"); err != nil {
- t.Fatal("/a/b/g should not exist after moving")
- }
- dm.MakeDirectory("/a/b/g/h/i")
- dm.DeleteDirectory("/a/b/e/f")
- dm.DeleteDirectory("/a/b/e")
- dirNames, _ := dm.ListDirectories("/a/b/e")
- for _, v := range dirNames {
- println("sub1 dir:", v.Name, "id", v.Id)
- }
- dm.logFile.Close()
-
- var path []string
- printTree(dm.Root, path)
-
- dm2, e := NewDirectoryManagerInMap("/tmp/dir.log")
- if e != nil {
- println("load error", e.Error())
- }
- if !compare(dm.Root, dm2.Root) {
- t.Fatal("restored dir not the same!")
- }
- printTree(dm2.Root, path)
-}
-
-func printTree(node *DirectoryEntryInMap, path []string) {
- println(strings.Join(path, "/") + "/" + node.Name)
- path = append(path, node.Name)
- for _, v := range node.subDirectories {
- printTree(v, path)
- }
-}
-
-func compare(root1 *DirectoryEntryInMap, root2 *DirectoryEntryInMap) bool {
- if len(root1.subDirectories) != len(root2.subDirectories) {
- return false
- }
- if root1.Name != root2.Name {
- return false
- }
- if root1.Id != root2.Id {
- return false
- }
- if !(root1.Parent == nil && root2.Parent == nil) {
- if root1.Parent.Id != root2.Parent.Id {
- return false
- }
- }
- for k, v := range root1.subDirectories {
- if !compare(v, root2.subDirectories[k]) {
- return false
- }
- }
- return true
-}
diff --git a/go/filer/embedded_filer/filer_embedded.go b/go/filer/embedded_filer/filer_embedded.go
deleted file mode 100644
index f467334de..000000000
--- a/go/filer/embedded_filer/filer_embedded.go
+++ /dev/null
@@ -1,141 +0,0 @@
-package embedded_filer
-
-import (
- "errors"
- "fmt"
- "path/filepath"
- "strings"
-
- "github.com/chrislusf/seaweedfs/go/filer"
- "github.com/chrislusf/seaweedfs/go/operation"
-)
-
-type FilerEmbedded struct {
- master string
- directories *DirectoryManagerInMap
- files *FileListInLevelDb
-}
-
-func NewFilerEmbedded(master string, dir string) (filer *FilerEmbedded, err error) {
- dm, de := NewDirectoryManagerInMap(filepath.Join(dir, "dir.log"))
- if de != nil {
- return nil, de
- }
- fl, fe := NewFileListInLevelDb(dir)
- if fe != nil {
- return nil, fe
- }
- filer = &FilerEmbedded{
- master: master,
- directories: dm,
- files: fl,
- }
- return
-}
-
-func (filer *FilerEmbedded) CreateFile(filePath string, fid string) (err error) {
- dir, file := filepath.Split(filePath)
- dirId, e := filer.directories.MakeDirectory(dir)
- if e != nil {
- return e
- }
- return filer.files.CreateFile(dirId, file, fid)
-}
-func (filer *FilerEmbedded) FindFile(filePath string) (fid string, err error) {
- dir, file := filepath.Split(filePath)
- dirId, e := filer.directories.FindDirectory(dir)
- if e != nil {
- return "", e
- }
- return filer.files.FindFile(dirId, file)
-}
-func (filer *FilerEmbedded) FindDirectory(dirPath string) (dirId filer.DirectoryId, err error) {
- return filer.directories.FindDirectory(dirPath)
-}
-func (filer *FilerEmbedded) ListDirectories(dirPath string) (dirs []filer.DirectoryEntry, err error) {
- return filer.directories.ListDirectories(dirPath)
-}
-func (filer *FilerEmbedded) ListFiles(dirPath string, lastFileName string, limit int) (files []filer.FileEntry, err error) {
- dirId, e := filer.directories.FindDirectory(dirPath)
- if e != nil {
- return nil, e
- }
- return filer.files.ListFiles(dirId, lastFileName, limit), nil
-}
-func (filer *FilerEmbedded) DeleteDirectory(dirPath string, recursive bool) (err error) {
- dirId, e := filer.directories.FindDirectory(dirPath)
- if e != nil {
- return e
- }
- if sub_dirs, sub_err := filer.directories.ListDirectories(dirPath); sub_err == nil {
- if len(sub_dirs) > 0 && !recursive {
- return fmt.Errorf("Fail to delete directory %s: %d sub directories found!", dirPath, len(sub_dirs))
- }
- for _, sub := range sub_dirs {
- if delete_sub_err := filer.DeleteDirectory(filepath.Join(dirPath, sub.Name), recursive); delete_sub_err != nil {
- return delete_sub_err
- }
- }
- }
- list := filer.files.ListFiles(dirId, "", 100)
- if len(list) != 0 && !recursive {
- if !recursive {
- return fmt.Errorf("Fail to delete non-empty directory %s!", dirPath)
- }
- }
- for {
- if len(list) == 0 {
- return filer.directories.DeleteDirectory(dirPath)
- }
- var fids []string
- for _, fileEntry := range list {
- fids = append(fids, string(fileEntry.Id))
- }
- if result_list, delete_file_err := operation.DeleteFiles(filer.master, fids); delete_file_err != nil {
- return delete_file_err
- } else {
- if len(result_list.Errors) > 0 {
- return errors.New(strings.Join(result_list.Errors, "\n"))
- }
- }
- lastFile := list[len(list)-1]
- list = filer.files.ListFiles(dirId, lastFile.Name, 100)
- }
-
-}
-
-func (filer *FilerEmbedded) DeleteFile(filePath string) (fid string, err error) {
- dir, file := filepath.Split(filePath)
- dirId, e := filer.directories.FindDirectory(dir)
- if e != nil {
- return "", e
- }
- return filer.files.DeleteFile(dirId, file)
-}
-
-/*
-Move a folder or a file, with 4 Use cases:
-mv fromDir toNewDir
-mv fromDir toOldDir
-mv fromFile toDir
-mv fromFile toFile
-*/
-func (filer *FilerEmbedded) Move(fromPath string, toPath string) error {
- if _, dir_err := filer.FindDirectory(fromPath); dir_err == nil {
- if _, err := filer.FindDirectory(toPath); err == nil {
- // move folder under an existing folder
- return filer.directories.MoveUnderDirectory(fromPath, toPath, "")
- }
- // move folder to a new folder
- return filer.directories.MoveUnderDirectory(fromPath, filepath.Dir(toPath), filepath.Base(toPath))
- }
- if fid, file_err := filer.DeleteFile(fromPath); file_err == nil {
- if _, err := filer.FindDirectory(toPath); err == nil {
- // move file under an existing folder
- return filer.CreateFile(filepath.Join(toPath, filepath.Base(fromPath)), fid)
- }
- // move to a folder with new name
- return filer.CreateFile(toPath, fid)
- }
- return fmt.Errorf("File %s is not found!", fromPath)
-}
diff --git a/go/filer/embedded_filer/files_in_leveldb.go b/go/filer/embedded_filer/files_in_leveldb.go
deleted file mode 100644
index b0b410a26..000000000
--- a/go/filer/embedded_filer/files_in_leveldb.go
+++ /dev/null
@@ -1,85 +0,0 @@
-package embedded_filer
-
-import (
- "bytes"
-
- "github.com/chrislusf/seaweedfs/go/filer"
- "github.com/chrislusf/seaweedfs/go/glog"
- "github.com/syndtr/goleveldb/leveldb"
- "github.com/syndtr/goleveldb/leveldb/util"
-)
-
-/*
-The entry in level db has this format:
- key: genKey(dirId, fileName)
- value: []byte(fid)
-And genKey(dirId, fileName) use first 4 bytes to store dirId, and rest for fileName
-*/
-
-type FileListInLevelDb struct {
- db *leveldb.DB
-}
-
-func NewFileListInLevelDb(dir string) (fl *FileListInLevelDb, err error) {
- fl = &FileListInLevelDb{}
- if fl.db, err = leveldb.OpenFile(dir, nil); err != nil {
- return
- }
- return
-}
-
-func genKey(dirId filer.DirectoryId, fileName string) []byte {
- ret := make([]byte, 0, 4+len(fileName))
- for i := 3; i >= 0; i-- {
- ret = append(ret, byte(dirId>>(uint(i)*8)))
- }
- ret = append(ret, []byte(fileName)...)
- return ret
-}
-
-func (fl *FileListInLevelDb) CreateFile(dirId filer.DirectoryId, fileName string, fid string) (err error) {
- glog.V(4).Infoln("directory", dirId, "fileName", fileName, "fid", fid)
- return fl.db.Put(genKey(dirId, fileName), []byte(fid), nil)
-}
-func (fl *FileListInLevelDb) DeleteFile(dirId filer.DirectoryId, fileName string) (fid string, err error) {
- if fid, err = fl.FindFile(dirId, fileName); err != nil {
- if err == leveldb.ErrNotFound {
- return "", nil
- }
- return
- }
- err = fl.db.Delete(genKey(dirId, fileName), nil)
- return fid, err
-}
-func (fl *FileListInLevelDb) FindFile(dirId filer.DirectoryId, fileName string) (fid string, err error) {
- data, e := fl.db.Get(genKey(dirId, fileName), nil)
- if e != nil {
- return "", e
- }
- return string(data), nil
-}
-func (fl *FileListInLevelDb) ListFiles(dirId filer.DirectoryId, lastFileName string, limit int) (files []filer.FileEntry) {
- glog.V(4).Infoln("directory", dirId, "lastFileName", lastFileName, "limit", limit)
- dirKey := genKey(dirId, "")
- iter := fl.db.NewIterator(&util.Range{Start: genKey(dirId, lastFileName)}, nil)
- limitCounter := 0
- for iter.Next() {
- key := iter.Key()
- if !bytes.HasPrefix(key, dirKey) {
- break
- }
- fileName := string(key[len(dirKey):])
- if fileName == lastFileName {
- continue
- }
- limitCounter++
- if limit > 0 {
- if limitCounter > limit {
- break
- }
- }
- files = append(files, filer.FileEntry{Name: fileName, Id: filer.FileId(string(iter.Value()))})
- }
- iter.Release()
- return
-}
diff --git a/go/filer/filer.go b/go/filer/filer.go
deleted file mode 100644
index fd23e119c..000000000
--- a/go/filer/filer.go
+++ /dev/null
@@ -1,28 +0,0 @@
-package filer
-
-type FileId string //file id in SeaweedFS
-
-type FileEntry struct {
- Name string `json:"name,omitempty"` //file name without path
- Id FileId `json:"fid,omitempty"`
-}
-
-type DirectoryId int32
-
-type DirectoryEntry struct {
- Name string //dir name without path
- Id DirectoryId
-}
-
-type Filer interface {
- CreateFile(fullFileName string, fid string) (err error)
- FindFile(fullFileName string) (fid string, err error)
- DeleteFile(fullFileName string) (fid string, err error)
-
- //Optional functions. embedded filer support these
- FindDirectory(dirPath string) (dirId DirectoryId, err error)
- ListDirectories(dirPath string) (dirs []DirectoryEntry, err error)
- ListFiles(dirPath string, lastFileName string, limit int) (files []FileEntry, err error)
- DeleteDirectory(dirPath string, recursive bool) (err error)
- Move(fromPath string, toPath string) (err error)
-}
diff --git a/go/filer/flat_namespace/flat_namespace_filer.go b/go/filer/flat_namespace/flat_namespace_filer.go
deleted file mode 100644
index 9959b2924..000000000
--- a/go/filer/flat_namespace/flat_namespace_filer.go
+++ /dev/null
@@ -1,50 +0,0 @@
-package flat_namespace
-
-import (
- "errors"
-
- "github.com/chrislusf/seaweedfs/go/filer"
-)
-
-type FlatNamespaceFiler struct {
- master string
- store FlatNamespaceStore
-}
-
-var (
- ErrNotImplemented = errors.New("Not Implemented for flat namespace meta data store")
-)
-
-func NewFlatNamespaceFiler(master string, store FlatNamespaceStore) *FlatNamespaceFiler {
- return &FlatNamespaceFiler{
- master: master,
- store: store,
- }
-}
-
-func (filer *FlatNamespaceFiler) CreateFile(fullFileName string, fid string) (err error) {
- return filer.store.Put(fullFileName, fid)
-}
-func (filer *FlatNamespaceFiler) FindFile(fullFileName string) (fid string, err error) {
- return filer.store.Get(fullFileName)
-}
-func (filer *FlatNamespaceFiler) FindDirectory(dirPath string) (dirId filer.DirectoryId, err error) {
- return 0, ErrNotImplemented
-}
-func (filer *FlatNamespaceFiler) ListDirectories(dirPath string) (dirs []filer.DirectoryEntry, err error) {
- return nil, ErrNotImplemented
-}
-func (filer *FlatNamespaceFiler) ListFiles(dirPath string, lastFileName string, limit int) (files []filer.FileEntry, err error) {
- return nil, ErrNotImplemented
-}
-func (filer *FlatNamespaceFiler) DeleteDirectory(dirPath string, recursive bool) (err error) {
- return ErrNotImplemented
-}
-
-func (filer *FlatNamespaceFiler) DeleteFile(fullFileName string) (fid string, err error) {
- return filer.store.Delete(fullFileName)
-}
-
-func (filer *FlatNamespaceFiler) Move(fromPath string, toPath string) error {
- return ErrNotImplemented
-}
diff --git a/go/filer/flat_namespace/flat_namespace_store.go b/go/filer/flat_namespace/flat_namespace_store.go
deleted file mode 100644
index 832b70e40..000000000
--- a/go/filer/flat_namespace/flat_namespace_store.go
+++ /dev/null
@@ -1,9 +0,0 @@
-package flat_namespace
-
-import ()
-
-type FlatNamespaceStore interface {
- Put(fullFileName string, fid string) (err error)
- Get(fullFileName string) (fid string, err error)
- Delete(fullFileName string) (fid string, err error)
-}
diff --git a/go/filer/redis_store/redis_store.go b/go/filer/redis_store/redis_store.go
deleted file mode 100644
index 939172557..000000000
--- a/go/filer/redis_store/redis_store.go
+++ /dev/null
@@ -1,48 +0,0 @@
-package redis_store
-
-import (
- redis "gopkg.in/redis.v2"
-)
-
-type RedisStore struct {
- Client *redis.Client
-}
-
-func NewRedisStore(hostPort string, password string, database int) *RedisStore {
- client := redis.NewTCPClient(&redis.Options{
- Addr: hostPort,
- Password: password,
- DB: int64(database),
- })
- return &RedisStore{Client: client}
-}
-
-func (s *RedisStore) Get(fullFileName string) (fid string, err error) {
- fid, err = s.Client.Get(fullFileName).Result()
- if err == redis.Nil {
- err = nil
- }
- return fid, err
-}
-func (s *RedisStore) Put(fullFileName string, fid string) (err error) {
- _, err = s.Client.Set(fullFileName, fid).Result()
- if err == redis.Nil {
- err = nil
- }
- return err
-}
-
-// Currently the fid is not returned
-func (s *RedisStore) Delete(fullFileName string) (fid string, err error) {
- _, err = s.Client.Del(fullFileName).Result()
- if err == redis.Nil {
- err = nil
- }
- return "", err
-}
-
-func (s *RedisStore) Close() {
- if s.Client != nil {
- s.Client.Close()
- }
-}