diff options
Diffstat (limited to 'go/weed')
44 files changed, 0 insertions, 4961 deletions
diff --git a/go/weed/backup.go b/go/weed/backup.go deleted file mode 100644 index 5e51a8b03..000000000 --- a/go/weed/backup.go +++ /dev/null @@ -1,90 +0,0 @@ -package main - -import ( - "fmt" - - "github.com/chrislusf/seaweedfs/go/operation" - "github.com/chrislusf/seaweedfs/go/storage" -) - -var ( - s BackupOptions -) - -type BackupOptions struct { - master *string - collection *string - dir *string - volumeId *int -} - -func init() { - cmdBackup.Run = runBackup // break init cycle - s.master = cmdBackup.Flag.String("server", "localhost:9333", "SeaweedFS master location") - s.collection = cmdBackup.Flag.String("collection", "", "collection name") - s.dir = cmdBackup.Flag.String("dir", ".", "directory to store volume data files") - s.volumeId = cmdBackup.Flag.Int("volumeId", -1, "a volume id. The volume .dat and .idx files should already exist in the dir.") -} - -var cmdBackup = &Command{ - UsageLine: "backup -dir=. -volumeId=234 -server=localhost:9333", - Short: "incrementally backup a volume to local folder", - Long: `Incrementally backup volume data. - - It is expected that you use this inside a script, to loop through - all possible volume ids that needs to be backup to local folder. - - The volume id does not need to exist locally or even remotely. - This will help to backup future new volumes. - - Usually backing up is just copying the .dat (and .idx) files. - But it's tricky to incremententally copy the differences. - - The complexity comes when there are multiple addition, deletion and compaction. - This tool will handle them correctly and efficiently, avoiding unnecessary data transporation. - `, -} - -func runBackup(cmd *Command, args []string) bool { - if *s.volumeId == -1 { - return false - } - vid := storage.VolumeId(*s.volumeId) - - // find volume location, replication, ttl info - lookup, err := operation.Lookup(*s.master, vid.String()) - if err != nil { - fmt.Printf("Error looking up volume %d: %v\n", vid, err) - return true - } - volumeServer := lookup.Locations[0].Url - - stats, err := operation.GetVolumeSyncStatus(volumeServer, vid.String()) - if err != nil { - fmt.Printf("Error get volume %d status: %v\n", vid, err) - return true - } - ttl, err := storage.ReadTTL(stats.Ttl) - if err != nil { - fmt.Printf("Error get volume %d ttl %s: %v\n", vid, stats.Ttl, err) - return true - } - replication, err := storage.NewReplicaPlacementFromString(stats.Replication) - if err != nil { - fmt.Printf("Error get volume %d replication %s : %v\n", vid, stats.Replication, err) - return true - } - - v, err := storage.NewVolume(*s.dir, *s.collection, vid, storage.NeedleMapInMemory, replication, ttl) - if err != nil { - fmt.Printf("Error creating or reading from volume %d: %v\n", vid, err) - return true - } - - if err := v.Synchronize(volumeServer); err != nil { - fmt.Printf("Error synchronizing volume %d: %v\n", vid, err) - return true - } - - return true -} diff --git a/go/weed/benchmark.go b/go/weed/benchmark.go deleted file mode 100644 index b63f0008e..000000000 --- a/go/weed/benchmark.go +++ /dev/null @@ -1,532 +0,0 @@ -package main - -import ( - "bufio" - "fmt" - "io" - "math" - "math/rand" - "os" - "runtime" - "runtime/pprof" - "sort" - "strings" - "sync" - "time" - - "github.com/chrislusf/seaweedfs/go/glog" - "github.com/chrislusf/seaweedfs/go/operation" - "github.com/chrislusf/seaweedfs/go/security" - "github.com/chrislusf/seaweedfs/go/util" -) - -type BenchmarkOptions struct { - server *string - concurrency *int - numberOfFiles *int - fileSize *int - idListFile *string - write *bool - deletePercentage *int - read *bool - sequentialRead *bool - collection *string - cpuprofile *string - maxCpu *int - secretKey *string -} - -var ( - b BenchmarkOptions - sharedBytes []byte -) - -func init() { - cmdBenchmark.Run = runbenchmark // break init cycle - cmdBenchmark.IsDebug = cmdBenchmark.Flag.Bool("debug", false, "verbose debug information") - b.server = cmdBenchmark.Flag.String("server", "localhost:9333", "SeaweedFS master location") - b.concurrency = cmdBenchmark.Flag.Int("c", 16, "number of concurrent write or read processes") - b.fileSize = cmdBenchmark.Flag.Int("size", 1024, "simulated file size in bytes, with random(0~63) bytes padding") - b.numberOfFiles = cmdBenchmark.Flag.Int("n", 1024*1024, "number of files to write for each thread") - b.idListFile = cmdBenchmark.Flag.String("list", os.TempDir()+"/benchmark_list.txt", "list of uploaded file ids") - b.write = cmdBenchmark.Flag.Bool("write", true, "enable write") - b.deletePercentage = cmdBenchmark.Flag.Int("deletePercent", 0, "the percent of writes that are deletes") - b.read = cmdBenchmark.Flag.Bool("read", true, "enable read") - b.sequentialRead = cmdBenchmark.Flag.Bool("readSequentially", false, "randomly read by ids from \"-list\" specified file") - b.collection = cmdBenchmark.Flag.String("collection", "benchmark", "write data to this collection") - b.cpuprofile = cmdBenchmark.Flag.String("cpuprofile", "", "cpu profile output file") - b.maxCpu = cmdBenchmark.Flag.Int("maxCpu", 0, "maximum number of CPUs. 0 means all available CPUs") - b.secretKey = cmdBenchmark.Flag.String("secure.secret", "", "secret to encrypt Json Web Token(JWT)") - sharedBytes = make([]byte, 1024) -} - -var cmdBenchmark = &Command{ - UsageLine: "benchmark -server=localhost:9333 -c=10 -n=100000", - Short: "benchmark on writing millions of files and read out", - Long: `benchmark on an empty SeaweedFS file system. - - Two tests during benchmark: - 1) write lots of small files to the system - 2) read the files out - - The file content is mostly zero, but no compression is done. - - You can choose to only benchmark read or write. - During write, the list of uploaded file ids is stored in "-list" specified file. - You can also use your own list of file ids to run read test. - - Write speed and read speed will be collected. - The numbers are used to get a sense of the system. - Usually your network or the hard drive is the real bottleneck. - - Another thing to watch is whether the volumes are evenly distributed - to each volume server. Because the 7 more benchmark volumes are randomly distributed - to servers with free slots, it's highly possible some servers have uneven amount of - benchmark volumes. To remedy this, you can use this to grow the benchmark volumes - before starting the benchmark command: - http://localhost:9333/vol/grow?collection=benchmark&count=5 - - After benchmarking, you can clean up the written data by deleting the benchmark collection - http://localhost:9333/col/delete?collection=benchmark - - `, -} - -var ( - wait sync.WaitGroup - writeStats *stats - readStats *stats -) - -func runbenchmark(cmd *Command, args []string) bool { - fmt.Printf("This is SeaweedFS version %s %s %s\n", util.VERSION, runtime.GOOS, runtime.GOARCH) - if *b.maxCpu < 1 { - *b.maxCpu = runtime.NumCPU() - } - runtime.GOMAXPROCS(*b.maxCpu) - if *b.cpuprofile != "" { - f, err := os.Create(*b.cpuprofile) - if err != nil { - glog.Fatal(err) - } - pprof.StartCPUProfile(f) - defer pprof.StopCPUProfile() - } - - if *b.write { - bench_write() - } - - if *b.read { - bench_read() - } - - return true -} - -func bench_write() { - fileIdLineChan := make(chan string) - finishChan := make(chan bool) - writeStats = newStats(*b.concurrency) - idChan := make(chan int) - go writeFileIds(*b.idListFile, fileIdLineChan, finishChan) - for i := 0; i < *b.concurrency; i++ { - wait.Add(1) - go writeFiles(idChan, fileIdLineChan, &writeStats.localStats[i]) - } - writeStats.start = time.Now() - writeStats.total = *b.numberOfFiles - go writeStats.checkProgress("Writing Benchmark", finishChan) - for i := 0; i < *b.numberOfFiles; i++ { - idChan <- i - } - close(idChan) - wait.Wait() - writeStats.end = time.Now() - wait.Add(2) - finishChan <- true - finishChan <- true - wait.Wait() - close(finishChan) - writeStats.printStats() -} - -func bench_read() { - fileIdLineChan := make(chan string) - finishChan := make(chan bool) - readStats = newStats(*b.concurrency) - go readFileIds(*b.idListFile, fileIdLineChan) - readStats.start = time.Now() - readStats.total = *b.numberOfFiles - go readStats.checkProgress("Randomly Reading Benchmark", finishChan) - for i := 0; i < *b.concurrency; i++ { - wait.Add(1) - go readFiles(fileIdLineChan, &readStats.localStats[i]) - } - wait.Wait() - wait.Add(1) - finishChan <- true - wait.Wait() - close(finishChan) - readStats.end = time.Now() - readStats.printStats() -} - -type delayedFile struct { - enterTime time.Time - fp *operation.FilePart -} - -func writeFiles(idChan chan int, fileIdLineChan chan string, s *stat) { - defer wait.Done() - delayedDeleteChan := make(chan *delayedFile, 100) - var waitForDeletions sync.WaitGroup - secret := security.Secret(*b.secretKey) - - for i := 0; i < 7; i++ { - waitForDeletions.Add(1) - go func() { - defer waitForDeletions.Done() - for df := range delayedDeleteChan { - if df.enterTime.After(time.Now()) { - time.Sleep(df.enterTime.Sub(time.Now())) - } - if e := util.Delete("http://"+df.fp.Server+"/"+df.fp.Fid, - security.GenJwt(secret, df.fp.Fid)); e == nil { - s.completed++ - } else { - s.failed++ - } - } - }() - } - - for id := range idChan { - start := time.Now() - fileSize := int64(*b.fileSize + rand.Intn(64)) - fp := &operation.FilePart{Reader: &FakeReader{id: uint64(id), size: fileSize}, FileSize: fileSize} - if assignResult, err := operation.Assign(*b.server, 1, "", *b.collection, ""); err == nil { - fp.Server, fp.Fid, fp.Collection = assignResult.Url, assignResult.Fid, *b.collection - if _, err := fp.Upload(0, *b.server, secret); err == nil { - if rand.Intn(100) < *b.deletePercentage { - s.total++ - delayedDeleteChan <- &delayedFile{time.Now().Add(time.Second), fp} - } else { - fileIdLineChan <- fp.Fid - } - s.completed++ - s.transferred += fileSize - } else { - s.failed++ - fmt.Printf("Failed to write with error:%v\n", err) - } - writeStats.addSample(time.Now().Sub(start)) - if *cmdBenchmark.IsDebug { - fmt.Printf("writing %d file %s\n", id, fp.Fid) - } - } else { - s.failed++ - println("writing file error:", err.Error()) - } - } - close(delayedDeleteChan) - waitForDeletions.Wait() -} - -func readFiles(fileIdLineChan chan string, s *stat) { - defer wait.Done() - for fid := range fileIdLineChan { - if len(fid) == 0 { - continue - } - if fid[0] == '#' { - continue - } - if *cmdBenchmark.IsDebug { - fmt.Printf("reading file %s\n", fid) - } - parts := strings.SplitN(fid, ",", 2) - vid := parts[0] - start := time.Now() - ret, err := operation.Lookup(*b.server, vid) - if err != nil || len(ret.Locations) == 0 { - s.failed++ - println("!!!! volume id ", vid, " location not found!!!!!") - continue - } - server := ret.Locations[rand.Intn(len(ret.Locations))].Url - url := "http://" + server + "/" + fid - if bytesRead, err := util.Get(url); err == nil { - s.completed++ - s.transferred += int64(len(bytesRead)) - readStats.addSample(time.Now().Sub(start)) - } else { - s.failed++ - fmt.Printf("Failed to read %s error:%v\n", url, err) - } - } -} - -func writeFileIds(fileName string, fileIdLineChan chan string, finishChan chan bool) { - file, err := os.OpenFile(fileName, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0644) - if err != nil { - glog.Fatalf("File to create file %s: %s\n", fileName, err) - } - defer file.Close() - - for { - select { - case <-finishChan: - wait.Done() - return - case line := <-fileIdLineChan: - file.Write([]byte(line)) - file.Write([]byte("\n")) - } - } -} - -func readFileIds(fileName string, fileIdLineChan chan string) { - file, err := os.Open(fileName) // For read access. - if err != nil { - glog.Fatalf("File to read file %s: %s\n", fileName, err) - } - defer file.Close() - - r := bufio.NewReader(file) - if *b.sequentialRead { - for { - if line, err := Readln(r); err == nil { - fileIdLineChan <- string(line) - } else { - break - } - } - } else { - lines := make([]string, 0, readStats.total) - for { - if line, err := Readln(r); err == nil { - lines = append(lines, string(line)) - } else { - break - } - } - if len(lines) > 0 { - for i := 0; i < readStats.total; i++ { - fileIdLineChan <- lines[rand.Intn(len(lines))] - } - } - } - - close(fileIdLineChan) -} - -const ( - benchResolution = 10000 //0.1 microsecond - benchBucket = 1000000000 / benchResolution -) - -// An efficient statics collecting and rendering -type stats struct { - data []int - overflow []int - localStats []stat - start time.Time - end time.Time - total int -} -type stat struct { - completed int - failed int - total int - transferred int64 -} - -var percentages = []int{50, 66, 75, 80, 90, 95, 98, 99, 100} - -func newStats(n int) *stats { - return &stats{ - data: make([]int, benchResolution), - overflow: make([]int, 0), - localStats: make([]stat, n), - } -} - -func (s *stats) addSample(d time.Duration) { - index := int(d / benchBucket) - if index < 0 { - fmt.Printf("This request takes %3.1f seconds, skipping!\n", float64(index)/10000) - } else if index < len(s.data) { - s.data[int(d/benchBucket)]++ - } else { - s.overflow = append(s.overflow, index) - } -} - -func (s *stats) checkProgress(testName string, finishChan chan bool) { - fmt.Printf("\n------------ %s ----------\n", testName) - ticker := time.Tick(time.Second) - lastCompleted, lastTransferred, lastTime := 0, int64(0), time.Now() - for { - select { - case <-finishChan: - wait.Done() - return - case t := <-ticker: - completed, transferred, taken, total := 0, int64(0), t.Sub(lastTime), s.total - for _, localStat := range s.localStats { - completed += localStat.completed - transferred += localStat.transferred - total += localStat.total - } - fmt.Printf("Completed %d of %d requests, %3.1f%% %3.1f/s %3.1fMB/s\n", - completed, total, float64(completed)*100/float64(total), - float64(completed-lastCompleted)*float64(int64(time.Second))/float64(int64(taken)), - float64(transferred-lastTransferred)*float64(int64(time.Second))/float64(int64(taken))/float64(1024*1024), - ) - lastCompleted, lastTransferred, lastTime = completed, transferred, t - } - } -} - -func (s *stats) printStats() { - completed, failed, transferred, total := 0, 0, int64(0), s.total - for _, localStat := range s.localStats { - completed += localStat.completed - failed += localStat.failed - transferred += localStat.transferred - total += localStat.total - } - timeTaken := float64(int64(s.end.Sub(s.start))) / 1000000000 - fmt.Printf("\nConcurrency Level: %d\n", *b.concurrency) - fmt.Printf("Time taken for tests: %.3f seconds\n", timeTaken) - fmt.Printf("Complete requests: %d\n", completed) - fmt.Printf("Failed requests: %d\n", failed) - fmt.Printf("Total transferred: %d bytes\n", transferred) - fmt.Printf("Requests per second: %.2f [#/sec]\n", float64(completed)/timeTaken) - fmt.Printf("Transfer rate: %.2f [Kbytes/sec]\n", float64(transferred)/1024/timeTaken) - n, sum := 0, 0 - min, max := 10000000, 0 - for i := 0; i < len(s.data); i++ { - n += s.data[i] - sum += s.data[i] * i - if s.data[i] > 0 { - if min > i { - min = i - } - if max < i { - max = i - } - } - } - n += len(s.overflow) - for i := 0; i < len(s.overflow); i++ { - sum += s.overflow[i] - if min > s.overflow[i] { - min = s.overflow[i] - } - if max < s.overflow[i] { - max = s.overflow[i] - } - } - avg := float64(sum) / float64(n) - varianceSum := 0.0 - for i := 0; i < len(s.data); i++ { - if s.data[i] > 0 { - d := float64(i) - avg - varianceSum += d * d * float64(s.data[i]) - } - } - for i := 0; i < len(s.overflow); i++ { - d := float64(s.overflow[i]) - avg - varianceSum += d * d - } - std := math.Sqrt(varianceSum / float64(n)) - fmt.Printf("\nConnection Times (ms)\n") - fmt.Printf(" min avg max std\n") - fmt.Printf("Total: %2.1f %3.1f %3.1f %3.1f\n", float32(min)/10, float32(avg)/10, float32(max)/10, std/10) - //printing percentiles - fmt.Printf("\nPercentage of the requests served within a certain time (ms)\n") - percentiles := make([]int, len(percentages)) - for i := 0; i < len(percentages); i++ { - percentiles[i] = n * percentages[i] / 100 - } - percentiles[len(percentiles)-1] = n - percentileIndex := 0 - currentSum := 0 - for i := 0; i < len(s.data); i++ { - currentSum += s.data[i] - if s.data[i] > 0 && percentileIndex < len(percentiles) && currentSum >= percentiles[percentileIndex] { - fmt.Printf(" %3d%% %5.1f ms\n", percentages[percentileIndex], float32(i)/10.0) - percentileIndex++ - for percentileIndex < len(percentiles) && currentSum >= percentiles[percentileIndex] { - percentileIndex++ - } - } - } - sort.Ints(s.overflow) - for i := 0; i < len(s.overflow); i++ { - currentSum++ - if percentileIndex < len(percentiles) && currentSum >= percentiles[percentileIndex] { - fmt.Printf(" %3d%% %5.1f ms\n", percentages[percentileIndex], float32(s.overflow[i])/10.0) - percentileIndex++ - for percentileIndex < len(percentiles) && currentSum >= percentiles[percentileIndex] { - percentileIndex++ - } - } - } -} - -// a fake reader to generate content to upload -type FakeReader struct { - id uint64 // an id number - size int64 // max bytes -} - -func (l *FakeReader) Read(p []byte) (n int, err error) { - if l.size <= 0 { - return 0, io.EOF - } - if int64(len(p)) > l.size { - n = int(l.size) - } else { - n = len(p) - } - if n >= 8 { - for i := 0; i < 8; i++ { - p[i] = byte(l.id >> uint(i*8)) - } - } - l.size -= int64(n) - return -} - -func (l *FakeReader) WriteTo(w io.Writer) (n int64, err error) { - size := int(l.size) - bufferSize := len(sharedBytes) - for size > 0 { - tempBuffer := sharedBytes - if size < bufferSize { - tempBuffer = sharedBytes[0:size] - } - count, e := w.Write(tempBuffer) - if e != nil { - return int64(size), e - } - size -= count - } - return l.size, nil -} - -func Readln(r *bufio.Reader) ([]byte, error) { - var ( - isPrefix = true - err error - line, ln []byte - ) - for isPrefix && err == nil { - line, isPrefix, err = r.ReadLine() - ln = append(ln, line...) - } - return ln, err -} diff --git a/go/weed/command.go b/go/weed/command.go deleted file mode 100644 index c8d86ca66..000000000 --- a/go/weed/command.go +++ /dev/null @@ -1,54 +0,0 @@ -package main - -import ( - "flag" - "fmt" - "os" - "strings" -) - -type Command struct { - // Run runs the command. - // The args are the arguments after the command name. - Run func(cmd *Command, args []string) bool - - // UsageLine is the one-line usage message. - // The first word in the line is taken to be the command name. - UsageLine string - - // Short is the short description shown in the 'go help' output. - Short string - - // Long is the long message shown in the 'go help <this-command>' output. - Long string - - // Flag is a set of flags specific to this command. - Flag flag.FlagSet - - IsDebug *bool -} - -// Name returns the command's name: the first word in the usage line. -func (c *Command) Name() string { - name := c.UsageLine - i := strings.Index(name, " ") - if i >= 0 { - name = name[:i] - } - return name -} - -func (c *Command) Usage() { - fmt.Fprintf(os.Stderr, "Example: weed %s\n", c.UsageLine) - fmt.Fprintf(os.Stderr, "Default Usage:\n") - c.Flag.PrintDefaults() - fmt.Fprintf(os.Stderr, "Description:\n") - fmt.Fprintf(os.Stderr, " %s\n", strings.TrimSpace(c.Long)) - os.Exit(2) -} - -// Runnable reports whether the command can be run; otherwise -// it is a documentation pseudo-command such as importpath. -func (c *Command) Runnable() bool { - return c.Run != nil -} diff --git a/go/weed/compact.go b/go/weed/compact.go deleted file mode 100644 index 673b96901..000000000 --- a/go/weed/compact.go +++ /dev/null @@ -1,45 +0,0 @@ -package main - -import ( - "github.com/chrislusf/seaweedfs/go/glog" - "github.com/chrislusf/seaweedfs/go/storage" -) - -func init() { - cmdCompact.Run = runCompact // break init cycle -} - -var cmdCompact = &Command{ - UsageLine: "compact -dir=/tmp -volumeId=234", - Short: "run weed tool compact on volume file", - Long: `Force an compaction to remove deleted files from volume files. - The compacted .dat file is stored as .cpd file. - The compacted .idx file is stored as .cpx file. - - `, -} - -var ( - compactVolumePath = cmdCompact.Flag.String("dir", ".", "data directory to store files") - compactVolumeCollection = cmdCompact.Flag.String("collection", "", "volume collection name") - compactVolumeId = cmdCompact.Flag.Int("volumeId", -1, "a volume id. The volume should already exist in the dir.") -) - -func runCompact(cmd *Command, args []string) bool { - - if *compactVolumeId == -1 { - return false - } - - vid := storage.VolumeId(*compactVolumeId) - v, err := storage.NewVolume(*compactVolumePath, *compactVolumeCollection, vid, - storage.NeedleMapInMemory, nil, nil) - if err != nil { - glog.Fatalf("Load Volume [ERROR] %s\n", err) - } - if err = v.Compact(); err != nil { - glog.Fatalf("Compact Volume [ERROR] %s\n", err) - } - - return true -} diff --git a/go/weed/download.go b/go/weed/download.go deleted file mode 100644 index 2e948a056..000000000 --- a/go/weed/download.go +++ /dev/null @@ -1,130 +0,0 @@ -package main - -import ( - "fmt" - "io" - "io/ioutil" - "os" - "path" - "strings" - - "github.com/chrislusf/seaweedfs/go/operation" - "github.com/chrislusf/seaweedfs/go/util" -) - -var ( - d DownloadOptions -) - -type DownloadOptions struct { - server *string - dir *string -} - -func init() { - cmdDownload.Run = runDownload // break init cycle - d.server = cmdDownload.Flag.String("server", "localhost:9333", "SeaweedFS master location") - d.dir = cmdDownload.Flag.String("dir", ".", "Download the whole folder recursively if specified.") -} - -var cmdDownload = &Command{ - UsageLine: "download -server=localhost:9333 -dir=one_directory fid1 [fid2 fid3 ...]", - Short: "download files by file id", - Long: `download files by file id. - - Usually you just need to use curl to lookup the file's volume server, and then download them directly. - This download tool combine the two steps into one. - - What's more, if you use "weed upload -maxMB=..." option to upload a big file divided into chunks, you can - use this tool to download the chunks and merge them automatically. - - `, -} - -func runDownload(cmd *Command, args []string) bool { - for _, fid := range args { - if e := downloadToFile(*d.server, fid, *d.dir); e != nil { - fmt.Println("Download Error: ", fid, e) - } - } - return true -} - -func downloadToFile(server, fileId, saveDir string) error { - fileUrl, lookupError := operation.LookupFileId(server, fileId) - if lookupError != nil { - return lookupError - } - filename, rc, err := util.DownloadUrl(fileUrl) - if err != nil { - return err - } - defer rc.Close() - if filename == "" { - filename = fileId - } - isFileList := false - if strings.HasSuffix(filename, "-list") { - // old command compatible - isFileList = true - filename = filename[0 : len(filename)-len("-list")] - } - f, err := os.OpenFile(path.Join(saveDir, filename), os.O_WRONLY|os.O_CREATE|os.O_TRUNC, os.ModePerm) - if err != nil { - return err - } - defer f.Close() - if isFileList { - content, err := ioutil.ReadAll(rc) - if err != nil { - return err - } - fids := strings.Split(string(content), "\n") - for _, partId := range fids { - var n int - _, part, err := fetchContent(*d.server, partId) - if err == nil { - n, err = f.Write(part) - } - if err == nil && n < len(part) { - err = io.ErrShortWrite - } - if err != nil { - return err - } - } - } else { - if _, err = io.Copy(f, rc); err != nil { - return err - } - - } - return nil -} - -func fetchContent(server string, fileId string) (filename string, content []byte, e error) { - fileUrl, lookupError := operation.LookupFileId(server, fileId) - if lookupError != nil { - return "", nil, lookupError - } - var rc io.ReadCloser - if filename, rc, e = util.DownloadUrl(fileUrl); e != nil { - return "", nil, e - } - content, e = ioutil.ReadAll(rc) - rc.Close() - return -} - -func WriteFile(filename string, data []byte, perm os.FileMode) error { - f, err := os.OpenFile(filename, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, perm) - if err != nil { - return err - } - n, err := f.Write(data) - f.Close() - if err == nil && n < len(data) { - err = io.ErrShortWrite - } - return err -} diff --git a/go/weed/export.go b/go/weed/export.go deleted file mode 100644 index 4417c65f4..000000000 --- a/go/weed/export.go +++ /dev/null @@ -1,213 +0,0 @@ -package main - -import ( - "archive/tar" - "bytes" - "fmt" - "os" - "path" - "path/filepath" - "strconv" - "strings" - "text/template" - "time" - - "github.com/chrislusf/seaweedfs/go/glog" - "github.com/chrislusf/seaweedfs/go/storage" -) - -const ( - defaultFnFormat = `{{.Mime}}/{{.Id}}:{{.Name}}` - timeFormat = "2006-01-02T15:04:05" -) - -var ( - export ExportOptions -) - -type ExportOptions struct { - dir *string - collection *string - volumeId *int -} - -var cmdExport = &Command{ - UsageLine: "export -dir=/tmp -volumeId=234 -o=/dir/name.tar -fileNameFormat={{.Name}} -newer='" + timeFormat + "'", - Short: "list or export files from one volume data file", - Long: `List all files in a volume, or Export all files in a volume to a tar file if the output is specified. - - The format of file name in the tar file can be customized. Default is {{.Mime}}/{{.Id}}:{{.Name}}. Also available is {{.Key}}. - - `, -} - -func init() { - cmdExport.Run = runExport // break init cycle - export.dir = cmdExport.Flag.String("dir", ".", "input data directory to store volume data files") - export.collection = cmdExport.Flag.String("collection", "", "the volume collection name") - export.volumeId = cmdExport.Flag.Int("volumeId", -1, "a volume id. The volume .dat and .idx files should already exist in the dir.") -} - -var ( - output = cmdExport.Flag.String("o", "", "output tar file name, must ends with .tar, or just a \"-\" for stdout") - format = cmdExport.Flag.String("fileNameFormat", defaultFnFormat, "filename formatted with {{.Mime}} {{.Id}} {{.Name}} {{.Ext}}") - newer = cmdExport.Flag.String("newer", "", "export only files newer than this time, default is all files. Must be specified in RFC3339 without timezone") - - tarOutputFile *tar.Writer - tarHeader tar.Header - fileNameTemplate *template.Template - fileNameTemplateBuffer = bytes.NewBuffer(nil) - newerThan time.Time - newerThanUnix int64 = -1 - localLocation, _ = time.LoadLocation("Local") -) - -func runExport(cmd *Command, args []string) bool { - - var err error - - if *newer != "" { - if newerThan, err = time.ParseInLocation(timeFormat, *newer, localLocation); err != nil { - fmt.Println("cannot parse 'newer' argument: " + err.Error()) - return false - } - newerThanUnix = newerThan.Unix() - } - - if *export.volumeId == -1 { - return false - } - - if *output != "" { - if *output != "-" && !strings.HasSuffix(*output, ".tar") { - fmt.Println("the output file", *output, "should be '-' or end with .tar") - return false - } - - if fileNameTemplate, err = template.New("name").Parse(*format); err != nil { - fmt.Println("cannot parse format " + *format + ": " + err.Error()) - return false - } - - var outputFile *os.File - if *output == "-" { - outputFile = os.Stdout - } else { - if outputFile, err = os.Create(*output); err != nil { - glog.Fatalf("cannot open output tar %s: %s", *output, err) - } - } - defer outputFile.Close() - tarOutputFile = tar.NewWriter(outputFile) - defer tarOutputFile.Close() - t := time.Now() - tarHeader = tar.Header{Mode: 0644, - ModTime: t, Uid: os.Getuid(), Gid: os.Getgid(), - Typeflag: tar.TypeReg, - AccessTime: t, ChangeTime: t} - } - - fileName := strconv.Itoa(*export.volumeId) - if *export.collection != "" { - fileName = *export.collection + "_" + fileName - } - vid := storage.VolumeId(*export.volumeId) - indexFile, err := os.OpenFile(path.Join(*export.dir, fileName+".idx"), os.O_RDONLY, 0644) - if err != nil { - glog.Fatalf("Create Volume Index [ERROR] %s\n", err) - } - defer indexFile.Close() - - needleMap, err := storage.LoadNeedleMap(indexFile) - if err != nil { - glog.Fatalf("cannot load needle map from %s: %s", indexFile.Name(), err) - } - - var version storage.Version - - err = storage.ScanVolumeFile(*export.dir, *export.collection, vid, - storage.NeedleMapInMemory, - func(superBlock storage.SuperBlock) error { - version = superBlock.Version() - return nil - }, true, func(n *storage.Needle, offset int64) error { - nv, ok := needleMap.Get(n.Id) - glog.V(3).Infof("key %d offset %d size %d disk_size %d gzip %v ok %v nv %+v", - n.Id, offset, n.Size, n.DiskSize(), n.IsGzipped(), ok, nv) - if ok && nv.Size > 0 && int64(nv.Offset)*8 == offset { - if newerThanUnix >= 0 && n.HasLastModifiedDate() && n.LastModified < uint64(newerThanUnix) { - glog.V(3).Infof("Skipping this file, as it's old enough: LastModified %d vs %d", - n.LastModified, newerThanUnix) - return nil - } - return walker(vid, n, version) - } - if !ok { - glog.V(2).Infof("This seems deleted %d size %d", n.Id, n.Size) - } else { - glog.V(2).Infof("Skipping later-updated Id %d size %d", n.Id, n.Size) - } - return nil - }) - if err != nil { - glog.Fatalf("Export Volume File [ERROR] %s\n", err) - } - return true -} - -type nameParams struct { - Name string - Id uint64 - Mime string - Key string - Ext string -} - -func walker(vid storage.VolumeId, n *storage.Needle, version storage.Version) (err error) { - key := storage.NewFileIdFromNeedle(vid, n).String() - if tarOutputFile != nil { - fileNameTemplateBuffer.Reset() - if err = fileNameTemplate.Execute(fileNameTemplateBuffer, - nameParams{ - Name: string(n.Name), - Id: n.Id, - Mime: string(n.Mime), - Key: key, - Ext: filepath.Ext(string(n.Name)), - }, - ); err != nil { - return err - } - - fileName := fileNameTemplateBuffer.String() - - if n.IsGzipped() && path.Ext(fileName) != ".gz" { - fileName = fileName + ".gz" - } - - tarHeader.Name, tarHeader.Size = fileName, int64(len(n.Data)) - if n.HasLastModifiedDate() { - tarHeader.ModTime = time.Unix(int64(n.LastModified), 0) - } else { - tarHeader.ModTime = time.Unix(0, 0) - } - tarHeader.ChangeTime = tarHeader.ModTime - if err = tarOutputFile.WriteHeader(&tarHeader); err != nil { - return err - } - _, err = tarOutputFile.Write(n.Data) - } else { - size := n.DataSize - if version == storage.Version1 { - size = n.Size - } - fmt.Printf("key=%s Name=%s Size=%d gzip=%t mime=%s\n", - key, - n.Name, - size, - n.IsGzipped(), - n.Mime, - ) - } - return -} diff --git a/go/weed/filer.go b/go/weed/filer.go deleted file mode 100644 index 68bc3e407..000000000 --- a/go/weed/filer.go +++ /dev/null @@ -1,105 +0,0 @@ -package main - -import ( - "net/http" - "os" - "strconv" - "time" - - "github.com/chrislusf/seaweedfs/go/glog" - "github.com/chrislusf/seaweedfs/go/util" - "github.com/chrislusf/seaweedfs/go/weed/weed_server" -) - -var ( - f FilerOptions -) - -type FilerOptions struct { - master *string - ip *string - port *int - collection *string - defaultReplicaPlacement *string - dir *string - redirectOnRead *bool - disableDirListing *bool - secretKey *string - cassandra_server *string - cassandra_keyspace *string - redis_server *string - redis_password *string - redis_database *int -} - -func init() { - cmdFiler.Run = runFiler // break init cycle - f.master = cmdFiler.Flag.String("master", "localhost:9333", "master server location") - f.collection = cmdFiler.Flag.String("collection", "", "all data will be stored in this collection") - f.ip = cmdFiler.Flag.String("ip", "", "filer server http listen ip address") - f.port = cmdFiler.Flag.Int("port", 8888, "filer server http listen port") - f.dir = cmdFiler.Flag.String("dir", os.TempDir(), "directory to store meta data") - f.defaultReplicaPlacement = cmdFiler.Flag.String("defaultReplicaPlacement", "000", "default replication type if not specified") - f.redirectOnRead = cmdFiler.Flag.Bool("redirectOnRead", false, "whether proxy or redirect to volume server during file GET request") - f.disableDirListing = cmdFiler.Flag.Bool("disableDirListing", false, "turn off directory listing") - f.cassandra_server = cmdFiler.Flag.String("cassandra.server", "", "host[:port] of the cassandra server") - f.cassandra_keyspace = cmdFiler.Flag.String("cassandra.keyspace", "seaweed", "keyspace of the cassandra server") - f.redis_server = cmdFiler.Flag.String("redis.server", "", "host:port of the redis server, e.g., 127.0.0.1:6379") - f.redis_password = cmdFiler.Flag.String("redis.password", "", "password in clear text") - f.redis_database = cmdFiler.Flag.Int("redis.database", 0, "the database on the redis server") - f.secretKey = cmdFiler.Flag.String("secure.secret", "", "secret to encrypt Json Web Token(JWT)") - -} - -var cmdFiler = &Command{ - UsageLine: "filer -port=8888 -dir=/tmp -master=<ip:port>", - Short: "start a file server that points to a master server", - Long: `start a file server which accepts REST operation for any files. - - //create or overwrite the file, the directories /path/to will be automatically created - POST /path/to/file - //get the file content - GET /path/to/file - //create or overwrite the file, the filename in the multipart request will be used - POST /path/to/ - //return a json format subdirectory and files listing - GET /path/to/ - - Current <fullpath~fileid> mapping metadata store is local embedded leveldb. - It should be highly scalable to hundreds of millions of files on a modest machine. - - Future we will ensure it can avoid of being SPOF. - - `, -} - -func runFiler(cmd *Command, args []string) bool { - - if err := util.TestFolderWritable(*f.dir); err != nil { - glog.Fatalf("Check Meta Folder (-dir) Writable %s : %s", *f.dir, err) - } - - r := http.NewServeMux() - _, nfs_err := weed_server.NewFilerServer(r, *f.ip, *f.port, *f.master, *f.dir, *f.collection, - *f.defaultReplicaPlacement, *f.redirectOnRead, *f.disableDirListing, - *f.secretKey, - *f.cassandra_server, *f.cassandra_keyspace, - *f.redis_server, *f.redis_password, *f.redis_database, - ) - if nfs_err != nil { - glog.Fatalf("Filer startup error: %v", nfs_err) - } - glog.V(0).Infoln("Start Seaweed Filer", util.VERSION, "at port", strconv.Itoa(*f.port)) - filerListener, e := util.NewListener( - ":"+strconv.Itoa(*f.port), - time.Duration(10)*time.Second, - ) - if e != nil { - glog.Fatalf("Filer listener error: %v", e) - } - if e := http.Serve(filerListener, r); e != nil { - glog.Fatalf("Filer Fail to serve: %v", e) - } - - return true -} diff --git a/go/weed/fix.go b/go/weed/fix.go deleted file mode 100644 index 3abcbc31c..000000000 --- a/go/weed/fix.go +++ /dev/null @@ -1,70 +0,0 @@ -package main - -import ( - "os" - "path" - "strconv" - - "github.com/chrislusf/seaweedfs/go/glog" - "github.com/chrislusf/seaweedfs/go/storage" -) - -func init() { - cmdFix.Run = runFix // break init cycle -} - -var cmdFix = &Command{ - UsageLine: "fix -dir=/tmp -volumeId=234", - Short: "run weed tool fix on index file if corrupted", - Long: `Fix runs the SeaweedFS fix command to re-create the index .idx file. - - `, -} - -var ( - fixVolumePath = cmdFix.Flag.String("dir", ".", "data directory to store files") - fixVolumeCollection = cmdFix.Flag.String("collection", "", "the volume collection name") - fixVolumeId = cmdFix.Flag.Int("volumeId", -1, "a volume id. The volume should already exist in the dir. The volume index file should not exist.") -) - -func runFix(cmd *Command, args []string) bool { - - if *fixVolumeId == -1 { - return false - } - - fileName := strconv.Itoa(*fixVolumeId) - if *fixVolumeCollection != "" { - fileName = *fixVolumeCollection + "_" + fileName - } - indexFile, err := os.OpenFile(path.Join(*fixVolumePath, fileName+".idx"), os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0644) - if err != nil { - glog.Fatalf("Create Volume Index [ERROR] %s\n", err) - } - defer indexFile.Close() - - nm := storage.NewNeedleMap(indexFile) - defer nm.Close() - - vid := storage.VolumeId(*fixVolumeId) - err = storage.ScanVolumeFile(*fixVolumePath, *fixVolumeCollection, vid, - storage.NeedleMapInMemory, - func(superBlock storage.SuperBlock) error { - return nil - }, false, func(n *storage.Needle, offset int64) error { - glog.V(2).Infof("key %d offset %d size %d disk_size %d gzip %v", n.Id, offset, n.Size, n.DiskSize(), n.IsGzipped()) - if n.Size > 0 { - pe := nm.Put(n.Id, uint32(offset/storage.NeedlePaddingSize), n.Size) - glog.V(2).Infof("saved %d with error %v", n.Size, pe) - } else { - glog.V(2).Infof("skipping deleted file ...") - return nm.Delete(n.Id) - } - return nil - }) - if err != nil { - glog.Fatalf("Export Volume File [ERROR] %s\n", err) - } - - return true -} diff --git a/go/weed/glide.lock b/go/weed/glide.lock deleted file mode 100644 index 3575d14d5..000000000 --- a/go/weed/glide.lock +++ /dev/null @@ -1,87 +0,0 @@ -hash: 97328ff2a0b9e682660bd51e424a7f850388df96bd153fa6f9ee419c993065c1 -updated: 2016-05-24T10:27:49.252798956-07:00 -imports: -- name: bazil.org/fuse - version: 5d02b06737b3b3c2e6a44e03348b6f2b44aa6835 - subpackages: - - fs - - fuseutil -- name: github.com/boltdb/bolt - version: dfb21201d9270c1082d5fb0f07f500311ff72f18 -- name: github.com/chrislusf/raft - version: 90f631ee823c83f594f27257bab64911190856af - subpackages: - - protobuf -- name: github.com/dgrijalva/jwt-go - version: 40bd0f3b4891a9d7f121bfb7b8e8b0525625e262 -- name: github.com/disintegration/imaging - version: d8bbae1de109b518dabc98c6c1633eb358c148a4 -- name: github.com/gocql/gocql - version: 7218e81e87ca6f3d2285c56e0aae80bc0d8fb655 - subpackages: - - internal/lru - - internal/murmur - - internal/streams -- name: github.com/gogo/protobuf - version: c18eea6ad611eecf94a9ba38471f59706199409e - subpackages: - - proto -- name: github.com/golang/protobuf - version: e51d002c610dbe8c136679a67a6ded5df4d49b5c - subpackages: - - proto -- name: github.com/golang/snappy - version: d6668316e43571d7dde95be6fd077f96de002f8b -- name: github.com/gorilla/context - version: a8d44e7d8e4d532b6a27a02dd82abb31cc1b01bd -- name: github.com/gorilla/mux - version: 9c19ed558d5df4da88e2ade9c8940d742aef0e7e -- name: github.com/hailocab/go-hostpool - version: e80d13ce29ede4452c43dea11e79b9bc8a15b478 -- name: github.com/hashicorp/golang-lru - version: a0d98a5f288019575c6d1f4bb1573fef2d1fcdc4 - subpackages: - - simplelru -- name: github.com/klauspost/crc32 - version: 19b0b332c9e4516a6370a0456e6182c3b5036720 -- name: github.com/rwcarlsen/goexif - version: 709fab3d192d7c62f86043caff1e7e3fb0f42bd8 - subpackages: - - exif - - tiff -- name: github.com/syndtr/goleveldb - version: cfa635847112c5dc4782e128fa7e0d05fdbfb394 - subpackages: - - leveldb - - leveldb/util - - leveldb/cache - - leveldb/comparer - - leveldb/errors - - leveldb/filter - - leveldb/iterator - - leveldb/journal - - leveldb/memdb - - leveldb/opt - - leveldb/storage - - leveldb/table -- name: golang.org/x/image - version: f551d3a6b7fc11df315ad9e18b404280680f8bec - subpackages: - - bmp - - tiff - - tiff/lzw -- name: golang.org/x/net - version: 0c607074acd38c5f23d1344dfe74c977464d1257 - subpackages: - - context -- name: golang.org/x/sys - version: d4feaf1a7e61e1d9e79e6c4e76c6349e9cab0a03 - subpackages: - - unix -- name: gopkg.in/bufio.v1 - version: 567b2bfa514e796916c4747494d6ff5132a1dfce -- name: gopkg.in/inf.v0 - version: 3887ee99ecf07df5b447e9b00d9c0b2adaa9f3e4 -- name: gopkg.in/redis.v2 - version: e6179049628164864e6e84e973cfb56335748dea -devImports: [] diff --git a/go/weed/glide.yaml b/go/weed/glide.yaml deleted file mode 100644 index d619b8727..000000000 --- a/go/weed/glide.yaml +++ /dev/null @@ -1,16 +0,0 @@ -package: github.com/chrislusf/seaweedfs/go/weed -import: -- package: bazil.org/fuse - subpackages: - - fs -- package: github.com/chrislusf/raft -- package: github.com/golang/protobuf - subpackages: - - proto -- package: github.com/gorilla/mux -- package: github.com/syndtr/goleveldb - subpackages: - - leveldb -- package: golang.org/x/net - subpackages: - - context diff --git a/go/weed/master.go b/go/weed/master.go deleted file mode 100644 index fda19429d..000000000 --- a/go/weed/master.go +++ /dev/null @@ -1,91 +0,0 @@ -package main - -import ( - "net/http" - "os" - "runtime" - "strconv" - "strings" - "time" - - "github.com/chrislusf/seaweedfs/go/glog" - "github.com/chrislusf/seaweedfs/go/util" - "github.com/chrislusf/seaweedfs/go/weed/weed_server" - "github.com/gorilla/mux" -) - -func init() { - cmdMaster.Run = runMaster // break init cycle -} - -var cmdMaster = &Command{ - UsageLine: "master -port=9333", - Short: "start a master server", - Long: `start a master server to provide volume=>location mapping service - and sequence number of file ids - - `, -} - -var ( - mport = cmdMaster.Flag.Int("port", 9333, "http listen port") - masterIp = cmdMaster.Flag.String("ip", "localhost", "master <ip>|<server> address") - masterBindIp = cmdMaster.Flag.String("ip.bind", "0.0.0.0", "ip address to bind to") - metaFolder = cmdMaster.Flag.String("mdir", os.TempDir(), "data directory to store meta data") - masterPeers = cmdMaster.Flag.String("peers", "", "other master nodes in comma separated ip:port list, example: 127.0.0.1:9093,127.0.0.1:9094") - volumeSizeLimitMB = cmdMaster.Flag.Uint("volumeSizeLimitMB", 30*1000, "Master stops directing writes to oversized volumes.") - mpulse = cmdMaster.Flag.Int("pulseSeconds", 5, "number of seconds between heartbeats") - confFile = cmdMaster.Flag.String("conf", "/etc/weedfs/weedfs.conf", "Deprecating! xml configuration file") - defaultReplicaPlacement = cmdMaster.Flag.String("defaultReplication", "000", "Default replication type if not specified.") - mTimeout = cmdMaster.Flag.Int("idleTimeout", 10, "connection idle seconds") - mMaxCpu = cmdMaster.Flag.Int("maxCpu", 0, "maximum number of CPUs. 0 means all available CPUs") - garbageThreshold = cmdMaster.Flag.String("garbageThreshold", "0.3", "threshold to vacuum and reclaim spaces") - masterWhiteListOption = cmdMaster.Flag.String("whiteList", "", "comma separated Ip addresses having write permission. No limit if empty.") - masterSecureKey = cmdMaster.Flag.String("secure.secret", "", "secret to encrypt Json Web Token(JWT)") - - masterWhiteList []string -) - -func runMaster(cmd *Command, args []string) bool { - if *mMaxCpu < 1 { - *mMaxCpu = runtime.NumCPU() - } - runtime.GOMAXPROCS(*mMaxCpu) - if err := util.TestFolderWritable(*metaFolder); err != nil { - glog.Fatalf("Check Meta Folder (-mdir) Writable %s : %s", *metaFolder, err) - } - if *masterWhiteListOption != "" { - masterWhiteList = strings.Split(*masterWhiteListOption, ",") - } - - r := mux.NewRouter() - ms := weed_server.NewMasterServer(r, *mport, *metaFolder, - *volumeSizeLimitMB, *mpulse, *confFile, *defaultReplicaPlacement, *garbageThreshold, - masterWhiteList, *masterSecureKey, - ) - - listeningAddress := *masterBindIp + ":" + strconv.Itoa(*mport) - - glog.V(0).Infoln("Start Seaweed Master", util.VERSION, "at", listeningAddress) - - listener, e := util.NewListener(listeningAddress, time.Duration(*mTimeout)*time.Second) - if e != nil { - glog.Fatalf("Master startup error: %v", e) - } - - go func() { - time.Sleep(100 * time.Millisecond) - myMasterAddress := *masterIp + ":" + strconv.Itoa(*mport) - var peers []string - if *masterPeers != "" { - peers = strings.Split(*masterPeers, ",") - } - raftServer := weed_server.NewRaftServer(r, peers, myMasterAddress, *metaFolder, ms.Topo, *mpulse) - ms.SetRaftServer(raftServer) - }() - - if e := http.Serve(listener, r); e != nil { - glog.Fatalf("Fail to serve: %v", e) - } - return true -} diff --git a/go/weed/mount.go b/go/weed/mount.go deleted file mode 100644 index e6f46c392..000000000 --- a/go/weed/mount.go +++ /dev/null @@ -1,35 +0,0 @@ -package main - -type MountOptions struct { - filer *string - dir *string -} - -var ( - mountOptions MountOptions -) - -func init() { - cmdMount.Run = runMount // break init cycle - cmdMount.IsDebug = cmdMount.Flag.Bool("debug", false, "verbose debug information") - mountOptions.filer = cmdMount.Flag.String("filer", "localhost:8888", "weed filer location") - mountOptions.dir = cmdMount.Flag.String("dir", ".", "mount weed filer to this directory") -} - -var cmdMount = &Command{ - UsageLine: "mount -filer=localhost:8888 -dir=/some/dir", - Short: "mount weed filer to a directory as file system in userspace(FUSE)", - Long: `mount weed filer to userspace. - - Pre-requisites: - 1) have SeaweedFS master and volume servers running - 2) have a "weed filer" running - These 2 requirements can be achieved with one command "weed server -filer=true" - - This uses bazil.org/fuse, whichenables writing FUSE file systems on - Linux, and OS X. - - On OS X, it requires OSXFUSE (http://osxfuse.github.com/). - - `, -} diff --git a/go/weed/mount_notsupported.go b/go/weed/mount_notsupported.go deleted file mode 100644 index bc77ffa16..000000000 --- a/go/weed/mount_notsupported.go +++ /dev/null @@ -1,15 +0,0 @@ -// +build !linux -// +build !darwin - -package main - -import ( - "fmt" - "runtime" -) - -func runMount(cmd *Command, args []string) bool { - fmt.Printf("Mount is not supported on %s %s\n", runtime.GOOS, runtime.GOARCH) - - return true -} diff --git a/go/weed/mount_std.go b/go/weed/mount_std.go deleted file mode 100644 index db097fb1d..000000000 --- a/go/weed/mount_std.go +++ /dev/null @@ -1,106 +0,0 @@ -// +build linux darwin - -package main - -import ( - "fmt" - "runtime" - - "bazil.org/fuse" - "bazil.org/fuse/fs" - "github.com/chrislusf/seaweedfs/go/filer" - "github.com/chrislusf/seaweedfs/go/glog" - "github.com/chrislusf/seaweedfs/go/storage" - "github.com/chrislusf/seaweedfs/go/util" - "golang.org/x/net/context" -) - -func runMount(cmd *Command, args []string) bool { - fmt.Printf("This is SeaweedFS version %s %s %s\n", util.VERSION, runtime.GOOS, runtime.GOARCH) - if *mountOptions.dir == "" { - fmt.Printf("Please specify the mount directory via \"-dir\"") - return false - } - - c, err := fuse.Mount(*mountOptions.dir) - if err != nil { - glog.Fatal(err) - return false - } - - OnInterrupt(func() { - fuse.Unmount(*mountOptions.dir) - c.Close() - }) - - err = fs.Serve(c, WFS{}) - if err != nil { - fuse.Unmount(*mountOptions.dir) - } - - // check if the mount process has an error to report - <-c.Ready - if err := c.MountError; err != nil { - glog.Fatal(err) - } - - return true -} - -type File struct { - FileId filer.FileId - Name string -} - -func (File) Attr(context context.Context, attr *fuse.Attr) error { - return nil -} -func (File) ReadAll(ctx context.Context) ([]byte, error) { - return []byte("hello, world\n"), nil -} - -type Dir struct { - Path string - Id uint64 -} - -func (dir Dir) Attr(context context.Context, attr *fuse.Attr) error { - return nil -} - -func (dir Dir) Lookup(ctx context.Context, name string) (fs.Node, error) { - files_result, e := filer.ListFiles(*mountOptions.filer, dir.Path, name) - if e != nil { - return nil, fuse.ENOENT - } - if len(files_result.Files) > 0 { - return File{files_result.Files[0].Id, files_result.Files[0].Name}, nil - } - return nil, fmt.Errorf("File Not Found for %s", name) -} - -type WFS struct{} - -func (WFS) Root() (fs.Node, error) { - return Dir{}, nil -} - -func (dir *Dir) ReadDir(ctx context.Context) ([]fuse.Dirent, error) { - var ret []fuse.Dirent - if dirs, e := filer.ListDirectories(*mountOptions.filer, dir.Path); e == nil { - for _, d := range dirs.Directories { - dirId := uint64(d.Id) - ret = append(ret, fuse.Dirent{Inode: dirId, Name: d.Name, Type: fuse.DT_Dir}) - } - } - if files, e := filer.ListFiles(*mountOptions.filer, dir.Path, ""); e == nil { - for _, f := range files.Files { - if fileId, e := storage.ParseFileId(string(f.Id)); e == nil { - fileInode := uint64(fileId.VolumeId)<<48 + fileId.Key - ret = append(ret, fuse.Dirent{Inode: fileInode, Name: f.Name, Type: fuse.DT_File}) - } - - } - } - return ret, nil -} diff --git a/go/weed/server.go b/go/weed/server.go deleted file mode 100644 index 242e50861..000000000 --- a/go/weed/server.go +++ /dev/null @@ -1,291 +0,0 @@ -package main - -import ( - "net/http" - "os" - "runtime" - "runtime/pprof" - "strconv" - "strings" - "sync" - "time" - - "github.com/chrislusf/seaweedfs/go/glog" - "github.com/chrislusf/seaweedfs/go/storage" - "github.com/chrislusf/seaweedfs/go/util" - "github.com/chrislusf/seaweedfs/go/weed/weed_server" - "github.com/gorilla/mux" -) - -type ServerOptions struct { - cpuprofile *string -} - -var ( - serverOptions ServerOptions - filerOptions FilerOptions -) - -func init() { - cmdServer.Run = runServer // break init cycle -} - -var cmdServer = &Command{ - UsageLine: "server -port=8080 -dir=/tmp -volume.max=5 -ip=server_name", - Short: "start a server, including volume server, and automatically elect a master server", - Long: `start both a volume server to provide storage spaces - and a master server to provide volume=>location mapping service and sequence number of file ids - - This is provided as a convenient way to start both volume server and master server. - The servers are exactly the same as starting them separately. - - So other volume servers can use this embedded master server also. - - Optionally, one filer server can be started. Logically, filer servers should not be in a cluster. - They run with meta data on disk, not shared. So each filer server is different. - - `, -} - -var ( - serverIp = cmdServer.Flag.String("ip", "localhost", "ip or server name") - serverBindIp = cmdServer.Flag.String("ip.bind", "0.0.0.0", "ip address to bind to") - serverMaxCpu = cmdServer.Flag.Int("maxCpu", 0, "maximum number of CPUs. 0 means all available CPUs") - serverTimeout = cmdServer.Flag.Int("idleTimeout", 10, "connection idle seconds") - serverDataCenter = cmdServer.Flag.String("dataCenter", "", "current volume server's data center name") - serverRack = cmdServer.Flag.String("rack", "", "current volume server's rack name") - serverWhiteListOption = cmdServer.Flag.String("whiteList", "", "comma separated Ip addresses having write permission. No limit if empty.") - serverPeers = cmdServer.Flag.String("master.peers", "", "other master nodes in comma separated ip:masterPort list") - serverSecureKey = cmdServer.Flag.String("secure.secret", "", "secret to encrypt Json Web Token(JWT)") - serverGarbageThreshold = cmdServer.Flag.String("garbageThreshold", "0.3", "threshold to vacuum and reclaim spaces") - masterPort = cmdServer.Flag.Int("master.port", 9333, "master server http listen port") - masterMetaFolder = cmdServer.Flag.String("master.dir", "", "data directory to store meta data, default to same as -dir specified") - masterVolumeSizeLimitMB = cmdServer.Flag.Uint("master.volumeSizeLimitMB", 30*1000, "Master stops directing writes to oversized volumes.") - masterConfFile = cmdServer.Flag.String("master.conf", "/etc/weedfs/weedfs.conf", "xml configuration file") - masterDefaultReplicaPlacement = cmdServer.Flag.String("master.defaultReplicaPlacement", "000", "Default replication type if not specified.") - volumePort = cmdServer.Flag.Int("volume.port", 8080, "volume server http listen port") - volumePublicPort = cmdServer.Flag.Int("volume.port.public", 0, "volume server public port") - volumeDataFolders = cmdServer.Flag.String("dir", os.TempDir(), "directories to store data files. dir[,dir]...") - volumeMaxDataVolumeCounts = cmdServer.Flag.String("volume.max", "7", "maximum numbers of volumes, count[,count]...") - volumePulse = cmdServer.Flag.Int("pulseSeconds", 5, "number of seconds between heartbeats") - volumeIndexType = cmdServer.Flag.String("volume.index", "memory", "Choose [memory|leveldb|boltdb] mode for memory~performance balance.") - volumeFixJpgOrientation = cmdServer.Flag.Bool("volume.images.fix.orientation", true, "Adjust jpg orientation when uploading.") - volumeReadRedirect = cmdServer.Flag.Bool("volume.read.redirect", true, "Redirect moved or non-local volumes.") - volumeServerPublicUrl = cmdServer.Flag.String("volume.publicUrl", "", "publicly accessible address") - isStartingFiler = cmdServer.Flag.Bool("filer", false, "whether to start filer") - - serverWhiteList []string -) - -func init() { - serverOptions.cpuprofile = cmdServer.Flag.String("cpuprofile", "", "cpu profile output file") - filerOptions.master = cmdServer.Flag.String("filer.master", "", "default to current master server") - filerOptions.collection = cmdServer.Flag.String("filer.collection", "", "all data will be stored in this collection") - filerOptions.port = cmdServer.Flag.Int("filer.port", 8888, "filer server http listen port") - filerOptions.dir = cmdServer.Flag.String("filer.dir", "", "directory to store meta data, default to a 'filer' sub directory of what -mdir is specified") - filerOptions.defaultReplicaPlacement = cmdServer.Flag.String("filer.defaultReplicaPlacement", "", "Default replication type if not specified during runtime.") - filerOptions.redirectOnRead = cmdServer.Flag.Bool("filer.redirectOnRead", false, "whether proxy or redirect to volume server during file GET request") - filerOptions.disableDirListing = cmdServer.Flag.Bool("filer.disableDirListing", false, "turn off directory listing") - filerOptions.cassandra_server = cmdServer.Flag.String("filer.cassandra.server", "", "host[:port] of the cassandra server") - filerOptions.cassandra_keyspace = cmdServer.Flag.String("filer.cassandra.keyspace", "seaweed", "keyspace of the cassandra server") - filerOptions.redis_server = cmdServer.Flag.String("filer.redis.server", "", "host:port of the redis server, e.g., 127.0.0.1:6379") - filerOptions.redis_password = cmdServer.Flag.String("filer.redis.password", "", "redis password in clear text") - filerOptions.redis_database = cmdServer.Flag.Int("filer.redis.database", 0, "the database on the redis server") -} - -func runServer(cmd *Command, args []string) bool { - filerOptions.secretKey = serverSecureKey - if *serverOptions.cpuprofile != "" { - f, err := os.Create(*serverOptions.cpuprofile) - if err != nil { - glog.Fatal(err) - } - pprof.StartCPUProfile(f) - defer pprof.StopCPUProfile() - } - - if *filerOptions.redirectOnRead { - *isStartingFiler = true - } - - *filerOptions.master = *serverIp + ":" + strconv.Itoa(*masterPort) - - if *filerOptions.defaultReplicaPlacement == "" { - *filerOptions.defaultReplicaPlacement = *masterDefaultReplicaPlacement - } - - if *volumePublicPort == 0 { - *volumePublicPort = *volumePort - } - - if *serverMaxCpu < 1 { - *serverMaxCpu = runtime.NumCPU() - } - runtime.GOMAXPROCS(*serverMaxCpu) - - folders := strings.Split(*volumeDataFolders, ",") - maxCountStrings := strings.Split(*volumeMaxDataVolumeCounts, ",") - var maxCounts []int - for _, maxString := range maxCountStrings { - if max, e := strconv.Atoi(maxString); e == nil { - maxCounts = append(maxCounts, max) - } else { - glog.Fatalf("The max specified in -max not a valid number %s", maxString) - } - } - if len(folders) != len(maxCounts) { - glog.Fatalf("%d directories by -dir, but only %d max is set by -max", len(folders), len(maxCounts)) - } - for _, folder := range folders { - if err := util.TestFolderWritable(folder); err != nil { - glog.Fatalf("Check Data Folder(-dir) Writable %s : %s", folder, err) - } - } - - if *masterMetaFolder == "" { - *masterMetaFolder = folders[0] - } - if *isStartingFiler { - if *filerOptions.dir == "" { - *filerOptions.dir = *masterMetaFolder + "/filer" - os.MkdirAll(*filerOptions.dir, 0700) - } - if err := util.TestFolderWritable(*filerOptions.dir); err != nil { - glog.Fatalf("Check Mapping Meta Folder (-filer.dir=\"%s\") Writable: %s", *filerOptions.dir, err) - } - } - if err := util.TestFolderWritable(*masterMetaFolder); err != nil { - glog.Fatalf("Check Meta Folder (-mdir=\"%s\") Writable: %s", *masterMetaFolder, err) - } - - if *serverWhiteListOption != "" { - serverWhiteList = strings.Split(*serverWhiteListOption, ",") - } - - if *isStartingFiler { - go func() { - r := http.NewServeMux() - _, nfs_err := weed_server.NewFilerServer(r, *serverBindIp, *filerOptions.port, *filerOptions.master, *filerOptions.dir, *filerOptions.collection, - *filerOptions.defaultReplicaPlacement, - *filerOptions.redirectOnRead, *filerOptions.disableDirListing, - *filerOptions.secretKey, - *filerOptions.cassandra_server, *filerOptions.cassandra_keyspace, - *filerOptions.redis_server, *filerOptions.redis_password, *filerOptions.redis_database, - ) - if nfs_err != nil { - glog.Fatalf("Filer startup error: %v", nfs_err) - } - glog.V(0).Infoln("Start Seaweed Filer", util.VERSION, "at port", strconv.Itoa(*filerOptions.port)) - filerListener, e := util.NewListener( - ":"+strconv.Itoa(*filerOptions.port), - time.Duration(10)*time.Second, - ) - if e != nil { - glog.Fatalf("Filer listener error: %v", e) - } - if e := http.Serve(filerListener, r); e != nil { - glog.Fatalf("Filer Fail to serve: %v", e) - } - }() - } - - var raftWaitForMaster sync.WaitGroup - var volumeWait sync.WaitGroup - - raftWaitForMaster.Add(1) - volumeWait.Add(1) - - go func() { - r := mux.NewRouter() - ms := weed_server.NewMasterServer(r, *masterPort, *masterMetaFolder, - *masterVolumeSizeLimitMB, *volumePulse, *masterConfFile, *masterDefaultReplicaPlacement, *serverGarbageThreshold, - serverWhiteList, *serverSecureKey, - ) - - glog.V(0).Infoln("Start Seaweed Master", util.VERSION, "at", *serverIp+":"+strconv.Itoa(*masterPort)) - masterListener, e := util.NewListener(*serverBindIp+":"+strconv.Itoa(*masterPort), time.Duration(*serverTimeout)*time.Second) - if e != nil { - glog.Fatalf("Master startup error: %v", e) - } - - go func() { - raftWaitForMaster.Wait() - time.Sleep(100 * time.Millisecond) - myAddress := *serverIp + ":" + strconv.Itoa(*masterPort) - var peers []string - if *serverPeers != "" { - peers = strings.Split(*serverPeers, ",") - } - raftServer := weed_server.NewRaftServer(r, peers, myAddress, *masterMetaFolder, ms.Topo, *volumePulse) - ms.SetRaftServer(raftServer) - volumeWait.Done() - }() - - raftWaitForMaster.Done() - if e := http.Serve(masterListener, r); e != nil { - glog.Fatalf("Master Fail to serve:%s", e.Error()) - } - }() - - volumeWait.Wait() - time.Sleep(100 * time.Millisecond) - if *volumePublicPort == 0 { - *volumePublicPort = *volumePort - } - if *volumeServerPublicUrl == "" { - *volumeServerPublicUrl = *serverIp + ":" + strconv.Itoa(*volumePublicPort) - } - isSeperatedPublicPort := *volumePublicPort != *volumePort - volumeMux := http.NewServeMux() - publicVolumeMux := volumeMux - if isSeperatedPublicPort { - publicVolumeMux = http.NewServeMux() - } - volumeNeedleMapKind := storage.NeedleMapInMemory - switch *volumeIndexType { - case "leveldb": - volumeNeedleMapKind = storage.NeedleMapLevelDb - case "boltdb": - volumeNeedleMapKind = storage.NeedleMapBoltDb - } - volumeServer := weed_server.NewVolumeServer(volumeMux, publicVolumeMux, - *serverIp, *volumePort, *volumeServerPublicUrl, - folders, maxCounts, - volumeNeedleMapKind, - *serverIp+":"+strconv.Itoa(*masterPort), *volumePulse, *serverDataCenter, *serverRack, - serverWhiteList, *volumeFixJpgOrientation, *volumeReadRedirect, - ) - - glog.V(0).Infoln("Start Seaweed volume server", util.VERSION, "at", *serverIp+":"+strconv.Itoa(*volumePort)) - volumeListener, eListen := util.NewListener( - *serverBindIp+":"+strconv.Itoa(*volumePort), - time.Duration(*serverTimeout)*time.Second, - ) - if eListen != nil { - glog.Fatalf("Volume server listener error: %v", eListen) - } - if isSeperatedPublicPort { - publicListeningAddress := *serverIp + ":" + strconv.Itoa(*volumePublicPort) - glog.V(0).Infoln("Start Seaweed volume server", util.VERSION, "public at", publicListeningAddress) - publicListener, e := util.NewListener(publicListeningAddress, time.Duration(*serverTimeout)*time.Second) - if e != nil { - glog.Fatalf("Volume server listener error:%v", e) - } - go func() { - if e := http.Serve(publicListener, publicVolumeMux); e != nil { - glog.Fatalf("Volume server fail to serve public: %v", e) - } - }() - } - - OnInterrupt(func() { - volumeServer.Shutdown() - pprof.StopCPUProfile() - }) - - if e := http.Serve(volumeListener, volumeMux); e != nil { - glog.Fatalf("Volume server fail to serve:%v", e) - } - - return true -} diff --git a/go/weed/shell.go b/go/weed/shell.go deleted file mode 100644 index 144621b09..000000000 --- a/go/weed/shell.go +++ /dev/null @@ -1,61 +0,0 @@ -package main - -import ( - "bufio" - "fmt" - "os" - - "github.com/chrislusf/seaweedfs/go/glog" -) - -func init() { - cmdShell.Run = runShell // break init cycle -} - -var cmdShell = &Command{ - UsageLine: "shell", - Short: "run interactive commands, now just echo", - Long: `run interactive commands. - - `, -} - -var () - -func runShell(command *Command, args []string) bool { - r := bufio.NewReader(os.Stdin) - o := bufio.NewWriter(os.Stdout) - e := bufio.NewWriter(os.Stderr) - prompt := func() { - var err error - if _, err = o.WriteString("> "); err != nil { - glog.V(0).Infoln("error writing to stdout:", err) - } - if err = o.Flush(); err != nil { - glog.V(0).Infoln("error flushing stdout:", err) - } - } - readLine := func() string { - ret, err := r.ReadString('\n') - if err != nil { - fmt.Fprint(e, err) - os.Exit(1) - } - return ret - } - execCmd := func(cmd string) int { - if cmd != "" { - if _, err := o.WriteString(cmd); err != nil { - glog.V(0).Infoln("error writing to stdout:", err) - } - } - return 0 - } - - cmd := "" - for { - prompt() - cmd = readLine() - execCmd(cmd) - } -} diff --git a/go/weed/signal_handling.go b/go/weed/signal_handling.go deleted file mode 100644 index a8f166382..000000000 --- a/go/weed/signal_handling.go +++ /dev/null @@ -1,31 +0,0 @@ -// +build !plan9 - -package main - -import ( - "os" - "os/signal" - "syscall" -) - -func OnInterrupt(fn func()) { - // deal with control+c,etc - signalChan := make(chan os.Signal, 1) - // controlling terminal close, daemon not exit - signal.Ignore(syscall.SIGHUP) - signal.Notify(signalChan, - os.Interrupt, - os.Kill, - syscall.SIGALRM, - // syscall.SIGHUP, - syscall.SIGINT, - syscall.SIGTERM, - // syscall.SIGQUIT, - ) - go func() { - for _ = range signalChan { - fn() - os.Exit(0) - } - }() -} diff --git a/go/weed/signal_handling_notsupported.go b/go/weed/signal_handling_notsupported.go deleted file mode 100644 index 343cf7de2..000000000 --- a/go/weed/signal_handling_notsupported.go +++ /dev/null @@ -1,6 +0,0 @@ -// +build plan9 - -package main - -func OnInterrupt(fn func()) { -} diff --git a/go/weed/upload.go b/go/weed/upload.go deleted file mode 100644 index df0e359b9..000000000 --- a/go/weed/upload.go +++ /dev/null @@ -1,108 +0,0 @@ -package main - -import ( - "encoding/json" - "fmt" - "os" - "path/filepath" - - "github.com/chrislusf/seaweedfs/go/operation" - "github.com/chrislusf/seaweedfs/go/security" -) - -var ( - upload UploadOptions -) - -type UploadOptions struct { - server *string - dir *string - include *string - replication *string - collection *string - ttl *string - maxMB *int - secretKey *string -} - -func init() { - cmdUpload.Run = runUpload // break init cycle - cmdUpload.IsDebug = cmdUpload.Flag.Bool("debug", false, "verbose debug information") - upload.server = cmdUpload.Flag.String("server", "localhost:9333", "SeaweedFS master location") - upload.dir = cmdUpload.Flag.String("dir", "", "Upload the whole folder recursively if specified.") - upload.include = cmdUpload.Flag.String("include", "", "pattens of files to upload, e.g., *.pdf, *.html, ab?d.txt, works together with -dir") - upload.replication = cmdUpload.Flag.String("replication", "", "replication type") - upload.collection = cmdUpload.Flag.String("collection", "", "optional collection name") - upload.ttl = cmdUpload.Flag.String("ttl", "", "time to live, e.g.: 1m, 1h, 1d, 1M, 1y") - upload.maxMB = cmdUpload.Flag.Int("maxMB", 0, "split files larger than the limit") - upload.secretKey = cmdUpload.Flag.String("secure.secret", "", "secret to encrypt Json Web Token(JWT)") -} - -var cmdUpload = &Command{ - UsageLine: "upload -server=localhost:9333 file1 [file2 file3]\n weed upload -server=localhost:9333 -dir=one_directory -include=*.pdf", - Short: "upload one or a list of files", - Long: `upload one or a list of files, or batch upload one whole folder recursively. - - If uploading a list of files: - It uses consecutive file keys for the list of files. - e.g. If the file1 uses key k, file2 can be read via k_1 - - If uploading a whole folder recursively: - All files under the folder and subfolders will be uploaded, each with its own file key. - Optional parameter "-include" allows you to specify the file name patterns. - - If any file has a ".gz" extension, the content are considered gzipped already, and will be stored as is. - This can save volume server's gzipped processing and allow customizable gzip compression level. - The file name will strip out ".gz" and stored. For example, "jquery.js.gz" will be stored as "jquery.js". - - If "maxMB" is set to a positive number, files larger than it would be split into chunks and uploaded separatedly. - The list of file ids of those chunks would be stored in an additional chunk, and this additional chunk's file id would be returned. - - `, -} - -func runUpload(cmd *Command, args []string) bool { - secret := security.Secret(*upload.secretKey) - if len(cmdUpload.Flag.Args()) == 0 { - if *upload.dir == "" { - return false - } - filepath.Walk(*upload.dir, func(path string, info os.FileInfo, err error) error { - if err == nil { - if !info.IsDir() { - if *upload.include != "" { - if ok, _ := filepath.Match(*upload.include, filepath.Base(path)); !ok { - return nil - } - } - parts, e := operation.NewFileParts([]string{path}) - if e != nil { - return e - } - results, e := operation.SubmitFiles(*upload.server, parts, - *upload.replication, *upload.collection, - *upload.ttl, *upload.maxMB, secret) - bytes, _ := json.Marshal(results) - fmt.Println(string(bytes)) - if e != nil { - return e - } - } - } else { - fmt.Println(err) - } - return err - }) - } else { - parts, e := operation.NewFileParts(args) - if e != nil { - fmt.Println(e.Error()) - } - results, _ := operation.SubmitFiles(*upload.server, parts, - *upload.replication, *upload.collection, - *upload.ttl, *upload.maxMB, secret) - bytes, _ := json.Marshal(results) - fmt.Println(string(bytes)) - } - return true -} diff --git a/go/weed/version.go b/go/weed/version.go deleted file mode 100644 index 8fef546f4..000000000 --- a/go/weed/version.go +++ /dev/null @@ -1,24 +0,0 @@ -package main - -import ( - "fmt" - "runtime" - - "github.com/chrislusf/seaweedfs/go/util" -) - -var cmdVersion = &Command{ - Run: runVersion, - UsageLine: "version", - Short: "print SeaweedFS version", - Long: `Version prints the SeaweedFS version`, -} - -func runVersion(cmd *Command, args []string) bool { - if len(args) != 0 { - cmd.Usage() - } - - fmt.Printf("version %s %s %s\n", util.VERSION, runtime.GOOS, runtime.GOARCH) - return true -} diff --git a/go/weed/volume.go b/go/weed/volume.go deleted file mode 100644 index f1b12dae8..000000000 --- a/go/weed/volume.go +++ /dev/null @@ -1,165 +0,0 @@ -package main - -import ( - "net/http" - "os" - "runtime" - "strconv" - "strings" - "time" - - "github.com/chrislusf/seaweedfs/go/glog" - "github.com/chrislusf/seaweedfs/go/storage" - "github.com/chrislusf/seaweedfs/go/util" - "github.com/chrislusf/seaweedfs/go/weed/weed_server" -) - -var ( - v VolumeServerOptions -) - -type VolumeServerOptions struct { - port *int - publicPort *int - folders []string - folderMaxLimits []int - ip *string - publicUrl *string - bindIp *string - master *string - pulseSeconds *int - idleConnectionTimeout *int - maxCpu *int - dataCenter *string - rack *string - whiteList []string - indexType *string - fixJpgOrientation *bool - readRedirect *bool -} - -func init() { - cmdVolume.Run = runVolume // break init cycle - v.port = cmdVolume.Flag.Int("port", 8080, "http listen port") - v.publicPort = cmdVolume.Flag.Int("port.public", 0, "port opened to public") - v.ip = cmdVolume.Flag.String("ip", "", "ip or server name") - v.publicUrl = cmdVolume.Flag.String("publicUrl", "", "Publicly accessible address") - v.bindIp = cmdVolume.Flag.String("ip.bind", "0.0.0.0", "ip address to bind to") - v.master = cmdVolume.Flag.String("mserver", "localhost:9333", "master server location") - v.pulseSeconds = cmdVolume.Flag.Int("pulseSeconds", 5, "number of seconds between heartbeats, must be smaller than or equal to the master's setting") - v.idleConnectionTimeout = cmdVolume.Flag.Int("idleTimeout", 10, "connection idle seconds") - v.maxCpu = cmdVolume.Flag.Int("maxCpu", 0, "maximum number of CPUs. 0 means all available CPUs") - v.dataCenter = cmdVolume.Flag.String("dataCenter", "", "current volume server's data center name") - v.rack = cmdVolume.Flag.String("rack", "", "current volume server's rack name") - v.indexType = cmdVolume.Flag.String("index", "memory", "Choose [memory|leveldb|boltdb] mode for memory~performance balance.") - v.fixJpgOrientation = cmdVolume.Flag.Bool("images.fix.orientation", true, "Adjust jpg orientation when uploading.") - v.readRedirect = cmdVolume.Flag.Bool("read.redirect", true, "Redirect moved or non-local volumes.") -} - -var cmdVolume = &Command{ - UsageLine: "volume -port=8080 -dir=/tmp -max=5 -ip=server_name -mserver=localhost:9333", - Short: "start a volume server", - Long: `start a volume server to provide storage spaces - - `, -} - -var ( - volumeFolders = cmdVolume.Flag.String("dir", os.TempDir(), "directories to store data files. dir[,dir]...") - maxVolumeCounts = cmdVolume.Flag.String("max", "7", "maximum numbers of volumes, count[,count]...") - volumeWhiteListOption = cmdVolume.Flag.String("whiteList", "", "comma separated Ip addresses having write permission. No limit if empty.") -) - -func runVolume(cmd *Command, args []string) bool { - if *v.maxCpu < 1 { - *v.maxCpu = runtime.NumCPU() - } - runtime.GOMAXPROCS(*v.maxCpu) - - //Set multiple folders and each folder's max volume count limit' - v.folders = strings.Split(*volumeFolders, ",") - maxCountStrings := strings.Split(*maxVolumeCounts, ",") - for _, maxString := range maxCountStrings { - if max, e := strconv.Atoi(maxString); e == nil { - v.folderMaxLimits = append(v.folderMaxLimits, max) - } else { - glog.Fatalf("The max specified in -max not a valid number %s", maxString) - } - } - if len(v.folders) != len(v.folderMaxLimits) { - glog.Fatalf("%d directories by -dir, but only %d max is set by -max", len(v.folders), len(v.folderMaxLimits)) - } - for _, folder := range v.folders { - if err := util.TestFolderWritable(folder); err != nil { - glog.Fatalf("Check Data Folder(-dir) Writable %s : %s", folder, err) - } - } - - //security related white list configuration - if *volumeWhiteListOption != "" { - v.whiteList = strings.Split(*volumeWhiteListOption, ",") - } - - if *v.ip == "" { - *v.ip = "127.0.0.1" - } - - if *v.publicPort == 0 { - *v.publicPort = *v.port - } - if *v.publicUrl == "" { - *v.publicUrl = *v.ip + ":" + strconv.Itoa(*v.publicPort) - } - isSeperatedPublicPort := *v.publicPort != *v.port - - volumeMux := http.NewServeMux() - publicVolumeMux := volumeMux - if isSeperatedPublicPort { - publicVolumeMux = http.NewServeMux() - } - - volumeNeedleMapKind := storage.NeedleMapInMemory - switch *v.indexType { - case "leveldb": - volumeNeedleMapKind = storage.NeedleMapLevelDb - case "boltdb": - volumeNeedleMapKind = storage.NeedleMapBoltDb - } - volumeServer := weed_server.NewVolumeServer(volumeMux, publicVolumeMux, - *v.ip, *v.port, *v.publicUrl, - v.folders, v.folderMaxLimits, - volumeNeedleMapKind, - *v.master, *v.pulseSeconds, *v.dataCenter, *v.rack, - v.whiteList, - *v.fixJpgOrientation, *v.readRedirect, - ) - - listeningAddress := *v.bindIp + ":" + strconv.Itoa(*v.port) - glog.V(0).Infoln("Start Seaweed volume server", util.VERSION, "at", listeningAddress) - listener, e := util.NewListener(listeningAddress, time.Duration(*v.idleConnectionTimeout)*time.Second) - if e != nil { - glog.Fatalf("Volume server listener error:%v", e) - } - if isSeperatedPublicPort { - publicListeningAddress := *v.bindIp + ":" + strconv.Itoa(*v.publicPort) - glog.V(0).Infoln("Start Seaweed volume server", util.VERSION, "public at", publicListeningAddress) - publicListener, e := util.NewListener(publicListeningAddress, time.Duration(*v.idleConnectionTimeout)*time.Second) - if e != nil { - glog.Fatalf("Volume server listener error:%v", e) - } - go func() { - if e := http.Serve(publicListener, publicVolumeMux); e != nil { - glog.Fatalf("Volume server fail to serve public: %v", e) - } - }() - } - - OnInterrupt(func() { - volumeServer.Shutdown() - }) - - if e := http.Serve(listener, volumeMux); e != nil { - glog.Fatalf("Volume server fail to serve: %v", e) - } - return true -} diff --git a/go/weed/volume_test.go b/go/weed/volume_test.go deleted file mode 100644 index ba06cb985..000000000 --- a/go/weed/volume_test.go +++ /dev/null @@ -1,13 +0,0 @@ -package main - -import ( - "net/http" - "testing" - "time" - - "github.com/chrislusf/seaweedfs/go/glog" -) - -func TestXYZ(t *testing.T) { - glog.V(0).Infoln("Last-Modified", time.Unix(int64(1373273596), 0).UTC().Format(http.TimeFormat)) -} diff --git a/go/weed/weed.go b/go/weed/weed.go deleted file mode 100644 index 49fe17eaa..000000000 --- a/go/weed/weed.go +++ /dev/null @@ -1,184 +0,0 @@ -package main - -import ( - "flag" - "fmt" - "io" - "math/rand" - "os" - "strings" - "sync" - "text/template" - "time" - "unicode" - "unicode/utf8" - - "github.com/chrislusf/seaweedfs/go/glog" -) - -var IsDebug *bool -var server *string - -var commands = []*Command{ - cmdBenchmark, - cmdBackup, - cmdCompact, - cmdFix, - cmdServer, - cmdMaster, - cmdFiler, - cmdUpload, - cmdDownload, - cmdShell, - cmdVersion, - cmdVolume, - cmdExport, - cmdMount, -} - -var exitStatus = 0 -var exitMu sync.Mutex - -func setExitStatus(n int) { - exitMu.Lock() - if exitStatus < n { - exitStatus = n - } - exitMu.Unlock() -} - -func main() { - glog.MaxSize = 1024 * 1024 * 32 - rand.Seed(time.Now().UnixNano()) - flag.Usage = usage - flag.Parse() - - args := flag.Args() - if len(args) < 1 { - usage() - } - - if args[0] == "help" { - help(args[1:]) - for _, cmd := range commands { - if len(args) >= 2 && cmd.Name() == args[1] && cmd.Run != nil { - fmt.Fprintf(os.Stderr, "Default Parameters:\n") - cmd.Flag.PrintDefaults() - } - } - return - } - - for _, cmd := range commands { - if cmd.Name() == args[0] && cmd.Run != nil { - cmd.Flag.Usage = func() { cmd.Usage() } - cmd.Flag.Parse(args[1:]) - args = cmd.Flag.Args() - IsDebug = cmd.IsDebug - if !cmd.Run(cmd, args) { - fmt.Fprintf(os.Stderr, "\n") - cmd.Flag.Usage() - fmt.Fprintf(os.Stderr, "Default Parameters:\n") - cmd.Flag.PrintDefaults() - } - exit() - return - } - } - - fmt.Fprintf(os.Stderr, "weed: unknown subcommand %q\nRun 'weed help' for usage.\n", args[0]) - setExitStatus(2) - exit() -} - -var usageTemplate = ` -SeaweedFS: store billions of files and serve them fast! - -Usage: - - weed command [arguments] - -The commands are: -{{range .}}{{if .Runnable}} - {{.Name | printf "%-11s"}} {{.Short}}{{end}}{{end}} - -Use "weed help [command]" for more information about a command. - -` - -var helpTemplate = `{{if .Runnable}}Usage: weed {{.UsageLine}} -{{end}} - {{.Long}} -` - -// tmpl executes the given template text on data, writing the result to w. -func tmpl(w io.Writer, text string, data interface{}) { - t := template.New("top") - t.Funcs(template.FuncMap{"trim": strings.TrimSpace, "capitalize": capitalize}) - template.Must(t.Parse(text)) - if err := t.Execute(w, data); err != nil { - panic(err) - } -} - -func capitalize(s string) string { - if s == "" { - return s - } - r, n := utf8.DecodeRuneInString(s) - return string(unicode.ToTitle(r)) + s[n:] -} - -func printUsage(w io.Writer) { - tmpl(w, usageTemplate, commands) -} - -func usage() { - printUsage(os.Stderr) - fmt.Fprintf(os.Stderr, "For Logging, use \"weed [logging_options] [command]\". The logging options are:\n") - flag.PrintDefaults() - os.Exit(2) -} - -// help implements the 'help' command. -func help(args []string) { - if len(args) == 0 { - printUsage(os.Stdout) - // not exit 2: succeeded at 'weed help'. - return - } - if len(args) != 1 { - fmt.Fprintf(os.Stderr, "usage: weed help command\n\nToo many arguments given.\n") - os.Exit(2) // failed at 'weed help' - } - - arg := args[0] - - for _, cmd := range commands { - if cmd.Name() == arg { - tmpl(os.Stdout, helpTemplate, cmd) - // not exit 2: succeeded at 'weed help cmd'. - return - } - } - - fmt.Fprintf(os.Stderr, "Unknown help topic %#q. Run 'weed help'.\n", arg) - os.Exit(2) // failed at 'weed help cmd' -} - -var atexitFuncs []func() - -func atexit(f func()) { - atexitFuncs = append(atexitFuncs, f) -} - -func exit() { - for _, f := range atexitFuncs { - f() - } - os.Exit(exitStatus) -} - -func debug(params ...interface{}) { - glog.V(4).Infoln(params) -} diff --git a/go/weed/weed_server/common.go b/go/weed/weed_server/common.go deleted file mode 100644 index a7fa2de53..000000000 --- a/go/weed/weed_server/common.go +++ /dev/null @@ -1,179 +0,0 @@ -package weed_server - -import ( - "bytes" - "encoding/json" - "errors" - "fmt" - "net/http" - "path/filepath" - "strconv" - "strings" - "time" - - "github.com/chrislusf/seaweedfs/go/glog" - "github.com/chrislusf/seaweedfs/go/operation" - "github.com/chrislusf/seaweedfs/go/security" - "github.com/chrislusf/seaweedfs/go/stats" - "github.com/chrislusf/seaweedfs/go/storage" - "github.com/chrislusf/seaweedfs/go/util" -) - -var serverStats *stats.ServerStats -var startTime = time.Now() - -func init() { - serverStats = stats.NewServerStats() - go serverStats.Start() - -} - -func writeJson(w http.ResponseWriter, r *http.Request, httpStatus int, obj interface{}) (err error) { - var bytes []byte - if r.FormValue("pretty") != "" { - bytes, err = json.MarshalIndent(obj, "", " ") - } else { - bytes, err = json.Marshal(obj) - } - if err != nil { - return - } - callback := r.FormValue("callback") - if callback == "" { - w.Header().Set("Content-Type", "application/json") - w.WriteHeader(httpStatus) - _, err = w.Write(bytes) - } else { - w.Header().Set("Content-Type", "application/javascript") - w.WriteHeader(httpStatus) - if _, err = w.Write([]uint8(callback)); err != nil { - return - } - if _, err = w.Write([]uint8("(")); err != nil { - return - } - fmt.Fprint(w, string(bytes)) - if _, err = w.Write([]uint8(")")); err != nil { - return - } - } - - return -} - -// wrapper for writeJson - just logs errors -func writeJsonQuiet(w http.ResponseWriter, r *http.Request, httpStatus int, obj interface{}) { - if err := writeJson(w, r, httpStatus, obj); err != nil { - glog.V(0).Infof("error writing JSON %s: %v", obj, err) - } -} -func writeJsonError(w http.ResponseWriter, r *http.Request, httpStatus int, err error) { - m := make(map[string]interface{}) - m["error"] = err.Error() - writeJsonQuiet(w, r, httpStatus, m) -} - -func debug(params ...interface{}) { - glog.V(4).Infoln(params) -} - -func submitForClientHandler(w http.ResponseWriter, r *http.Request, masterUrl string) { - jwt := security.GetJwt(r) - m := make(map[string]interface{}) - if r.Method != "POST" { - writeJsonError(w, r, http.StatusMethodNotAllowed, errors.New("Only submit via POST!")) - return - } - - debug("parsing upload file...") - fname, data, mimeType, isGzipped, lastModified, _, _, pe := storage.ParseUpload(r) - if pe != nil { - writeJsonError(w, r, http.StatusBadRequest, pe) - return - } - - debug("assigning file id for", fname) - r.ParseForm() - assignResult, ae := operation.Assign(masterUrl, 1, r.FormValue("replication"), r.FormValue("collection"), r.FormValue("ttl")) - if ae != nil { - writeJsonError(w, r, http.StatusInternalServerError, ae) - return - } - - url := "http://" + assignResult.Url + "/" + assignResult.Fid - if lastModified != 0 { - url = url + "?ts=" + strconv.FormatUint(lastModified, 10) - } - - debug("upload file to store", url) - uploadResult, err := operation.Upload(url, fname, bytes.NewReader(data), isGzipped, mimeType, jwt) - if err != nil { - writeJsonError(w, r, http.StatusInternalServerError, err) - return - } - - m["fileName"] = fname - m["fid"] = assignResult.Fid - m["fileUrl"] = assignResult.PublicUrl + "/" + assignResult.Fid - m["size"] = uploadResult.Size - writeJsonQuiet(w, r, http.StatusCreated, m) - return -} - -func deleteForClientHandler(w http.ResponseWriter, r *http.Request, masterUrl string) { - r.ParseForm() - fids := r.Form["fid"] - ret, err := operation.DeleteFiles(masterUrl, fids) - if err != nil { - writeJsonError(w, r, http.StatusInternalServerError, err) - return - } - writeJsonQuiet(w, r, http.StatusAccepted, ret) -} - -func parseURLPath(path string) (vid, fid, filename, ext string, isVolumeIdOnly bool) { - switch strings.Count(path, "/") { - case 3: - parts := strings.Split(path, "/") - vid, fid, filename = parts[1], parts[2], parts[3] - ext = filepath.Ext(filename) - case 2: - parts := strings.Split(path, "/") - vid, fid = parts[1], parts[2] - dotIndex := strings.LastIndex(fid, ".") - if dotIndex > 0 { - ext = fid[dotIndex:] - fid = fid[0:dotIndex] - } - default: - sepIndex := strings.LastIndex(path, "/") - commaIndex := strings.LastIndex(path[sepIndex:], ",") - if commaIndex <= 0 { - vid, isVolumeIdOnly = path[sepIndex+1:], true - return - } - dotIndex := strings.LastIndex(path[sepIndex:], ".") - vid = path[sepIndex+1 : commaIndex] - fid = path[commaIndex+1:] - ext = "" - if dotIndex > 0 { - fid = path[commaIndex+1 : dotIndex] - ext = path[dotIndex:] - } - } - return -} - -func statsCounterHandler(w http.ResponseWriter, r *http.Request) { - m := make(map[string]interface{}) - m["Version"] = util.VERSION - m["Counters"] = serverStats - writeJsonQuiet(w, r, http.StatusOK, m) -} - -func statsMemoryHandler(w http.ResponseWriter, r *http.Request) { - m := make(map[string]interface{}) - m["Version"] = util.VERSION - m["Memory"] = stats.MemStat() - writeJsonQuiet(w, r, http.StatusOK, m) -} diff --git a/go/weed/weed_server/filer_server.go b/go/weed/weed_server/filer_server.go deleted file mode 100644 index e3c45d9e5..000000000 --- a/go/weed/weed_server/filer_server.go +++ /dev/null @@ -1,67 +0,0 @@ -package weed_server - -import ( - "net/http" - "strconv" - - "github.com/chrislusf/seaweedfs/go/filer" - "github.com/chrislusf/seaweedfs/go/filer/cassandra_store" - "github.com/chrislusf/seaweedfs/go/filer/embedded_filer" - "github.com/chrislusf/seaweedfs/go/filer/flat_namespace" - "github.com/chrislusf/seaweedfs/go/filer/redis_store" - "github.com/chrislusf/seaweedfs/go/glog" - "github.com/chrislusf/seaweedfs/go/security" -) - -type FilerServer struct { - port string - master string - collection string - defaultReplication string - redirectOnRead bool - disableDirListing bool - secret security.Secret - filer filer.Filer -} - -func NewFilerServer(r *http.ServeMux, ip string, port int, master string, dir string, collection string, - replication string, redirectOnRead bool, disableDirListing bool, - secret string, - cassandra_server string, cassandra_keyspace string, - redis_server string, redis_password string, redis_database int, -) (fs *FilerServer, err error) { - fs = &FilerServer{ - master: master, - collection: collection, - defaultReplication: replication, - redirectOnRead: redirectOnRead, - disableDirListing: disableDirListing, - port: ip + ":" + strconv.Itoa(port), - } - - if cassandra_server != "" { - cassandra_store, err := cassandra_store.NewCassandraStore(cassandra_keyspace, cassandra_server) - if err != nil { - glog.Fatalf("Can not connect to cassandra server %s with keyspace %s: %v", cassandra_server, cassandra_keyspace, err) - } - fs.filer = flat_namespace.NewFlatNamespaceFiler(master, cassandra_store) - } else if redis_server != "" { - redis_store := redis_store.NewRedisStore(redis_server, redis_password, redis_database) - fs.filer = flat_namespace.NewFlatNamespaceFiler(master, redis_store) - } else { - if fs.filer, err = embedded_filer.NewFilerEmbedded(master, dir); err != nil { - glog.Fatalf("Can not start filer in dir %s : %v", dir, err) - return - } - - r.HandleFunc("/admin/mv", fs.moveHandler) - } - - r.HandleFunc("/", fs.filerHandler) - - return fs, nil -} - -func (fs *FilerServer) jwt(fileId string) security.EncodedJwt { - return security.GenJwt(fs.secret, fileId) -} diff --git a/go/weed/weed_server/filer_server_handlers.go b/go/weed/weed_server/filer_server_handlers.go deleted file mode 100644 index efc4c0381..000000000 --- a/go/weed/weed_server/filer_server_handlers.go +++ /dev/null @@ -1,265 +0,0 @@ -package weed_server - -import ( - "bytes" - "encoding/json" - "errors" - "io" - "io/ioutil" - "net/http" - "net/url" - "strconv" - "strings" - - "github.com/chrislusf/seaweedfs/go/glog" - "github.com/chrislusf/seaweedfs/go/operation" - "github.com/chrislusf/seaweedfs/go/storage" - "github.com/chrislusf/seaweedfs/go/util" - "github.com/syndtr/goleveldb/leveldb" -) - -func (fs *FilerServer) filerHandler(w http.ResponseWriter, r *http.Request) { - switch r.Method { - case "GET": - fs.GetOrHeadHandler(w, r, true) - case "HEAD": - fs.GetOrHeadHandler(w, r, false) - case "DELETE": - fs.DeleteHandler(w, r) - case "PUT": - fs.PostHandler(w, r) - case "POST": - fs.PostHandler(w, r) - } -} - -// listDirectoryHandler lists directories and folers under a directory -// files are sorted by name and paginated via "lastFileName" and "limit". -// sub directories are listed on the first page, when "lastFileName" -// is empty. -func (fs *FilerServer) listDirectoryHandler(w http.ResponseWriter, r *http.Request) { - if !strings.HasSuffix(r.URL.Path, "/") { - return - } - dirlist, err := fs.filer.ListDirectories(r.URL.Path) - if err == leveldb.ErrNotFound { - glog.V(3).Infoln("Directory Not Found in db", r.URL.Path) - w.WriteHeader(http.StatusNotFound) - return - } - m := make(map[string]interface{}) - m["Directory"] = r.URL.Path - lastFileName := r.FormValue("lastFileName") - if lastFileName == "" { - m["Subdirectories"] = dirlist - } - limit, limit_err := strconv.Atoi(r.FormValue("limit")) - if limit_err != nil { - limit = 100 - } - m["Files"], _ = fs.filer.ListFiles(r.URL.Path, lastFileName, limit) - writeJsonQuiet(w, r, http.StatusOK, m) -} - -func (fs *FilerServer) GetOrHeadHandler(w http.ResponseWriter, r *http.Request, isGetMethod bool) { - if strings.HasSuffix(r.URL.Path, "/") { - if fs.disableDirListing { - w.WriteHeader(http.StatusMethodNotAllowed) - return - } - fs.listDirectoryHandler(w, r) - return - } - - fileId, err := fs.filer.FindFile(r.URL.Path) - if err == leveldb.ErrNotFound { - glog.V(3).Infoln("Not found in db", r.URL.Path) - w.WriteHeader(http.StatusNotFound) - return - } - - urlLocation, err := operation.LookupFileId(fs.master, fileId) - if err != nil { - glog.V(1).Infoln("operation LookupFileId %s failed, err is %s", fileId, err.Error()) - w.WriteHeader(http.StatusNotFound) - return - } - urlString := urlLocation - if fs.redirectOnRead { - http.Redirect(w, r, urlString, http.StatusFound) - return - } - u, _ := url.Parse(urlString) - request := &http.Request{ - Method: r.Method, - URL: u, - Proto: r.Proto, - ProtoMajor: r.ProtoMajor, - ProtoMinor: r.ProtoMinor, - Header: r.Header, - Body: r.Body, - Host: r.Host, - ContentLength: r.ContentLength, - } - glog.V(3).Infoln("retrieving from", u) - resp, do_err := util.Do(request) - if do_err != nil { - glog.V(0).Infoln("failing to connect to volume server", do_err.Error()) - writeJsonError(w, r, http.StatusInternalServerError, do_err) - return - } - defer resp.Body.Close() - for k, v := range resp.Header { - w.Header()[k] = v - } - w.WriteHeader(resp.StatusCode) - io.Copy(w, resp.Body) -} - -type analogueReader struct { - *bytes.Buffer -} - -// So that it implements the io.ReadCloser interface -func (m analogueReader) Close() error { return nil } - -func (fs *FilerServer) PostHandler(w http.ResponseWriter, r *http.Request) { - query := r.URL.Query() - replication := query.Get("replication") - if replication == "" { - replication = fs.defaultReplication - } - collection := query.Get("collection") - if collection == "" { - collection = fs.collection - } - - var fileId string - var err error - var urlLocation string - if r.Method == "PUT" { - buf, _ := ioutil.ReadAll(r.Body) - r.Body = analogueReader{bytes.NewBuffer(buf)} - fileName, _, _, _, _, _, _, pe := storage.ParseUpload(r) - if pe != nil { - glog.V(0).Infoln("failing to parse post body", pe.Error()) - writeJsonError(w, r, http.StatusInternalServerError, pe) - return - } - //reconstruct http request body for following new request to volume server - r.Body = analogueReader{bytes.NewBuffer(buf)} - - path := r.URL.Path - if strings.HasSuffix(path, "/") { - if fileName != "" { - path += fileName - } - } - - if fileId, err = fs.filer.FindFile(path); err != nil && err != leveldb.ErrNotFound { - glog.V(0).Infoln("failing to find path in filer store", path, err.Error()) - writeJsonError(w, r, http.StatusInternalServerError, err) - return - } else if fileId != "" && err == nil { - var le error - urlLocation, le = operation.LookupFileId(fs.master, fileId) - if le != nil { - glog.V(1).Infoln("operation LookupFileId %s failed, err is %s", fileId, le.Error()) - w.WriteHeader(http.StatusNotFound) - return - } - } - } else { - assignResult, ae := operation.Assign(fs.master, 1, replication, collection, query.Get("ttl")) - if ae != nil { - glog.V(0).Infoln("failing to assign a file id", ae.Error()) - writeJsonError(w, r, http.StatusInternalServerError, ae) - return - } - fileId = assignResult.Fid - urlLocation = "http://" + assignResult.Url + "/" + assignResult.Fid - } - - u, _ := url.Parse(urlLocation) - glog.V(4).Infoln("post to", u) - request := &http.Request{ - Method: r.Method, - URL: u, - Proto: r.Proto, - ProtoMajor: r.ProtoMajor, - ProtoMinor: r.ProtoMinor, - Header: r.Header, - Body: r.Body, - Host: r.Host, - ContentLength: r.ContentLength, - } - resp, do_err := util.Do(request) - if do_err != nil { - glog.V(0).Infoln("failing to connect to volume server", r.RequestURI, do_err.Error()) - writeJsonError(w, r, http.StatusInternalServerError, do_err) - return - } - defer resp.Body.Close() - resp_body, ra_err := ioutil.ReadAll(resp.Body) - if ra_err != nil { - glog.V(0).Infoln("failing to upload to volume server", r.RequestURI, ra_err.Error()) - writeJsonError(w, r, http.StatusInternalServerError, ra_err) - return - } - glog.V(4).Infoln("post result", string(resp_body)) - var ret operation.UploadResult - unmarshal_err := json.Unmarshal(resp_body, &ret) - if unmarshal_err != nil { - glog.V(0).Infoln("failing to read upload resonse", r.RequestURI, string(resp_body)) - writeJsonError(w, r, http.StatusInternalServerError, unmarshal_err) - return - } - if ret.Error != "" { - glog.V(0).Infoln("failing to post to volume server", r.RequestURI, ret.Error) - writeJsonError(w, r, http.StatusInternalServerError, errors.New(ret.Error)) - return - } - path := r.URL.Path - if strings.HasSuffix(path, "/") { - if ret.Name != "" { - path += ret.Name - } else { - operation.DeleteFile(fs.master, fileId, fs.jwt(fileId)) //clean up - glog.V(0).Infoln("Can not to write to folder", path, "without a file name!") - writeJsonError(w, r, http.StatusInternalServerError, - errors.New("Can not to write to folder "+path+" without a file name")) - return - } - } - glog.V(4).Infoln("saving", path, "=>", fileId) - if db_err := fs.filer.CreateFile(path, fileId); db_err != nil { - operation.DeleteFile(fs.master, fileId, fs.jwt(fileId)) //clean up - glog.V(0).Infof("failing to write %s to filer server : %v", path, db_err) - writeJsonError(w, r, http.StatusInternalServerError, db_err) - return - } - w.WriteHeader(http.StatusCreated) - w.Write(resp_body) -} - -// curl -X DELETE http://localhost:8888/path/to -// curl -X DELETE http://localhost:8888/path/to?recursive=true -func (fs *FilerServer) DeleteHandler(w http.ResponseWriter, r *http.Request) { - var err error - var fid string - if strings.HasSuffix(r.URL.Path, "/") { - isRecursive := r.FormValue("recursive") == "true" - err = fs.filer.DeleteDirectory(r.URL.Path, isRecursive) - } else { - fid, err = fs.filer.DeleteFile(r.URL.Path) - if err == nil && fid != "" { - err = operation.DeleteFile(fs.master, fid, fs.jwt(fid)) - } - } - if err == nil { - writeJsonQuiet(w, r, http.StatusAccepted, map[string]string{"error": ""}) - } else { - glog.V(4).Infoln("deleting", r.URL.Path, ":", err.Error()) - writeJsonError(w, r, http.StatusInternalServerError, err) - } -} diff --git a/go/weed/weed_server/filer_server_handlers_admin.go b/go/weed/weed_server/filer_server_handlers_admin.go deleted file mode 100644 index 2f317ff79..000000000 --- a/go/weed/weed_server/filer_server_handlers_admin.go +++ /dev/null @@ -1,29 +0,0 @@ -package weed_server - -import ( - "net/http" - - "github.com/chrislusf/seaweedfs/go/glog" -) - -/* -Move a folder or a file, with 4 Use cases: - mv fromDir toNewDir - mv fromDir toOldDir - mv fromFile toDir - mv fromFile toFile - -Wildcard is not supported. - -*/ -func (fs *FilerServer) moveHandler(w http.ResponseWriter, r *http.Request) { - from := r.FormValue("from") - to := r.FormValue("to") - err := fs.filer.Move(from, to) - if err != nil { - glog.V(4).Infoln("moving", from, "->", to, err.Error()) - writeJsonError(w, r, http.StatusInternalServerError, err) - } else { - w.WriteHeader(http.StatusOK) - } -} diff --git a/go/weed/weed_server/master_server.go b/go/weed/weed_server/master_server.go deleted file mode 100644 index db70ca6b1..000000000 --- a/go/weed/weed_server/master_server.go +++ /dev/null @@ -1,131 +0,0 @@ -package weed_server - -import ( - "fmt" - "net/http" - "net/http/httputil" - "net/url" - "sync" - - "github.com/chrislusf/raft" - "github.com/chrislusf/seaweedfs/go/glog" - "github.com/chrislusf/seaweedfs/go/security" - "github.com/chrislusf/seaweedfs/go/sequence" - "github.com/chrislusf/seaweedfs/go/topology" - "github.com/chrislusf/seaweedfs/go/util" - "github.com/gorilla/mux" -) - -type MasterServer struct { - port int - metaFolder string - volumeSizeLimitMB uint - pulseSeconds int - defaultReplicaPlacement string - garbageThreshold string - guard *security.Guard - - Topo *topology.Topology - vg *topology.VolumeGrowth - vgLock sync.Mutex - - bounedLeaderChan chan int -} - -func NewMasterServer(r *mux.Router, port int, metaFolder string, - volumeSizeLimitMB uint, - pulseSeconds int, - confFile string, - defaultReplicaPlacement string, - garbageThreshold string, - whiteList []string, - secureKey string, -) *MasterServer { - ms := &MasterServer{ - port: port, - volumeSizeLimitMB: volumeSizeLimitMB, - pulseSeconds: pulseSeconds, - defaultReplicaPlacement: defaultReplicaPlacement, - garbageThreshold: garbageThreshold, - } - ms.bounedLeaderChan = make(chan int, 16) - seq := sequence.NewMemorySequencer() - var e error - if ms.Topo, e = topology.NewTopology("topo", confFile, seq, - uint64(volumeSizeLimitMB)*1024*1024, pulseSeconds); e != nil { - glog.Fatalf("cannot create topology:%s", e) - } - ms.vg = topology.NewDefaultVolumeGrowth() - glog.V(0).Infoln("Volume Size Limit is", volumeSizeLimitMB, "MB") - - ms.guard = security.NewGuard(whiteList, secureKey) - - r.HandleFunc("/", ms.uiStatusHandler) - r.HandleFunc("/ui/index.html", ms.uiStatusHandler) - r.HandleFunc("/dir/assign", ms.proxyToLeader(ms.guard.WhiteList(ms.dirAssignHandler))) - r.HandleFunc("/dir/lookup", ms.proxyToLeader(ms.guard.WhiteList(ms.dirLookupHandler))) - r.HandleFunc("/dir/join", ms.proxyToLeader(ms.guard.WhiteList(ms.dirJoinHandler))) - r.HandleFunc("/dir/status", ms.proxyToLeader(ms.guard.WhiteList(ms.dirStatusHandler))) - r.HandleFunc("/col/delete", ms.proxyToLeader(ms.guard.WhiteList(ms.collectionDeleteHandler))) - r.HandleFunc("/vol/lookup", ms.proxyToLeader(ms.guard.WhiteList(ms.volumeLookupHandler))) - r.HandleFunc("/vol/grow", ms.proxyToLeader(ms.guard.WhiteList(ms.volumeGrowHandler))) - r.HandleFunc("/vol/status", ms.proxyToLeader(ms.guard.WhiteList(ms.volumeStatusHandler))) - r.HandleFunc("/vol/vacuum", ms.proxyToLeader(ms.guard.WhiteList(ms.volumeVacuumHandler))) - r.HandleFunc("/submit", ms.guard.WhiteList(ms.submitFromMasterServerHandler)) - r.HandleFunc("/delete", ms.guard.WhiteList(ms.deleteFromMasterServerHandler)) - r.HandleFunc("/{fileId}", ms.proxyToLeader(ms.redirectHandler)) - r.HandleFunc("/stats/counter", ms.guard.WhiteList(statsCounterHandler)) - r.HandleFunc("/stats/memory", ms.guard.WhiteList(statsMemoryHandler)) - - ms.Topo.StartRefreshWritableVolumes(garbageThreshold) - - return ms -} - -func (ms *MasterServer) SetRaftServer(raftServer *RaftServer) { - ms.Topo.RaftServer = raftServer.raftServer - ms.Topo.RaftServer.AddEventListener(raft.LeaderChangeEventType, func(e raft.Event) { - if ms.Topo.RaftServer.Leader() != "" { - glog.V(0).Infoln("[", ms.Topo.RaftServer.Name(), "]", ms.Topo.RaftServer.Leader(), "becomes leader.") - } - }) - if ms.Topo.IsLeader() { - glog.V(0).Infoln("[", ms.Topo.RaftServer.Name(), "]", "I am the leader!") - } else { - if ms.Topo.RaftServer.Leader() != "" { - glog.V(0).Infoln("[", ms.Topo.RaftServer.Name(), "]", ms.Topo.RaftServer.Leader(), "is the leader.") - } - } -} - -func (ms *MasterServer) proxyToLeader(f func(w http.ResponseWriter, r *http.Request)) func(w http.ResponseWriter, r *http.Request) { - return func(w http.ResponseWriter, r *http.Request) { - if ms.Topo.IsLeader() { - f(w, r) - } else if ms.Topo.RaftServer != nil && ms.Topo.RaftServer.Leader() != "" { - ms.bounedLeaderChan <- 1 - defer func() { <-ms.bounedLeaderChan }() - targetUrl, err := url.Parse("http://" + ms.Topo.RaftServer.Leader()) - if err != nil { - writeJsonError(w, r, http.StatusInternalServerError, - fmt.Errorf("Leader URL http://%s Parse Error: %v", ms.Topo.RaftServer.Leader(), err)) - return - } - glog.V(4).Infoln("proxying to leader", ms.Topo.RaftServer.Leader()) - proxy := httputil.NewSingleHostReverseProxy(targetUrl) - director := proxy.Director - proxy.Director = func(req *http.Request) { - actualHost, err := security.GetActualRemoteHost(req) - if err == nil { - req.Header.Set("HTTP_X_FORWARDED_FOR", actualHost) - } - director(req) - } - proxy.Transport = util.Transport - proxy.ServeHTTP(w, r) - } else { - //drop it to the floor - //writeJsonError(w, r, errors.New(ms.Topo.RaftServer.Name()+" does not know Leader yet:"+ms.Topo.RaftServer.Leader())) - } - } -} diff --git a/go/weed/weed_server/master_server_handlers.go b/go/weed/weed_server/master_server_handlers.go deleted file mode 100644 index 2be5d9524..000000000 --- a/go/weed/weed_server/master_server_handlers.go +++ /dev/null @@ -1,104 +0,0 @@ -package weed_server - -import ( - "fmt" - "net/http" - "strconv" - "strings" - - "github.com/chrislusf/seaweedfs/go/operation" - "github.com/chrislusf/seaweedfs/go/stats" - "github.com/chrislusf/seaweedfs/go/storage" -) - -func (ms *MasterServer) lookupVolumeId(vids []string, collection string) (volumeLocations map[string]operation.LookupResult) { - volumeLocations = make(map[string]operation.LookupResult) - for _, vid := range vids { - commaSep := strings.Index(vid, ",") - if commaSep > 0 { - vid = vid[0:commaSep] - } - if _, ok := volumeLocations[vid]; ok { - continue - } - volumeId, err := storage.NewVolumeId(vid) - if err == nil { - machines := ms.Topo.Lookup(collection, volumeId) - if machines != nil { - var ret []operation.Location - for _, dn := range machines { - ret = append(ret, operation.Location{Url: dn.Url(), PublicUrl: dn.PublicUrl}) - } - volumeLocations[vid] = operation.LookupResult{VolumeId: vid, Locations: ret} - } else { - volumeLocations[vid] = operation.LookupResult{VolumeId: vid, Error: "volumeId not found."} - } - } else { - volumeLocations[vid] = operation.LookupResult{VolumeId: vid, Error: "Unknown volumeId format."} - } - } - return -} - -// Takes one volumeId only, can not do batch lookup -func (ms *MasterServer) dirLookupHandler(w http.ResponseWriter, r *http.Request) { - vid := r.FormValue("volumeId") - commaSep := strings.Index(vid, ",") - if commaSep > 0 { - vid = vid[0:commaSep] - } - vids := []string{vid} - collection := r.FormValue("collection") //optional, but can be faster if too many collections - volumeLocations := ms.lookupVolumeId(vids, collection) - location := volumeLocations[vid] - httpStatus := http.StatusOK - if location.Error != "" { - httpStatus = http.StatusNotFound - } - writeJsonQuiet(w, r, httpStatus, location) -} - -// This can take batched volumeIds, &volumeId=x&volumeId=y&volumeId=z -func (ms *MasterServer) volumeLookupHandler(w http.ResponseWriter, r *http.Request) { - r.ParseForm() - vids := r.Form["volumeId"] - collection := r.FormValue("collection") //optional, but can be faster if too many collections - volumeLocations := ms.lookupVolumeId(vids, collection) - writeJsonQuiet(w, r, http.StatusOK, volumeLocations) -} - -func (ms *MasterServer) dirAssignHandler(w http.ResponseWriter, r *http.Request) { - stats.AssignRequest() - requestedCount, e := strconv.ParseUint(r.FormValue("count"), 10, 64) - if e != nil || requestedCount == 0 { - requestedCount = 1 - } - - option, err := ms.getVolumeGrowOption(r) - if err != nil { - writeJsonQuiet(w, r, http.StatusNotAcceptable, operation.AssignResult{Error: err.Error()}) - return - } - - if !ms.Topo.HasWritableVolume(option) { - if ms.Topo.FreeSpace() <= 0 { - writeJsonQuiet(w, r, http.StatusNotFound, operation.AssignResult{Error: "No free volumes left!"}) - return - } - ms.vgLock.Lock() - defer ms.vgLock.Unlock() - if !ms.Topo.HasWritableVolume(option) { - if _, err = ms.vg.AutomaticGrowByType(option, ms.Topo); err != nil { - writeJsonError(w, r, http.StatusInternalServerError, - fmt.Errorf("Cannot grow volume group! %v", err)) - return - } - } - } - fid, count, dn, err := ms.Topo.PickForWrite(requestedCount, option) - if err == nil { - writeJsonQuiet(w, r, http.StatusOK, operation.AssignResult{Fid: fid, Url: dn.Url(), PublicUrl: dn.PublicUrl, Count: count}) - } else { - writeJsonQuiet(w, r, http.StatusNotAcceptable, operation.AssignResult{Error: err.Error()}) - } -} diff --git a/go/weed/weed_server/master_server_handlers_admin.go b/go/weed/weed_server/master_server_handlers_admin.go deleted file mode 100644 index 07399596a..000000000 --- a/go/weed/weed_server/master_server_handlers_admin.go +++ /dev/null @@ -1,193 +0,0 @@ -package weed_server - -import ( - "encoding/json" - "errors" - "fmt" - "io/ioutil" - "math/rand" - "net/http" - "strconv" - "strings" - - "github.com/chrislusf/seaweedfs/go/glog" - "github.com/chrislusf/seaweedfs/go/operation" - "github.com/chrislusf/seaweedfs/go/storage" - "github.com/chrislusf/seaweedfs/go/topology" - "github.com/chrislusf/seaweedfs/go/util" - "github.com/golang/protobuf/proto" -) - -func (ms *MasterServer) collectionDeleteHandler(w http.ResponseWriter, r *http.Request) { - collection, ok := ms.Topo.FindCollection(r.FormValue("collection")) - if !ok { - writeJsonError(w, r, http.StatusBadRequest, fmt.Errorf("collection %s does not exist", r.FormValue("collection"))) - return - } - for _, server := range collection.ListVolumeServers() { - _, err := util.Get("http://" + server.Ip + ":" + strconv.Itoa(server.Port) + "/admin/delete_collection?collection=" + r.FormValue("collection")) - if err != nil { - writeJsonError(w, r, http.StatusInternalServerError, err) - return - } - } - ms.Topo.DeleteCollection(r.FormValue("collection")) -} - -func (ms *MasterServer) dirJoinHandler(w http.ResponseWriter, r *http.Request) { - body, err := ioutil.ReadAll(r.Body) - if err != nil { - writeJsonError(w, r, http.StatusBadRequest, err) - return - } - joinMessage := &operation.JoinMessage{} - if err = proto.Unmarshal(body, joinMessage); err != nil { - writeJsonError(w, r, http.StatusBadRequest, err) - return - } - if *joinMessage.Ip == "" { - *joinMessage.Ip = r.RemoteAddr[0:strings.LastIndex(r.RemoteAddr, ":")] - } - if glog.V(4) { - if jsonData, jsonError := json.Marshal(joinMessage); jsonError != nil { - glog.V(0).Infoln("json marshaling error: ", jsonError) - writeJsonError(w, r, http.StatusBadRequest, jsonError) - return - } else { - glog.V(4).Infoln("Proto size", len(body), "json size", len(jsonData), string(jsonData)) - } - } - - ms.Topo.ProcessJoinMessage(joinMessage) - writeJsonQuiet(w, r, http.StatusOK, operation.JoinResult{ - VolumeSizeLimit: uint64(ms.volumeSizeLimitMB) * 1024 * 1024, - SecretKey: string(ms.guard.SecretKey), - }) -} - -func (ms *MasterServer) dirStatusHandler(w http.ResponseWriter, r *http.Request) { - m := make(map[string]interface{}) - m["Version"] = util.VERSION - m["Topology"] = ms.Topo.ToMap() - writeJsonQuiet(w, r, http.StatusOK, m) -} - -func (ms *MasterServer) volumeVacuumHandler(w http.ResponseWriter, r *http.Request) { - gcThreshold := r.FormValue("garbageThreshold") - if gcThreshold == "" { - gcThreshold = ms.garbageThreshold - } - glog.Infoln("garbageThreshold =", gcThreshold) - ms.Topo.Vacuum(gcThreshold) - ms.dirStatusHandler(w, r) -} - -func (ms *MasterServer) volumeGrowHandler(w http.ResponseWriter, r *http.Request) { - count := 0 - option, err := ms.getVolumeGrowOption(r) - if err != nil { - writeJsonError(w, r, http.StatusNotAcceptable, err) - return - } - if err == nil { - if count, err = strconv.Atoi(r.FormValue("count")); err == nil { - if ms.Topo.FreeSpace() < count*option.ReplicaPlacement.GetCopyCount() { - err = errors.New("Only " + strconv.Itoa(ms.Topo.FreeSpace()) + " volumes left! Not enough for " + strconv.Itoa(count*option.ReplicaPlacement.GetCopyCount())) - } else { - count, err = ms.vg.GrowByCountAndType(count, option, ms.Topo) - } - } else { - err = errors.New("parameter count is not found") - } - } - if err != nil { - writeJsonError(w, r, http.StatusNotAcceptable, err) - } else { - writeJsonQuiet(w, r, http.StatusOK, map[string]interface{}{"count": count}) - } -} - -func (ms *MasterServer) volumeStatusHandler(w http.ResponseWriter, r *http.Request) { - m := make(map[string]interface{}) - m["Version"] = util.VERSION - m["Volumes"] = ms.Topo.ToVolumeMap() - writeJsonQuiet(w, r, http.StatusOK, m) -} - -func (ms *MasterServer) redirectHandler(w http.ResponseWriter, r *http.Request) { - vid, _, _, _, _ := parseURLPath(r.URL.Path) - volumeId, err := storage.NewVolumeId(vid) - if err != nil { - debug("parsing error:", err, r.URL.Path) - return - } - collection := r.FormValue("collection") - machines := ms.Topo.Lookup(collection, volumeId) - if machines != nil && len(machines) > 0 { - var url string - if r.URL.RawQuery != "" { - url = util.NormalizeUrl(machines[rand.Intn(len(machines))].PublicUrl) + r.URL.Path + "?" + r.URL.RawQuery - } else { - url = util.NormalizeUrl(machines[rand.Intn(len(machines))].PublicUrl) + r.URL.Path - } - http.Redirect(w, r, url, http.StatusMovedPermanently) - } else { - writeJsonError(w, r, http.StatusNotFound, fmt.Errorf("volume id %d or collection %s not found", volumeId, collection)) - } -} - -func (ms *MasterServer) selfUrl(r *http.Request) string { - if r.Host != "" { - return r.Host - } - return "localhost:" + strconv.Itoa(ms.port) -} -func (ms *MasterServer) submitFromMasterServerHandler(w http.ResponseWriter, r *http.Request) { - if ms.Topo.IsLeader() { - submitForClientHandler(w, r, ms.selfUrl(r)) - } else { - masterUrl, err := ms.Topo.Leader() - if err != nil { - writeJsonError(w, r, http.StatusInternalServerError, err) - } else { - submitForClientHandler(w, r, masterUrl) - } - } -} - -func (ms *MasterServer) deleteFromMasterServerHandler(w http.ResponseWriter, r *http.Request) { - if ms.Topo.IsLeader() { - deleteForClientHandler(w, r, ms.selfUrl(r)) - } else { - deleteForClientHandler(w, r, ms.Topo.RaftServer.Leader()) - } -} - -func (ms *MasterServer) HasWritableVolume(option *topology.VolumeGrowOption) bool { - vl := ms.Topo.GetVolumeLayout(option.Collection, option.ReplicaPlacement, option.Ttl) - return vl.GetActiveVolumeCount(option) > 0 -} - -func (ms *MasterServer) getVolumeGrowOption(r *http.Request) (*topology.VolumeGrowOption, error) { - replicationString := r.FormValue("replication") - if replicationString == "" { - replicationString = ms.defaultReplicaPlacement - } - replicaPlacement, err := storage.NewReplicaPlacementFromString(replicationString) - if err != nil { - return nil, err - } - ttl, err := storage.ReadTTL(r.FormValue("ttl")) - if err != nil { - return nil, err - } - volumeGrowOption := &topology.VolumeGrowOption{ - Collection: r.FormValue("collection"), - ReplicaPlacement: replicaPlacement, - Ttl: ttl, - DataCenter: r.FormValue("dataCenter"), - Rack: r.FormValue("rack"), - DataNode: r.FormValue("dataNode"), - } - return volumeGrowOption, nil -} diff --git a/go/weed/weed_server/master_server_handlers_ui.go b/go/weed/weed_server/master_server_handlers_ui.go deleted file mode 100644 index af7261ab3..000000000 --- a/go/weed/weed_server/master_server_handlers_ui.go +++ /dev/null @@ -1,30 +0,0 @@ -package weed_server - -import ( - "net/http" - - "github.com/chrislusf/seaweedfs/go/stats" - "github.com/chrislusf/seaweedfs/go/util" - ui "github.com/chrislusf/seaweedfs/go/weed/weed_server/master_ui" -) - -func (ms *MasterServer) uiStatusHandler(w http.ResponseWriter, r *http.Request) { - infos := make(map[string]interface{}) - infos["Version"] = util.VERSION - args := struct { - Version string - Topology interface{} - Leader string - Peers interface{} - Stats map[string]interface{} - Counters *stats.ServerStats - }{ - util.VERSION, - ms.Topo.ToMap(), - ms.Topo.RaftServer.Leader(), - ms.Topo.RaftServer.Peers(), - infos, - serverStats, - } - ui.StatusTpl.Execute(w, args) -} diff --git a/go/weed/weed_server/master_ui/templates.go b/go/weed/weed_server/master_ui/templates.go deleted file mode 100644 index e9ee2d8d2..000000000 --- a/go/weed/weed_server/master_ui/templates.go +++ /dev/null @@ -1,102 +0,0 @@ -package master_ui - -import ( - "html/template" -) - -var StatusTpl = template.Must(template.New("status").Parse(`<!DOCTYPE html> -<html> - <head> - <title>SeaweedFS {{ .Version }}</title> - <link rel="icon" href="http://7viirv.com1.z0.glb.clouddn.com/seaweed_favicon.png" sizes="32x32" /> - <link rel="stylesheet" href="https://maxcdn.bootstrapcdn.com/bootstrap/3.3.1/css/bootstrap.min.css"> - </head> - <body> - <div class="container"> - <div class="page-header"> - <h1> - <img src="http://7viirv.com1.z0.glb.clouddn.com/seaweed50x50.png"></img> - SeaweedFS <small>{{ .Version }}</small> - </h1> - </div> - - <div class="row"> - <div class="col-sm-6"> - <h2>Cluster status</h2> - <table class="table"> - <tbody> - <tr> - <th>Free</th> - <td>{{ .Topology.Free }}</td> - </tr> - <tr> - <th>Max</th> - <td>{{ .Topology.Max }}</td> - </tr> - <tr> - <th>Leader</th> - <td><a href="http://{{ .Leader }}">{{ .Leader }}</a></td> - </tr> - <tr> - <td class="col-sm-2 field-label"><label>Peers:</label></td> - <td class="col-sm-10"><ul class="list-unstyled"> - {{ range $k, $p := .Peers }} - <li><a href="{{ $p.ConnectionString }}">{{ $p.Name }}</a></li> - {{ end }} - </ul></td> - </tr> - </tbody> - </table> - </div> - - <div class="col-sm-6"> - <h2>System Stats</h2> - <table class="table table-condensed table-striped"> - <tr> - <th>Concurrent Connections</th> - <td>{{ .Counters.Connections.WeekCounter.Sum }}</td> - </tr> - {{ range $key, $val := .Stats }} - <tr> - <th>{{ $key }}</th> - <td>{{ $val }}</td> - </tr> - {{ end }} - </table> - </div> - </div> - - <div class="row"> - <h2>Topology</h2> - <table class="table table-striped"> - <thead> - <tr> - <th>Data Center</th> - <th>Rack</th> - <th>RemoteAddr</th> - <th>#Volumes</th> - <th>Max</th> - </tr> - </thead> - <tbody> - {{ range $dc_index, $dc := .Topology.DataCenters }} - {{ range $rack_index, $rack := $dc.Racks }} - {{ range $dn_index, $dn := $rack.DataNodes }} - <tr> - <td><code>{{ $dc.Id }}</code></td> - <td>{{ $rack.Id }}</td> - <td><a href="http://{{ $dn.Url }}/ui/index.html">{{ $dn.Url }}</a></td> - <td>{{ $dn.Volumes }}</td> - <td>{{ $dn.Max }}</td> - </tr> - {{ end }} - {{ end }} - {{ end }} - </tbody> - </table> - </div> - - </div> - </body> -</html> -`)) diff --git a/go/weed/weed_server/raft_server.go b/go/weed/weed_server/raft_server.go deleted file mode 100644 index bc0414679..000000000 --- a/go/weed/weed_server/raft_server.go +++ /dev/null @@ -1,217 +0,0 @@ -package weed_server - -import ( - "bytes" - "encoding/json" - "errors" - "fmt" - "io/ioutil" - "math/rand" - "net/http" - "net/url" - "os" - "path" - "reflect" - "sort" - "strings" - "time" - - "github.com/chrislusf/raft" - "github.com/chrislusf/seaweedfs/go/glog" - "github.com/chrislusf/seaweedfs/go/topology" - "github.com/gorilla/mux" -) - -type RaftServer struct { - peers []string // initial peers to join with - raftServer raft.Server - dataDir string - httpAddr string - router *mux.Router - topo *topology.Topology -} - -func NewRaftServer(r *mux.Router, peers []string, httpAddr string, dataDir string, topo *topology.Topology, pulseSeconds int) *RaftServer { - s := &RaftServer{ - peers: peers, - httpAddr: httpAddr, - dataDir: dataDir, - router: r, - topo: topo, - } - - if glog.V(4) { - raft.SetLogLevel(2) - } - - raft.RegisterCommand(&topology.MaxVolumeIdCommand{}) - - var err error - transporter := raft.NewHTTPTransporter("/cluster", 0) - transporter.Transport.MaxIdleConnsPerHost = 1024 - glog.V(1).Infof("Starting RaftServer with IP:%v:", httpAddr) - - // Clear old cluster configurations if peers are changed - if oldPeers, changed := isPeersChanged(s.dataDir, httpAddr, s.peers); changed { - glog.V(0).Infof("Peers Change: %v => %v", oldPeers, s.peers) - os.RemoveAll(path.Join(s.dataDir, "conf")) - os.RemoveAll(path.Join(s.dataDir, "log")) - os.RemoveAll(path.Join(s.dataDir, "snapshot")) - } - - s.raftServer, err = raft.NewServer(s.httpAddr, s.dataDir, transporter, nil, topo, "") - if err != nil { - glog.V(0).Infoln(err) - return nil - } - transporter.Install(s.raftServer, s) - s.raftServer.SetHeartbeatInterval(1 * time.Second) - s.raftServer.SetElectionTimeout(time.Duration(pulseSeconds) * 3450 * time.Millisecond) - s.raftServer.Start() - - s.router.HandleFunc("/cluster/join", s.joinHandler).Methods("POST") - s.router.HandleFunc("/cluster/status", s.statusHandler).Methods("GET") - - if len(s.peers) > 0 { - // Join to leader if specified. - for { - glog.V(0).Infoln("Joining cluster:", strings.Join(s.peers, ",")) - time.Sleep(time.Duration(rand.Intn(1000)) * time.Millisecond) - firstJoinError := s.Join(s.peers) - if firstJoinError != nil { - glog.V(0).Infoln("No existing server found. Starting as leader in the new cluster.") - _, err := s.raftServer.Do(&raft.DefaultJoinCommand{ - Name: s.raftServer.Name(), - ConnectionString: "http://" + s.httpAddr, - }) - if err != nil { - glog.V(0).Infoln(err) - } else { - break - } - } else { - break - } - } - } else if s.raftServer.IsLogEmpty() { - // Initialize the server by joining itself. - glog.V(0).Infoln("Initializing new cluster") - - _, err := s.raftServer.Do(&raft.DefaultJoinCommand{ - Name: s.raftServer.Name(), - ConnectionString: "http://" + s.httpAddr, - }) - - if err != nil { - glog.V(0).Infoln(err) - return nil - } - - } else { - glog.V(0).Infoln("Old conf,log,snapshot should have been removed.") - } - - return s -} - -func (s *RaftServer) Peers() (members []string) { - peers := s.raftServer.Peers() - - for _, p := range peers { - members = append(members, strings.TrimPrefix(p.ConnectionString, "http://")) - } - - return -} - -func isPeersChanged(dir string, self string, peers []string) (oldPeers []string, changed bool) { - confPath := path.Join(dir, "conf") - // open conf file - b, err := ioutil.ReadFile(confPath) - if err != nil { - return oldPeers, true - } - conf := &raft.Config{} - if err = json.Unmarshal(b, conf); err != nil { - return oldPeers, true - } - - for _, p := range conf.Peers { - oldPeers = append(oldPeers, strings.TrimPrefix(p.ConnectionString, "http://")) - } - oldPeers = append(oldPeers, self) - - sort.Strings(peers) - sort.Strings(oldPeers) - - return oldPeers, reflect.DeepEqual(peers, oldPeers) - -} - -// Join joins an existing cluster. -func (s *RaftServer) Join(peers []string) error { - command := &raft.DefaultJoinCommand{ - Name: s.raftServer.Name(), - ConnectionString: "http://" + s.httpAddr, - } - - var err error - var b bytes.Buffer - json.NewEncoder(&b).Encode(command) - for _, m := range peers { - if m == s.httpAddr { - continue - } - target := fmt.Sprintf("http://%s/cluster/join", strings.TrimSpace(m)) - glog.V(0).Infoln("Attempting to connect to:", target) - - err = postFollowingOneRedirect(target, "application/json", &b) - - if err != nil { - glog.V(0).Infoln("Post returned error: ", err.Error()) - if _, ok := err.(*url.Error); ok { - // If we receive a network error try the next member - continue - } - } else { - return nil - } - } - - return errors.New("Could not connect to any cluster peers") -} - -// a workaround because http POST following redirection misses request body -func postFollowingOneRedirect(target string, contentType string, b *bytes.Buffer) error { - backupReader := bytes.NewReader(b.Bytes()) - resp, err := http.Post(target, contentType, b) - if err != nil { - return err - } - defer resp.Body.Close() - reply, _ := ioutil.ReadAll(resp.Body) - statusCode := resp.StatusCode - - if statusCode == http.StatusMovedPermanently { - var urlStr string - if urlStr = resp.Header.Get("Location"); urlStr == "" { - return fmt.Errorf("%d response missing Location header", resp.StatusCode) - } - - glog.V(0).Infoln("Post redirected to ", urlStr) - resp2, err2 := http.Post(urlStr, contentType, backupReader) - if err2 != nil { - return err2 - } - defer resp2.Body.Close() - reply, _ = ioutil.ReadAll(resp2.Body) - statusCode = resp2.StatusCode - } - - glog.V(0).Infoln("Post returned status: ", statusCode, string(reply)) - if statusCode != http.StatusOK { - return errors.New(string(reply)) - } - - return nil -} diff --git a/go/weed/weed_server/raft_server_handlers.go b/go/weed/weed_server/raft_server_handlers.go deleted file mode 100644 index b1d964a32..000000000 --- a/go/weed/weed_server/raft_server_handlers.go +++ /dev/null @@ -1,64 +0,0 @@ -package weed_server - -import ( - "encoding/json" - "io/ioutil" - "net/http" - "strings" - - "github.com/chrislusf/raft" - "github.com/chrislusf/seaweedfs/go/glog" - "github.com/chrislusf/seaweedfs/go/operation" -) - -// Handles incoming RAFT joins. -func (s *RaftServer) joinHandler(w http.ResponseWriter, req *http.Request) { - glog.V(0).Infoln("Processing incoming join. Current Leader", s.raftServer.Leader(), "Self", s.raftServer.Name(), "Peers", s.raftServer.Peers()) - command := &raft.DefaultJoinCommand{} - - commandText, _ := ioutil.ReadAll(req.Body) - glog.V(0).Info("Command:", string(commandText)) - if err := json.NewDecoder(strings.NewReader(string(commandText))).Decode(&command); err != nil { - glog.V(0).Infoln("Error decoding json message:", err, string(commandText)) - http.Error(w, err.Error(), http.StatusInternalServerError) - return - } - - glog.V(0).Infoln("join command from Name", command.Name, "Connection", command.ConnectionString) - - if _, err := s.raftServer.Do(command); err != nil { - switch err { - case raft.NotLeaderError: - s.redirectToLeader(w, req) - default: - glog.V(0).Infoln("Error processing join:", err) - http.Error(w, err.Error(), http.StatusInternalServerError) - } - } -} - -func (s *RaftServer) HandleFunc(pattern string, handler func(http.ResponseWriter, *http.Request)) { - s.router.HandleFunc(pattern, handler) -} - -func (s *RaftServer) redirectToLeader(w http.ResponseWriter, req *http.Request) { - if leader, e := s.topo.Leader(); e == nil { - //http.StatusMovedPermanently does not cause http POST following redirection - glog.V(0).Infoln("Redirecting to", http.StatusMovedPermanently, "http://"+leader+req.URL.Path) - http.Redirect(w, req, "http://"+leader+req.URL.Path, http.StatusMovedPermanently) - } else { - glog.V(0).Infoln("Error: Leader Unknown") - http.Error(w, "Leader unknown", http.StatusInternalServerError) - } -} - -func (s *RaftServer) statusHandler(w http.ResponseWriter, r *http.Request) { - ret := operation.ClusterStatusResult{ - IsLeader: s.topo.IsLeader(), - Peers: s.Peers(), - } - if leader, e := s.topo.Leader(); e == nil { - ret.Leader = leader - } - writeJsonQuiet(w, r, http.StatusOK, ret) -} diff --git a/go/weed/weed_server/volume_server.go b/go/weed/weed_server/volume_server.go deleted file mode 100644 index 229d66cb9..000000000 --- a/go/weed/weed_server/volume_server.go +++ /dev/null @@ -1,125 +0,0 @@ -package weed_server - -import ( - "math/rand" - "net/http" - "sync" - "time" - - "github.com/chrislusf/seaweedfs/go/glog" - "github.com/chrislusf/seaweedfs/go/security" - "github.com/chrislusf/seaweedfs/go/storage" -) - -type VolumeServer struct { - masterNode string - mnLock sync.RWMutex - pulseSeconds int - dataCenter string - rack string - store *storage.Store - guard *security.Guard - - needleMapKind storage.NeedleMapType - FixJpgOrientation bool - ReadRedirect bool -} - -func NewVolumeServer(adminMux, publicMux *http.ServeMux, ip string, - port int, publicUrl string, - folders []string, maxCounts []int, - needleMapKind storage.NeedleMapType, - masterNode string, pulseSeconds int, - dataCenter string, rack string, - whiteList []string, - fixJpgOrientation bool, - readRedirect bool) *VolumeServer { - vs := &VolumeServer{ - pulseSeconds: pulseSeconds, - dataCenter: dataCenter, - rack: rack, - needleMapKind: needleMapKind, - FixJpgOrientation: fixJpgOrientation, - ReadRedirect: readRedirect, - } - vs.SetMasterNode(masterNode) - vs.store = storage.NewStore(port, ip, publicUrl, folders, maxCounts, vs.needleMapKind) - - vs.guard = security.NewGuard(whiteList, "") - - adminMux.HandleFunc("/ui/index.html", vs.uiStatusHandler) - adminMux.HandleFunc("/status", vs.guard.WhiteList(vs.statusHandler)) - adminMux.HandleFunc("/admin/assign_volume", vs.guard.WhiteList(vs.assignVolumeHandler)) - adminMux.HandleFunc("/admin/vacuum/check", vs.guard.WhiteList(vs.vacuumVolumeCheckHandler)) - adminMux.HandleFunc("/admin/vacuum/compact", vs.guard.WhiteList(vs.vacuumVolumeCompactHandler)) - adminMux.HandleFunc("/admin/vacuum/commit", vs.guard.WhiteList(vs.vacuumVolumeCommitHandler)) - adminMux.HandleFunc("/admin/delete_collection", vs.guard.WhiteList(vs.deleteCollectionHandler)) - adminMux.HandleFunc("/admin/sync/status", vs.guard.WhiteList(vs.getVolumeSyncStatusHandler)) - adminMux.HandleFunc("/admin/sync/index", vs.guard.WhiteList(vs.getVolumeIndexContentHandler)) - adminMux.HandleFunc("/admin/sync/data", vs.guard.WhiteList(vs.getVolumeDataContentHandler)) - adminMux.HandleFunc("/stats/counter", vs.guard.WhiteList(statsCounterHandler)) - adminMux.HandleFunc("/stats/memory", vs.guard.WhiteList(statsMemoryHandler)) - adminMux.HandleFunc("/stats/disk", vs.guard.WhiteList(vs.statsDiskHandler)) - adminMux.HandleFunc("/delete", vs.guard.WhiteList(vs.batchDeleteHandler)) - adminMux.HandleFunc("/", vs.privateStoreHandler) - if publicMux != adminMux { - // separated admin and public port - publicMux.HandleFunc("/favicon.ico", vs.faviconHandler) - publicMux.HandleFunc("/", vs.publicReadOnlyHandler) - } - - go func() { - connected := true - - glog.V(0).Infof("Volume server bootstraps with master %s", vs.GetMasterNode()) - vs.store.SetBootstrapMaster(vs.GetMasterNode()) - vs.store.SetDataCenter(vs.dataCenter) - vs.store.SetRack(vs.rack) - for { - glog.V(4).Infof("Volume server sending to master %s", vs.GetMasterNode()) - master, secretKey, err := vs.store.SendHeartbeatToMaster() - if err == nil { - if !connected { - connected = true - vs.SetMasterNode(master) - vs.guard.SecretKey = secretKey - glog.V(0).Infoln("Volume Server Connected with master at", master) - } - } else { - glog.V(1).Infof("Volume Server Failed to talk with master %s: %v", vs.masterNode, err) - if connected { - connected = false - } - } - if connected { - time.Sleep(time.Duration(float32(vs.pulseSeconds*1e3)*(1+rand.Float32())) * time.Millisecond) - } else { - time.Sleep(time.Duration(float32(vs.pulseSeconds*1e3)*0.25) * time.Millisecond) - } - } - }() - - return vs -} - -func (vs *VolumeServer) GetMasterNode() string { - vs.mnLock.RLock() - defer vs.mnLock.RUnlock() - return vs.masterNode -} - -func (vs *VolumeServer) SetMasterNode(masterNode string) { - vs.mnLock.Lock() - defer vs.mnLock.Unlock() - vs.masterNode = masterNode -} - -func (vs *VolumeServer) Shutdown() { - glog.V(0).Infoln("Shutting down volume server...") - vs.store.Close() - glog.V(0).Infoln("Shut down successfully!") -} - -func (vs *VolumeServer) jwt(fileId string) security.EncodedJwt { - return security.GenJwt(vs.guard.SecretKey, fileId) -} diff --git a/go/weed/weed_server/volume_server_handlers.go b/go/weed/weed_server/volume_server_handlers.go deleted file mode 100644 index accc280cd..000000000 --- a/go/weed/weed_server/volume_server_handlers.go +++ /dev/null @@ -1,57 +0,0 @@ -package weed_server - -import ( - "net/http" - - "github.com/chrislusf/seaweedfs/go/stats" -) - -/* - -If volume server is started with a separated public port, the public port will -be more "secure". - -Public port currently only supports reads. - -Later writes on public port can have one of the 3 -security settings: -1. not secured -2. secured by white list -3. secured by JWT(Json Web Token) - -*/ - -func (vs *VolumeServer) privateStoreHandler(w http.ResponseWriter, r *http.Request) { - switch r.Method { - case "GET": - stats.ReadRequest() - vs.GetOrHeadHandler(w, r) - case "HEAD": - stats.ReadRequest() - vs.GetOrHeadHandler(w, r) - case "DELETE": - stats.DeleteRequest() - vs.guard.WhiteList(vs.DeleteHandler)(w, r) - case "PUT": - stats.WriteRequest() - vs.guard.WhiteList(vs.PostHandler)(w, r) - case "POST": - stats.WriteRequest() - vs.guard.WhiteList(vs.PostHandler)(w, r) - } -} - -func (vs *VolumeServer) publicReadOnlyHandler(w http.ResponseWriter, r *http.Request) { - switch r.Method { - case "GET": - stats.ReadRequest() - vs.GetOrHeadHandler(w, r) - case "HEAD": - stats.ReadRequest() - vs.GetOrHeadHandler(w, r) - } -} - -func (vs *VolumeServer) faviconHandler(w http.ResponseWriter, r *http.Request) { - vs.FaviconHandler(w, r) -} diff --git a/go/weed/weed_server/volume_server_handlers_admin.go b/go/weed/weed_server/volume_server_handlers_admin.go deleted file mode 100644 index 80aeb3f1d..000000000 --- a/go/weed/weed_server/volume_server_handlers_admin.go +++ /dev/null @@ -1,50 +0,0 @@ -package weed_server - -import ( - "net/http" - "path/filepath" - - "github.com/chrislusf/seaweedfs/go/glog" - "github.com/chrislusf/seaweedfs/go/stats" - "github.com/chrislusf/seaweedfs/go/util" -) - -func (vs *VolumeServer) statusHandler(w http.ResponseWriter, r *http.Request) { - m := make(map[string]interface{}) - m["Version"] = util.VERSION - m["Volumes"] = vs.store.Status() - writeJsonQuiet(w, r, http.StatusOK, m) -} - -func (vs *VolumeServer) assignVolumeHandler(w http.ResponseWriter, r *http.Request) { - err := vs.store.AddVolume(r.FormValue("volume"), r.FormValue("collection"), vs.needleMapKind, r.FormValue("replication"), r.FormValue("ttl")) - if err == nil { - writeJsonQuiet(w, r, http.StatusAccepted, map[string]string{"error": ""}) - } else { - writeJsonError(w, r, http.StatusNotAcceptable, err) - } - glog.V(2).Infoln("assign volume =", r.FormValue("volume"), ", collection =", r.FormValue("collection"), ", replication =", r.FormValue("replication"), ", error =", err) -} - -func (vs *VolumeServer) deleteCollectionHandler(w http.ResponseWriter, r *http.Request) { - err := vs.store.DeleteCollection(r.FormValue("collection")) - if err == nil { - writeJsonQuiet(w, r, http.StatusOK, map[string]string{"error": ""}) - } else { - writeJsonError(w, r, http.StatusInternalServerError, err) - } - glog.V(2).Infoln("deleting collection =", r.FormValue("collection"), ", error =", err) -} - -func (vs *VolumeServer) statsDiskHandler(w http.ResponseWriter, r *http.Request) { - m := make(map[string]interface{}) - m["Version"] = util.VERSION - var ds []*stats.DiskStatus - for _, loc := range vs.store.Locations { - if dir, e := filepath.Abs(loc.Directory); e == nil { - ds = append(ds, stats.NewDiskStatus(dir)) - } - } - m["DiskStatuses"] = ds - writeJsonQuiet(w, r, http.StatusOK, m) -} diff --git a/go/weed/weed_server/volume_server_handlers_helper.go b/go/weed/weed_server/volume_server_handlers_helper.go deleted file mode 100644 index 2bab35e45..000000000 --- a/go/weed/weed_server/volume_server_handlers_helper.go +++ /dev/null @@ -1,115 +0,0 @@ -package weed_server - -import ( - "errors" - "fmt" - "mime/multipart" - "net/textproto" - "strconv" - "strings" -) - -// copied from src/pkg/net/http/fs.go - -// httpRange specifies the byte range to be sent to the client. -type httpRange struct { - start, length int64 -} - -func (r httpRange) contentRange(size int64) string { - return fmt.Sprintf("bytes %d-%d/%d", r.start, r.start+r.length-1, size) -} - -func (r httpRange) mimeHeader(contentType string, size int64) textproto.MIMEHeader { - return textproto.MIMEHeader{ - "Content-Range": {r.contentRange(size)}, - "Content-Type": {contentType}, - } -} - -// parseRange parses a Range header string as per RFC 2616. -func parseRange(s string, size int64) ([]httpRange, error) { - if s == "" { - return nil, nil // header not present - } - const b = "bytes=" - if !strings.HasPrefix(s, b) { - return nil, errors.New("invalid range") - } - var ranges []httpRange - for _, ra := range strings.Split(s[len(b):], ",") { - ra = strings.TrimSpace(ra) - if ra == "" { - continue - } - i := strings.Index(ra, "-") - if i < 0 { - return nil, errors.New("invalid range") - } - start, end := strings.TrimSpace(ra[:i]), strings.TrimSpace(ra[i+1:]) - var r httpRange - if start == "" { - // If no start is specified, end specifies the - // range start relative to the end of the file. - i, err := strconv.ParseInt(end, 10, 64) - if err != nil { - return nil, errors.New("invalid range") - } - if i > size { - i = size - } - r.start = size - i - r.length = size - r.start - } else { - i, err := strconv.ParseInt(start, 10, 64) - if err != nil || i > size || i < 0 { - return nil, errors.New("invalid range") - } - r.start = i - if end == "" { - // If no end is specified, range extends to end of the file. - r.length = size - r.start - } else { - i, err := strconv.ParseInt(end, 10, 64) - if err != nil || r.start > i { - return nil, errors.New("invalid range") - } - if i >= size { - i = size - 1 - } - r.length = i - r.start + 1 - } - } - ranges = append(ranges, r) - } - return ranges, nil -} - -// countingWriter counts how many bytes have been written to it. -type countingWriter int64 - -func (w *countingWriter) Write(p []byte) (n int, err error) { - *w += countingWriter(len(p)) - return len(p), nil -} - -// rangesMIMESize returns the number of bytes it takes to encode the -// provided ranges as a multipart response. -func rangesMIMESize(ranges []httpRange, contentType string, contentSize int64) (encSize int64) { - var w countingWriter - mw := multipart.NewWriter(&w) - for _, ra := range ranges { - mw.CreatePart(ra.mimeHeader(contentType, contentSize)) - encSize += ra.length - } - mw.Close() - encSize += int64(w) - return -} - -func sumRangesSize(ranges []httpRange) (size int64) { - for _, ra := range ranges { - size += ra.length - } - return -} diff --git a/go/weed/weed_server/volume_server_handlers_read.go b/go/weed/weed_server/volume_server_handlers_read.go deleted file mode 100644 index 3eb33a8c9..000000000 --- a/go/weed/weed_server/volume_server_handlers_read.go +++ /dev/null @@ -1,301 +0,0 @@ -package weed_server - -import ( - "bytes" - "io" - "mime" - "mime/multipart" - "net/http" - "path" - "strconv" - "strings" - "time" - - "net/url" - - "github.com/chrislusf/seaweedfs/go/glog" - "github.com/chrislusf/seaweedfs/go/images" - "github.com/chrislusf/seaweedfs/go/operation" - "github.com/chrislusf/seaweedfs/go/storage" - "github.com/chrislusf/seaweedfs/go/util" -) - -var fileNameEscaper = strings.NewReplacer("\\", "\\\\", "\"", "\\\"") - -func (vs *VolumeServer) GetOrHeadHandler(w http.ResponseWriter, r *http.Request) { - n := new(storage.Needle) - vid, fid, filename, ext, _ := parseURLPath(r.URL.Path) - volumeId, err := storage.NewVolumeId(vid) - if err != nil { - glog.V(2).Infoln("parsing error:", err, r.URL.Path) - w.WriteHeader(http.StatusBadRequest) - return - } - err = n.ParsePath(fid) - if err != nil { - glog.V(2).Infoln("parsing fid error:", err, r.URL.Path) - w.WriteHeader(http.StatusBadRequest) - return - } - - glog.V(4).Infoln("volume", volumeId, "reading", n) - if !vs.store.HasVolume(volumeId) { - if !vs.ReadRedirect { - glog.V(2).Infoln("volume is not local:", err, r.URL.Path) - w.WriteHeader(http.StatusNotFound) - return - } - lookupResult, err := operation.Lookup(vs.GetMasterNode(), volumeId.String()) - glog.V(2).Infoln("volume", volumeId, "found on", lookupResult, "error", err) - if err == nil && len(lookupResult.Locations) > 0 { - u, _ := url.Parse(util.NormalizeUrl(lookupResult.Locations[0].PublicUrl)) - u.Path = r.URL.Path - arg := url.Values{} - if c := r.FormValue("collection"); c != "" { - arg.Set("collection", c) - } - u.RawQuery = arg.Encode() - http.Redirect(w, r, u.String(), http.StatusMovedPermanently) - - } else { - glog.V(2).Infoln("lookup error:", err, r.URL.Path) - w.WriteHeader(http.StatusNotFound) - } - return - } - cookie := n.Cookie - count, e := vs.store.ReadVolumeNeedle(volumeId, n) - glog.V(4).Infoln("read bytes", count, "error", e) - if e != nil || count <= 0 { - glog.V(0).Infoln("read error:", e, r.URL.Path) - w.WriteHeader(http.StatusNotFound) - return - } - defer n.ReleaseMemory() - if n.Cookie != cookie { - glog.V(0).Infoln("request", r.URL.Path, "with unmaching cookie seen:", cookie, "expected:", n.Cookie, "from", r.RemoteAddr, "agent", r.UserAgent()) - w.WriteHeader(http.StatusNotFound) - return - } - if n.LastModified != 0 { - w.Header().Set("Last-Modified", time.Unix(int64(n.LastModified), 0).UTC().Format(http.TimeFormat)) - if r.Header.Get("If-Modified-Since") != "" { - if t, parseError := time.Parse(http.TimeFormat, r.Header.Get("If-Modified-Since")); parseError == nil { - if t.Unix() >= int64(n.LastModified) { - w.WriteHeader(http.StatusNotModified) - return - } - } - } - } - etag := n.Etag() - if inm := r.Header.Get("If-None-Match"); inm == etag { - w.WriteHeader(http.StatusNotModified) - return - } - w.Header().Set("Etag", etag) - - if vs.tryHandleChunkedFile(n, filename, w, r) { - return - } - - if n.NameSize > 0 && filename == "" { - filename = string(n.Name) - if ext == "" { - ext = path.Ext(filename) - } - } - mtype := "" - if n.MimeSize > 0 { - mt := string(n.Mime) - if !strings.HasPrefix(mt, "application/octet-stream") { - mtype = mt - } - } - - if ext != ".gz" { - if n.IsGzipped() { - if strings.Contains(r.Header.Get("Accept-Encoding"), "gzip") { - w.Header().Set("Content-Encoding", "gzip") - } else { - if n.Data, err = operation.UnGzipData(n.Data); err != nil { - glog.V(0).Infoln("ungzip error:", err, r.URL.Path) - } - } - } - } - if ext == ".png" || ext == ".jpg" || ext == ".gif" { - width, height := 0, 0 - if r.FormValue("width") != "" { - width, _ = strconv.Atoi(r.FormValue("width")) - } - if r.FormValue("height") != "" { - height, _ = strconv.Atoi(r.FormValue("height")) - } - n.Data, _, _ = images.Resized(ext, n.Data, width, height) - } - - if e := writeResponseContent(filename, mtype, bytes.NewReader(n.Data), w, r); e != nil { - glog.V(2).Infoln("response write error:", e) - } -} - -func (vs *VolumeServer) FaviconHandler(w http.ResponseWriter, r *http.Request) { - data, err := images.Asset("favicon/favicon.ico") - if err != nil { - glog.V(2).Infoln("favicon read error:", err) - return - } - - if e := writeResponseContent("favicon.ico", "image/x-icon", bytes.NewReader(data), w, r); e != nil { - glog.V(2).Infoln("response write error:", e) - } -} - -func (vs *VolumeServer) tryHandleChunkedFile(n *storage.Needle, fileName string, w http.ResponseWriter, r *http.Request) (processed bool) { - if !n.IsChunkedManifest() { - return false - } - - chunkManifest, e := operation.LoadChunkManifest(n.Data, n.IsGzipped()) - if e != nil { - glog.V(0).Infof("load chunked manifest (%s) error: %v", r.URL.Path, e) - return false - } - if fileName == "" && chunkManifest.Name != "" { - fileName = chunkManifest.Name - } - mType := "" - if chunkManifest.Mime != "" { - mt := chunkManifest.Mime - if !strings.HasPrefix(mt, "application/octet-stream") { - mType = mt - } - } - - w.Header().Set("X-File-Store", "chunked") - - chunkedFileReader := &operation.ChunkedFileReader{ - Manifest: chunkManifest, - Master: vs.GetMasterNode(), - } - defer chunkedFileReader.Close() - if e := writeResponseContent(fileName, mType, chunkedFileReader, w, r); e != nil { - glog.V(2).Infoln("response write error:", e) - } - return true -} - -func writeResponseContent(filename, mimeType string, rs io.ReadSeeker, w http.ResponseWriter, r *http.Request) error { - totalSize, e := rs.Seek(0, 2) - if mimeType == "" { - if ext := path.Ext(filename); ext != "" { - mimeType = mime.TypeByExtension(ext) - } - } - if mimeType != "" { - w.Header().Set("Content-Type", mimeType) - } - if filename != "" { - contentDisposition := "inline" - if r.FormValue("dl") != "" { - if dl, _ := strconv.ParseBool(r.FormValue("dl")); dl { - contentDisposition = "attachment" - } - } - w.Header().Set("Content-Disposition", contentDisposition+`; filename="`+fileNameEscaper.Replace(filename)+`"`) - } - w.Header().Set("Accept-Ranges", "bytes") - if r.Method == "HEAD" { - w.Header().Set("Content-Length", strconv.FormatInt(totalSize, 10)) - return nil - } - rangeReq := r.Header.Get("Range") - if rangeReq == "" { - w.Header().Set("Content-Length", strconv.FormatInt(totalSize, 10)) - if _, e = rs.Seek(0, 0); e != nil { - return e - } - _, e = io.Copy(w, rs) - return e - } - - //the rest is dealing with partial content request - //mostly copy from src/pkg/net/http/fs.go - ranges, err := parseRange(rangeReq, totalSize) - if err != nil { - http.Error(w, err.Error(), http.StatusRequestedRangeNotSatisfiable) - return nil - } - if sumRangesSize(ranges) > totalSize { - // The total number of bytes in all the ranges - // is larger than the size of the file by - // itself, so this is probably an attack, or a - // dumb client. Ignore the range request. - return nil - } - if len(ranges) == 0 { - return nil - } - if len(ranges) == 1 { - // RFC 2616, Section 14.16: - // "When an HTTP message includes the content of a single - // range (for example, a response to a request for a - // single range, or to a request for a set of ranges - // that overlap without any holes), this content is - // transmitted with a Content-Range header, and a - // Content-Length header showing the number of bytes - // actually transferred. - // ... - // A response to a request for a single range MUST NOT - // be sent using the multipart/byteranges media type." - ra := ranges[0] - w.Header().Set("Content-Length", strconv.FormatInt(ra.length, 10)) - w.Header().Set("Content-Range", ra.contentRange(totalSize)) - w.WriteHeader(http.StatusPartialContent) - if _, e = rs.Seek(ra.start, 0); e != nil { - return e - } - - _, e = io.CopyN(w, rs, ra.length) - return e - } - // process multiple ranges - for _, ra := range ranges { - if ra.start > totalSize { - http.Error(w, "Out of Range", http.StatusRequestedRangeNotSatisfiable) - return nil - } - } - sendSize := rangesMIMESize(ranges, mimeType, totalSize) - pr, pw := io.Pipe() - mw := multipart.NewWriter(pw) - w.Header().Set("Content-Type", "multipart/byteranges; boundary="+mw.Boundary()) - sendContent := pr - defer pr.Close() // cause writing goroutine to fail and exit if CopyN doesn't finish. - go func() { - for _, ra := range ranges { - part, e := mw.CreatePart(ra.mimeHeader(mimeType, totalSize)) - if e != nil { - pw.CloseWithError(e) - return - } - if _, e = rs.Seek(ra.start, 0); e != nil { - pw.CloseWithError(e) - return - } - if _, e = io.CopyN(part, rs, ra.length); e != nil { - pw.CloseWithError(e) - return - } - } - mw.Close() - pw.Close() - }() - if w.Header().Get("Content-Encoding") == "" { - w.Header().Set("Content-Length", strconv.FormatInt(sendSize, 10)) - } - w.WriteHeader(http.StatusPartialContent) - _, e = io.CopyN(w, sendContent, sendSize) - return e -} diff --git a/go/weed/weed_server/volume_server_handlers_sync.go b/go/weed/weed_server/volume_server_handlers_sync.go deleted file mode 100644 index c52c93bd2..000000000 --- a/go/weed/weed_server/volume_server_handlers_sync.go +++ /dev/null @@ -1,87 +0,0 @@ -package weed_server - -import ( - "fmt" - "net/http" - - "github.com/chrislusf/seaweedfs/go/glog" - "github.com/chrislusf/seaweedfs/go/storage" - "github.com/chrislusf/seaweedfs/go/util" -) - -func (vs *VolumeServer) getVolumeSyncStatusHandler(w http.ResponseWriter, r *http.Request) { - v, err := vs.getVolume("volume", r) - if v == nil { - writeJsonError(w, r, http.StatusBadRequest, err) - return - } - syncStat := v.GetVolumeSyncStatus() - if syncStat.Error != "" { - writeJsonError(w, r, http.StatusInternalServerError, fmt.Errorf("Get Volume %d status error: %s", v.Id, syncStat.Error)) - glog.V(2).Infoln("getVolumeSyncStatusHandler volume =", r.FormValue("volume"), ", error =", err) - } else { - writeJsonQuiet(w, r, http.StatusOK, syncStat) - } -} - -func (vs *VolumeServer) getVolumeIndexContentHandler(w http.ResponseWriter, r *http.Request) { - v, err := vs.getVolume("volume", r) - if v == nil { - writeJsonError(w, r, http.StatusBadRequest, err) - return - } - content, err := v.IndexFileContent() - if err != nil { - writeJsonError(w, r, http.StatusInternalServerError, err) - return - } - w.Write(content) -} - -func (vs *VolumeServer) getVolumeDataContentHandler(w http.ResponseWriter, r *http.Request) { - v, err := vs.getVolume("volume", r) - if v == nil { - writeJsonError(w, r, http.StatusBadRequest, fmt.Errorf("Not Found volume: %v", err)) - return - } - if int(v.SuperBlock.CompactRevision) != util.ParseInt(r.FormValue("revision"), 0) { - writeJsonError(w, r, http.StatusExpectationFailed, fmt.Errorf("Requested Volume Revision is %s, but current revision is %d", r.FormValue("revision"), v.SuperBlock.CompactRevision)) - return - } - offset := uint32(util.ParseUint64(r.FormValue("offset"), 0)) - size := uint32(util.ParseUint64(r.FormValue("size"), 0)) - content, block, err := storage.ReadNeedleBlob(v.DataFile(), int64(offset)*storage.NeedlePaddingSize, size) - defer storage.ReleaseBytes(block.Bytes) - if err != nil { - writeJsonError(w, r, http.StatusInternalServerError, err) - return - } - - id := util.ParseUint64(r.FormValue("id"), 0) - n := new(storage.Needle) - n.ParseNeedleHeader(content) - if id != n.Id { - writeJsonError(w, r, http.StatusNotFound, fmt.Errorf("Expected file entry id %d, but found %d", id, n.Id)) - return - } - - w.Write(content) -} - -func (vs *VolumeServer) getVolume(volumeParameterName string, r *http.Request) (*storage.Volume, error) { - volumeIdString := r.FormValue(volumeParameterName) - if volumeIdString == "" { - err := fmt.Errorf("Empty Volume Id: Need to pass in %s=the_volume_id.", volumeParameterName) - return nil, err - } - vid, err := storage.NewVolumeId(volumeIdString) - if err != nil { - err = fmt.Errorf("Volume Id %s is not a valid unsigned integer", volumeIdString) - return nil, err - } - v := vs.store.GetVolume(vid) - if v == nil { - return nil, fmt.Errorf("Not Found Volume Id %s: %d", volumeIdString, vid) - } - return v, nil -} diff --git a/go/weed/weed_server/volume_server_handlers_ui.go b/go/weed/weed_server/volume_server_handlers_ui.go deleted file mode 100644 index 5925b5a88..000000000 --- a/go/weed/weed_server/volume_server_handlers_ui.go +++ /dev/null @@ -1,38 +0,0 @@ -package weed_server - -import ( - "net/http" - "path/filepath" - "time" - - "github.com/chrislusf/seaweedfs/go/stats" - "github.com/chrislusf/seaweedfs/go/util" - ui "github.com/chrislusf/seaweedfs/go/weed/weed_server/volume_server_ui" -) - -func (vs *VolumeServer) uiStatusHandler(w http.ResponseWriter, r *http.Request) { - infos := make(map[string]interface{}) - infos["Up Time"] = time.Now().Sub(startTime).String() - var ds []*stats.DiskStatus - for _, loc := range vs.store.Locations { - if dir, e := filepath.Abs(loc.Directory); e == nil { - ds = append(ds, stats.NewDiskStatus(dir)) - } - } - args := struct { - Version string - Master string - Volumes interface{} - DiskStatuses interface{} - Stats interface{} - Counters *stats.ServerStats - }{ - util.VERSION, - vs.masterNode, - vs.store.Status(), - ds, - infos, - serverStats, - } - ui.StatusTpl.Execute(w, args) -} diff --git a/go/weed/weed_server/volume_server_handlers_vacuum.go b/go/weed/weed_server/volume_server_handlers_vacuum.go deleted file mode 100644 index e174835ca..000000000 --- a/go/weed/weed_server/volume_server_handlers_vacuum.go +++ /dev/null @@ -1,35 +0,0 @@ -package weed_server - -import ( - "net/http" - - "github.com/chrislusf/seaweedfs/go/glog" -) - -func (vs *VolumeServer) vacuumVolumeCheckHandler(w http.ResponseWriter, r *http.Request) { - err, ret := vs.store.CheckCompactVolume(r.FormValue("volume"), r.FormValue("garbageThreshold")) - if err == nil { - writeJsonQuiet(w, r, http.StatusOK, map[string]interface{}{"error": "", "result": ret}) - } else { - writeJsonQuiet(w, r, http.StatusInternalServerError, map[string]interface{}{"error": err.Error(), "result": false}) - } - glog.V(2).Infoln("checked compacting volume =", r.FormValue("volume"), "garbageThreshold =", r.FormValue("garbageThreshold"), "vacuum =", ret) -} -func (vs *VolumeServer) vacuumVolumeCompactHandler(w http.ResponseWriter, r *http.Request) { - err := vs.store.CompactVolume(r.FormValue("volume")) - if err == nil { - writeJsonQuiet(w, r, http.StatusOK, map[string]string{"error": ""}) - } else { - writeJsonError(w, r, http.StatusInternalServerError, err) - } - glog.V(2).Infoln("compacted volume =", r.FormValue("volume"), ", error =", err) -} -func (vs *VolumeServer) vacuumVolumeCommitHandler(w http.ResponseWriter, r *http.Request) { - err := vs.store.CommitCompactVolume(r.FormValue("volume")) - if err == nil { - writeJsonQuiet(w, r, http.StatusOK, map[string]string{"error": ""}) - } else { - writeJsonError(w, r, http.StatusInternalServerError, err) - } - glog.V(2).Infoln("commit compact volume =", r.FormValue("volume"), ", error =", err) -} diff --git a/go/weed/weed_server/volume_server_handlers_write.go b/go/weed/weed_server/volume_server_handlers_write.go deleted file mode 100644 index b0a4c7031..000000000 --- a/go/weed/weed_server/volume_server_handlers_write.go +++ /dev/null @@ -1,165 +0,0 @@ -package weed_server - -import ( - "errors" - "fmt" - "net/http" - - "github.com/chrislusf/seaweedfs/go/glog" - "github.com/chrislusf/seaweedfs/go/operation" - "github.com/chrislusf/seaweedfs/go/storage" - "github.com/chrislusf/seaweedfs/go/topology" -) - -func (vs *VolumeServer) PostHandler(w http.ResponseWriter, r *http.Request) { - if e := r.ParseForm(); e != nil { - glog.V(0).Infoln("form parse error:", e) - writeJsonError(w, r, http.StatusBadRequest, e) - return - } - vid, _, _, _, _ := parseURLPath(r.URL.Path) - volumeId, ve := storage.NewVolumeId(vid) - if ve != nil { - glog.V(0).Infoln("NewVolumeId error:", ve) - writeJsonError(w, r, http.StatusBadRequest, ve) - return - } - needle, ne := storage.NewNeedle(r, vs.FixJpgOrientation) - if ne != nil { - writeJsonError(w, r, http.StatusBadRequest, ne) - return - } - - ret := operation.UploadResult{} - size, errorStatus := topology.ReplicatedWrite(vs.GetMasterNode(), - vs.store, volumeId, needle, r) - httpStatus := http.StatusCreated - if errorStatus != "" { - httpStatus = http.StatusInternalServerError - ret.Error = errorStatus - } - if needle.HasName() { - ret.Name = string(needle.Name) - } - ret.Size = size - writeJsonQuiet(w, r, httpStatus, ret) -} - -func (vs *VolumeServer) DeleteHandler(w http.ResponseWriter, r *http.Request) { - n := new(storage.Needle) - vid, fid, _, _, _ := parseURLPath(r.URL.Path) - volumeId, _ := storage.NewVolumeId(vid) - n.ParsePath(fid) - - glog.V(2).Infoln("deleting", n) - - cookie := n.Cookie - - _, ok := vs.store.ReadVolumeNeedle(volumeId, n) - if ok != nil { - m := make(map[string]uint32) - m["size"] = 0 - writeJsonQuiet(w, r, http.StatusNotFound, m) - return - } - defer n.ReleaseMemory() - - if n.Cookie != cookie { - glog.V(0).Infoln("delete", r.URL.Path, "with unmaching cookie from ", r.RemoteAddr, "agent", r.UserAgent()) - writeJsonError(w, r, http.StatusBadRequest, errors.New("File Random Cookie does not match.")) - return - } - - count := int64(n.Size) - - if n.IsChunkedManifest() { - chunkManifest, e := operation.LoadChunkManifest(n.Data, n.IsGzipped()) - if e != nil { - writeJsonError(w, r, http.StatusInternalServerError, fmt.Errorf("Load chunks manifest error: %v", e)) - return - } - // make sure all chunks had deleted before delete manifest - if e := chunkManifest.DeleteChunks(vs.GetMasterNode()); e != nil { - writeJsonError(w, r, http.StatusInternalServerError, fmt.Errorf("Delete chunks error: %v", e)) - return - } - count = chunkManifest.Size - } - - _, err := topology.ReplicatedDelete(vs.GetMasterNode(), vs.store, volumeId, n, r) - - if err == nil { - m := make(map[string]int64) - m["size"] = count - writeJsonQuiet(w, r, http.StatusAccepted, m) - } else { - writeJsonError(w, r, http.StatusInternalServerError, fmt.Errorf("Deletion Failed: %v", err)) - } - -} - -//Experts only: takes multiple fid parameters. This function does not propagate deletes to replicas. -func (vs *VolumeServer) batchDeleteHandler(w http.ResponseWriter, r *http.Request) { - r.ParseForm() - var ret []operation.DeleteResult - for _, fid := range r.Form["fid"] { - vid, id_cookie, err := operation.ParseFileId(fid) - if err != nil { - ret = append(ret, operation.DeleteResult{ - Fid: fid, - Status: http.StatusBadRequest, - Error: err.Error()}) - continue - } - n := new(storage.Needle) - volumeId, _ := storage.NewVolumeId(vid) - n.ParsePath(id_cookie) - glog.V(4).Infoln("batch deleting", n) - cookie := n.Cookie - if _, err := vs.store.ReadVolumeNeedle(volumeId, n); err != nil { - ret = append(ret, operation.DeleteResult{ - Fid: fid, - Status: http.StatusNotFound, - Error: err.Error(), - }) - continue - } - - if n.IsChunkedManifest() { - ret = append(ret, operation.DeleteResult{ - Fid: fid, - Status: http.StatusNotAcceptable, - Error: "ChunkManifest: not allowed in batch delete mode.", - }) - n.ReleaseMemory() - continue - } - - if n.Cookie != cookie { - ret = append(ret, operation.DeleteResult{ - Fid: fid, - Status: http.StatusBadRequest, - Error: "File Random Cookie does not match.", - }) - glog.V(0).Infoln("deleting", fid, "with unmaching cookie from ", r.RemoteAddr, "agent", r.UserAgent()) - n.ReleaseMemory() - return - } - if size, err := vs.store.Delete(volumeId, n); err != nil { - ret = append(ret, operation.DeleteResult{ - Fid: fid, - Status: http.StatusInternalServerError, - Error: err.Error()}, - ) - } else { - ret = append(ret, operation.DeleteResult{ - Fid: fid, - Status: http.StatusAccepted, - Size: int(size)}, - ) - } - n.ReleaseMemory() - } - - writeJsonQuiet(w, r, http.StatusAccepted, ret) -} diff --git a/go/weed/weed_server/volume_server_ui/templates.go b/go/weed/weed_server/volume_server_ui/templates.go deleted file mode 100644 index c3db6e92a..000000000 --- a/go/weed/weed_server/volume_server_ui/templates.go +++ /dev/null @@ -1,135 +0,0 @@ -package master_ui - -import ( - "html/template" - "strconv" - "strings" -) - -func join(data []int64) string { - var ret []string - for _, d := range data { - ret = append(ret, strconv.Itoa(int(d))) - } - return strings.Join(ret, ",") -} - -var funcMap = template.FuncMap{ - "join": join, -} - -var StatusTpl = template.Must(template.New("status").Funcs(funcMap).Parse(`<!DOCTYPE html> -<html> - <head> - <title>SeaweedFS {{ .Version }}</title> - <link rel="icon" href="http://7viirv.com1.z0.glb.clouddn.com/seaweed_favicon.png" sizes="32x32" /> - <link rel="stylesheet" href="https://maxcdn.bootstrapcdn.com/bootstrap/3.3.1/css/bootstrap.min.css"> - <script type="text/javascript" src="https://code.jquery.com/jquery-2.1.3.min.js"></script> - <script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/jquery-sparklines/2.1.2/jquery.sparkline.min.js"></script> - <script type="text/javascript"> - $(function() { - var periods = ['second', 'minute', 'hour', 'day']; - for (i = 0; i < periods.length; i++) { - var period = periods[i]; - $('.inlinesparkline-'+period).sparkline('html', { - type: 'line', - barColor: 'red', - tooltipSuffix:' request per '+period, - }); - } - }); - </script> - <style> - #jqstooltip{ - height: 28px !important; - width: 150px !important; - } - </style> - </head> - <body> - <div class="container"> - <div class="page-header"> - <h1> - <img src="http://7viirv.com1.z0.glb.clouddn.com/seaweed50x50.png"></img> - SeaweedFS <small>{{ .Version }}</small> - </h1> - </div> - - <div class="row"> - <div class="col-sm-6"> - <h2>Disk Stats</h2> - <table class="table table-condensed table-striped"> - {{ range .DiskStatuses }} - <tr> - <th>{{ .Dir }}</th> - <td>{{ .Free }} Bytes Free</td> - </tr> - {{ end }} - </table> - </div> - - <div class="col-sm-6"> - <h2>System Stats</h2> - <table class="table table-condensed table-striped"> - <tr> - <th>Master</th> - <td><a href="http://{{.Master}}/ui/index.html">{{.Master}}</a></td> - </tr> - <tr> - <th>Weekly # ReadRequests</th> - <td><span class="inlinesparkline-day">{{ .Counters.ReadRequests.WeekCounter.ToList | join }}</span></td> - </tr> - <tr> - <th>Daily # ReadRequests</th> - <td><span class="inlinesparkline-hour">{{ .Counters.ReadRequests.DayCounter.ToList | join }}</span></td> - </tr> - <tr> - <th>Hourly # ReadRequests</th> - <td><span class="inlinesparkline-minute">{{ .Counters.ReadRequests.HourCounter.ToList | join }}</span></td> - </tr> - <tr> - <th>Last Minute # ReadRequests</th> - <td><span class="inlinesparkline-second">{{ .Counters.ReadRequests.MinuteCounter.ToList | join }}</span></td> - </tr> - {{ range $key, $val := .Stats }} - <tr> - <th>{{ $key }}</th> - <td>{{ $val }}</td> - </tr> - {{ end }} - </table> - </div> - </div> - - <div class="row"> - <h2>Volumes</h2> - <table class="table table-striped"> - <thead> - <tr> - <th>Id</th> - <th>Collection</th> - <th>Size</th> - <th>Files</th> - <th>Trash</th> - <th>TTL</th> - </tr> - </thead> - <tbody> - {{ range .Volumes }} - <tr> - <td><code>{{ .Id }}</code></td> - <td>{{ .Collection }}</td> - <td>{{ .Size }} Bytes</td> - <td>{{ .FileCount }}</td> - <td>{{ .DeleteCount }} / {{.DeletedByteCount}} Bytes</td> - <td>{{ .Ttl }}</td> - </tr> - {{ end }} - </tbody> - </table> - </div> - - </div> - </body> -</html> -`)) |