diff options
Diffstat (limited to 'go/weed')
| -rw-r--r-- | go/weed/benchmark.go | 52 | ||||
| -rw-r--r-- | go/weed/upload.go | 8 | ||||
| -rw-r--r-- | go/weed/weed_server/common.go | 2 |
3 files changed, 42 insertions, 20 deletions
diff --git a/go/weed/benchmark.go b/go/weed/benchmark.go index f9a775c32..2fc83e578 100644 --- a/go/weed/benchmark.go +++ b/go/weed/benchmark.go @@ -8,6 +8,7 @@ import ( "fmt" "io" "math" + "math/rand" "os" "strings" "sync" @@ -15,14 +16,16 @@ import ( ) type BenchmarkOptions struct { - server *string - concurrency *int - numberOfFiles *int - fileSize *int - idListFile *string - write *bool - read *bool - vid2server map[string]string //cache for vid locations + server *string + concurrency *int + numberOfFiles *int + fileSize *int + idListFile *string + write *bool + read *bool + sequentialRead *bool + collection *string + vid2server map[string]string //cache for vid locations } var ( @@ -39,6 +42,8 @@ func init() { b.idListFile = cmdBenchmark.Flag.String("list", os.TempDir()+"/benchmark_list.txt", "list of uploaded file ids") b.write = cmdBenchmark.Flag.Bool("write", true, "enable write") b.read = cmdBenchmark.Flag.Bool("read", true, "enable read") + b.sequentialRead = cmdBenchmark.Flag.Bool("readSequentially", false, "randomly read by ids from \"-list\" specified file") + b.collection = cmdBenchmark.Flag.String("collection", "benchmark", "write data to this collection") } var cmdBenchmark = &Command{ @@ -120,9 +125,9 @@ func writeFiles(idChan chan int, fileIdLineChan chan string, s *stats) { if id, ok := <-idChan; ok { start := time.Now() fp := &operation.FilePart{Reader: &FakeReader{id: uint64(id), size: int64(*b.fileSize)}, FileSize: int64(*b.fileSize)} - if assignResult, err := operation.Assign(*b.server, 1, ""); err == nil { - fp.Server, fp.Fid = assignResult.PublicUrl, assignResult.Fid - fp.Upload(0, *b.server, "") + if assignResult, err := operation.Assign(*b.server, 1, "", *b.collection); err == nil { + fp.Server, fp.Fid, fp.Collection = assignResult.PublicUrl, assignResult.Fid, *b.collection + fp.Upload(0, *b.server) writeStats.addSample(time.Now().Sub(start)) fileIdLineChan <- fp.Fid s.transferred += int64(*b.fileSize) @@ -212,13 +217,28 @@ func readFileIds(fileName string, fileIdLineChan chan string) { defer file.Close() r := bufio.NewReader(file) - for { - if line, err := Readln(r); err == nil { - fileIdLineChan <- string(line) - } else { - break + if *b.sequentialRead { + for { + if line, err := Readln(r); err == nil { + fileIdLineChan <- string(line) + } else { + break + } + } + } else { + lines := make([]string, 0, *b.numberOfFiles) + for { + if line, err := Readln(r); err == nil { + lines = append(lines, string(line)) + } else { + break + } + } + for i := 0; i < *b.numberOfFiles; i++ { + fileIdLineChan <- lines[rand.Intn(len(lines))] } } + close(fileIdLineChan) } diff --git a/go/weed/upload.go b/go/weed/upload.go index 42e753981..b59313a2a 100644 --- a/go/weed/upload.go +++ b/go/weed/upload.go @@ -10,6 +10,7 @@ import ( var ( uploadReplication *string + uploadCollection *string uploadDir *string include *string maxMB *int @@ -21,7 +22,8 @@ func init() { server = cmdUpload.Flag.String("server", "localhost:9333", "weedfs master location") uploadDir = cmdUpload.Flag.String("dir", "", "Upload the whole folder recursively if specified.") include = cmdUpload.Flag.String("include", "", "pattens of files to upload, e.g., *.pdf, *.html, ab?d.txt, works together with -dir") - uploadReplication = cmdUpload.Flag.String("replication", "", "replication type(000,001,010,100,110,200)") + uploadReplication = cmdUpload.Flag.String("replication", "", "replication type") + uploadCollection = cmdUpload.Flag.String("collection", "", "optional collection name") maxMB = cmdUpload.Flag.Int("maxMB", 0, "split files larger than the limit") } @@ -65,7 +67,7 @@ func runUpload(cmd *Command, args []string) bool { if e != nil { return e } - results, e := operation.SubmitFiles(*server, parts, *uploadReplication, *maxMB) + results, e := operation.SubmitFiles(*server, parts, *uploadReplication, *uploadCollection, *maxMB) bytes, _ := json.Marshal(results) fmt.Println(string(bytes)) if e != nil { @@ -82,7 +84,7 @@ func runUpload(cmd *Command, args []string) bool { if e != nil { fmt.Println(e.Error()) } - results, _ := operation.SubmitFiles(*server, parts, *uploadReplication, *maxMB) + results, _ := operation.SubmitFiles(*server, parts, *uploadReplication, *uploadCollection, *maxMB) bytes, _ := json.Marshal(results) fmt.Println(string(bytes)) } diff --git a/go/weed/weed_server/common.go b/go/weed/weed_server/common.go index d0d52f488..d4492e719 100644 --- a/go/weed/weed_server/common.go +++ b/go/weed/weed_server/common.go @@ -94,7 +94,7 @@ func submitForClientHandler(w http.ResponseWriter, r *http.Request, masterUrl st } debug("assigning file id for", fname) - assignResult, ae := operation.Assign(masterUrl, 1, r.FormValue("replication")) + assignResult, ae := operation.Assign(masterUrl, 1, r.FormValue("replication"), r.FormValue("collection")) if ae != nil { writeJsonError(w, r, ae) return |