From 19b6a16003325ec93bb0e261d5a9c08cd3e03cad Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Tue, 29 Oct 2019 00:35:16 -0700 Subject: changed from os.file to backend.DataStorageBackend --- unmaintained/fix_dat/fix_dat.go | 23 +++++++++++++---------- 1 file changed, 13 insertions(+), 10 deletions(-) (limited to 'unmaintained/fix_dat/fix_dat.go') diff --git a/unmaintained/fix_dat/fix_dat.go b/unmaintained/fix_dat/fix_dat.go index a72a78eed..1ad560a7d 100644 --- a/unmaintained/fix_dat/fix_dat.go +++ b/unmaintained/fix_dat/fix_dat.go @@ -10,6 +10,7 @@ import ( "github.com/chrislusf/seaweedfs/weed/glog" "github.com/chrislusf/seaweedfs/weed/storage" + "github.com/chrislusf/seaweedfs/weed/storage/backend" "github.com/chrislusf/seaweedfs/weed/storage/needle" "github.com/chrislusf/seaweedfs/weed/storage/types" "github.com/chrislusf/seaweedfs/weed/util" @@ -44,11 +45,13 @@ func main() { glog.Fatalf("Read Volume Index %v", err) } defer indexFile.Close() - datFile, err := os.OpenFile(path.Join(*fixVolumePath, fileName+".dat"), os.O_RDONLY, 0644) + datFileName := path.Join(*fixVolumePath, fileName+".dat") + datFile, err := os.OpenFile(datFileName, os.O_RDONLY, 0644) if err != nil { glog.Fatalf("Read Volume Data %v", err) } - defer datFile.Close() + datBackend := backend.NewDiskFile(datFileName, datFile) + defer datBackend.Close() newDatFile, err := os.Create(path.Join(*fixVolumePath, fileName+".dat_fixed")) if err != nil { @@ -56,21 +59,21 @@ func main() { } defer newDatFile.Close() - superBlock, err := storage.ReadSuperBlock(datFile) + superBlock, err := storage.ReadSuperBlock(datBackend) if err != nil { glog.Fatalf("Read Volume Data superblock %v", err) } newDatFile.Write(superBlock.Bytes()) - iterateEntries(datFile, indexFile, func(n *needle.Needle, offset int64) { + iterateEntries(datBackend, indexFile, func(n *needle.Needle, offset int64) { fmt.Printf("needle id=%v name=%s size=%d dataSize=%d\n", n.Id, string(n.Name), n.Size, n.DataSize) - _, s, _, e := n.Append(newDatFile, superBlock.Version()) + _, s, _, e := n.Append(datBackend, superBlock.Version()) fmt.Printf("size %d error %v\n", s, e) }) } -func iterateEntries(datFile, idxFile *os.File, visitNeedle func(n *needle.Needle, offset int64)) { +func iterateEntries(datBackend backend.DataStorageBackend, idxFile *os.File, visitNeedle func(n *needle.Needle, offset int64)) { // start to read index file var readerOffset int64 bytes := make([]byte, 16) @@ -78,14 +81,14 @@ func iterateEntries(datFile, idxFile *os.File, visitNeedle func(n *needle.Needle readerOffset += int64(count) // start to read dat file - superBlock, err := storage.ReadSuperBlock(datFile) + superBlock, err := storage.ReadSuperBlock(datBackend) if err != nil { fmt.Printf("cannot read dat file super block: %v", err) return } offset := int64(superBlock.BlockSize()) version := superBlock.Version() - n, _, rest, err := needle.ReadNeedleHeader(datFile, version, offset) + n, _, rest, err := needle.ReadNeedleHeader(datBackend, version, offset) if err != nil { fmt.Printf("cannot read needle header: %v", err) return @@ -115,7 +118,7 @@ func iterateEntries(datFile, idxFile *os.File, visitNeedle func(n *needle.Needle fmt.Println("Recovered in f", r) } }() - if _, err = n.ReadNeedleBody(datFile, version, offset+int64(types.NeedleHeaderSize), rest); err != nil { + if _, err = n.ReadNeedleBody(datBackend, version, offset+int64(types.NeedleHeaderSize), rest); err != nil { fmt.Printf("cannot read needle body: offset %d body %d %v\n", offset, rest, err) } }() @@ -127,7 +130,7 @@ func iterateEntries(datFile, idxFile *os.File, visitNeedle func(n *needle.Needle offset += types.NeedleHeaderSize + rest //fmt.Printf("==> new entry offset %d\n", offset) - if n, _, rest, err = needle.ReadNeedleHeader(datFile, version, offset); err != nil { + if n, _, rest, err = needle.ReadNeedleHeader(datBackend, version, offset); err != nil { if err == io.EOF { return } -- cgit v1.2.3 From 4b5ba4927fa4d130d9badbd506195fa0cd7e9168 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Tue, 29 Oct 2019 22:37:36 -0700 Subject: adjust NewDiskFile() paramaters --- unmaintained/fix_dat/fix_dat.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'unmaintained/fix_dat/fix_dat.go') diff --git a/unmaintained/fix_dat/fix_dat.go b/unmaintained/fix_dat/fix_dat.go index 1ad560a7d..e88a48c96 100644 --- a/unmaintained/fix_dat/fix_dat.go +++ b/unmaintained/fix_dat/fix_dat.go @@ -50,7 +50,7 @@ func main() { if err != nil { glog.Fatalf("Read Volume Data %v", err) } - datBackend := backend.NewDiskFile(datFileName, datFile) + datBackend := backend.NewDiskFile(datFile) defer datBackend.Close() newDatFile, err := os.Create(path.Join(*fixVolumePath, fileName+".dat_fixed")) -- cgit v1.2.3 From f60154f330a81354c433da37d612f235d1c0e4e9 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Thu, 28 Nov 2019 18:33:18 -0800 Subject: master load backend storage config from master.toml --- unmaintained/fix_dat/fix_dat.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'unmaintained/fix_dat/fix_dat.go') diff --git a/unmaintained/fix_dat/fix_dat.go b/unmaintained/fix_dat/fix_dat.go index e88a48c96..bee30b2bd 100644 --- a/unmaintained/fix_dat/fix_dat.go +++ b/unmaintained/fix_dat/fix_dat.go @@ -73,7 +73,7 @@ func main() { } -func iterateEntries(datBackend backend.DataStorageBackend, idxFile *os.File, visitNeedle func(n *needle.Needle, offset int64)) { +func iterateEntries(datBackend backend.BackendStorageFile, idxFile *os.File, visitNeedle func(n *needle.Needle, offset int64)) { // start to read index file var readerOffset int64 bytes := make([]byte, 16) -- cgit v1.2.3 From 09ca936c78c1044412f47dc06caff1bf08273e60 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Mon, 23 Dec 2019 12:48:20 -0800 Subject: shell: add ec.decode command --- unmaintained/fix_dat/fix_dat.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'unmaintained/fix_dat/fix_dat.go') diff --git a/unmaintained/fix_dat/fix_dat.go b/unmaintained/fix_dat/fix_dat.go index bee30b2bd..51417a9d2 100644 --- a/unmaintained/fix_dat/fix_dat.go +++ b/unmaintained/fix_dat/fix_dat.go @@ -9,9 +9,9 @@ import ( "strconv" "github.com/chrislusf/seaweedfs/weed/glog" - "github.com/chrislusf/seaweedfs/weed/storage" "github.com/chrislusf/seaweedfs/weed/storage/backend" "github.com/chrislusf/seaweedfs/weed/storage/needle" + "github.com/chrislusf/seaweedfs/weed/storage/super_block" "github.com/chrislusf/seaweedfs/weed/storage/types" "github.com/chrislusf/seaweedfs/weed/util" ) @@ -59,7 +59,7 @@ func main() { } defer newDatFile.Close() - superBlock, err := storage.ReadSuperBlock(datBackend) + superBlock, err := super_block.ReadSuperBlock(datBackend) if err != nil { glog.Fatalf("Read Volume Data superblock %v", err) } @@ -81,7 +81,7 @@ func iterateEntries(datBackend backend.BackendStorageFile, idxFile *os.File, vis readerOffset += int64(count) // start to read dat file - superBlock, err := storage.ReadSuperBlock(datBackend) + superBlock, err := super_block.ReadSuperBlock(datBackend) if err != nil { fmt.Printf("cannot read dat file super block: %v", err) return -- cgit v1.2.3 From b7bc08cf52fa8471c76310b04200abb5ee4bf531 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Sat, 28 Dec 2019 11:50:42 -0800 Subject: fix compilation --- unmaintained/fix_dat/fix_dat.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'unmaintained/fix_dat/fix_dat.go') diff --git a/unmaintained/fix_dat/fix_dat.go b/unmaintained/fix_dat/fix_dat.go index 51417a9d2..d6110d870 100644 --- a/unmaintained/fix_dat/fix_dat.go +++ b/unmaintained/fix_dat/fix_dat.go @@ -67,7 +67,7 @@ func main() { iterateEntries(datBackend, indexFile, func(n *needle.Needle, offset int64) { fmt.Printf("needle id=%v name=%s size=%d dataSize=%d\n", n.Id, string(n.Name), n.Size, n.DataSize) - _, s, _, e := n.Append(datBackend, superBlock.Version()) + _, s, _, e := n.Append(datBackend, superBlock.Version) fmt.Printf("size %d error %v\n", s, e) }) @@ -87,7 +87,7 @@ func iterateEntries(datBackend backend.BackendStorageFile, idxFile *os.File, vis return } offset := int64(superBlock.BlockSize()) - version := superBlock.Version() + version := superBlock.Version n, _, rest, err := needle.ReadNeedleHeader(datBackend, version, offset) if err != nil { fmt.Printf("cannot read needle header: %v", err) -- cgit v1.2.3