aboutsummaryrefslogtreecommitdiff
path: root/unmaintained/see_dat
diff options
context:
space:
mode:
authorChris Lu <chris.lu@gmail.com>2020-08-18 17:23:01 -0700
committerChris Lu <chris.lu@gmail.com>2020-08-18 17:23:01 -0700
commit332caf0cd7881ef4881099fee09ac9c8a63d8f0b (patch)
treecc0dc9ece8db282dbc2fdc049a1bd90f22011cb3 /unmaintained/see_dat
parent6a92f0bc7a2cbf0828c720422220b600263b5217 (diff)
downloadseaweedfs-332caf0cd7881ef4881099fee09ac9c8a63d8f0b.tar.xz
seaweedfs-332caf0cd7881ef4881099fee09ac9c8a63d8f0b.zip
maintain the unmaintained
Diffstat (limited to 'unmaintained/see_dat')
-rw-r--r--unmaintained/see_dat/see_dat_gzip.go83
1 files changed, 0 insertions, 83 deletions
diff --git a/unmaintained/see_dat/see_dat_gzip.go b/unmaintained/see_dat/see_dat_gzip.go
deleted file mode 100644
index cec073e3f..000000000
--- a/unmaintained/see_dat/see_dat_gzip.go
+++ /dev/null
@@ -1,83 +0,0 @@
-package main
-
-import (
- "bytes"
- "compress/gzip"
- "crypto/md5"
- "flag"
- "io"
- "io/ioutil"
- "net/http"
- "time"
- "github.com/chrislusf/seaweedfs/weed/glog"
- "github.com/chrislusf/seaweedfs/weed/storage"
- "github.com/chrislusf/seaweedfs/weed/storage/needle"
- "github.com/chrislusf/seaweedfs/weed/storage/super_block"
- "github.com/chrislusf/seaweedfs/weed/util"
-)
-
-type VolumeFileScanner4SeeDat struct {
- version needle.Version
-}
-
-func (scanner *VolumeFileScanner4SeeDat) VisitSuperBlock(superBlock super_block.SuperBlock) error {
- scanner.version = superBlock.Version
- return nil
-}
-
-func (scanner *VolumeFileScanner4SeeDat) ReadNeedleBody() bool {
- return true
-}
-
-var (
- files = int64(0)
- filebytes = int64(0)
- diffbytes = int64(0)
-)
-
-func Compresssion(data []byte) float64 {
- if len(data) <= 128 {
- return 100.0
- }
- compressed, _ := util.GzipData(data[0:128])
- return float64(len(compressed)*10) / 1280.0
-}
-
-func (scanner *VolumeFileScanner4SeeDat) VisitNeedle(n *needle.Needle, offset int64, needleHeader, needleBody []byte) error {
- t := time.Unix(int64(n.AppendAtNs)/int64(time.Second), int64(n.AppendAtNs)%int64(time.Second))
- glog.V(0).Info("----------------------------------------------------------------------------------")
- glog.V(0).Infof("%d,%s%x offset %d size %d(%s) cookie %x appendedAt %v hasmime[%t] mime[%s] (len: %d)",
- *volumeId, n.Id, n.Cookie, offset, n.Size, util.BytesToHumanReadable(uint64(n.Size)), n.Cookie, t, n.HasMime(), string(n.Mime), len(n.Mime))
- r, err := gzip.NewReader(bytes.NewReader(n.Data))
- if err == nil {
- buf := bytes.Buffer{}
- h := md5.New()
- c, _ := io.Copy(&buf, r)
- d := buf.Bytes()
- io.Copy(h, bytes.NewReader(d))
- diff := (int64(n.DataSize) - int64(c))
- diffbytes += diff
- glog.V(0).Infof("was gzip! stored_size: %d orig_size: %d diff: %d(%d) mime:%s compression-of-128: %.2f md5: %x", n.DataSize, c, diff, diffbytes, http.DetectContentType(d), Compresssion(d), h.Sum(nil))
- } else {
- glog.V(0).Infof("no gzip!")
- }
- return nil
-}
-
-var (
- _ = ioutil.ReadAll
- volumePath = flag.String("dir", "/tmp", "data directory to store files")
- volumeCollection = flag.String("collection", "", "the volume collection name")
- volumeId = flag.Int("volumeId", -1, "a volume id. The volume should already exist in the dir. The volume index file should not exist.")
-)
-
-func main() {
- flag.Parse()
- vid := needle.VolumeId(*volumeId)
- glog.V(0).Info("Starting")
- scanner := &VolumeFileScanner4SeeDat{}
- err := storage.ScanVolumeFile(*volumePath, *volumeCollection, vid, storage.NeedleMapInMemory, scanner)
- if err != nil {
- glog.Fatalf("Reading Volume File [ERROR] %s\n", err)
- }
-}