migrate volume sync to gRpc

4 files changed, 65 insertions, 93 deletions

diff --git a/weed/server/volume_grpc_sync.go b/weed/server/volume_grpc_sync.go
index 7dffc3ce0..084069211 100644
--- a/weed/server/volume_grpc_sync.go
+++ b/weed/server/volume_grpc_sync.go
@@ -7,6 +7,7 @@ import (
 	"github.com/chrislusf/seaweedfs/weed/glog"
 	"github.com/chrislusf/seaweedfs/weed/pb/volume_server_pb"
 	"github.com/chrislusf/seaweedfs/weed/storage"
+	"github.com/chrislusf/seaweedfs/weed/storage/types"
 )
 
 func (vs *VolumeServer) VolumeSyncStatus(ctx context.Context, req *volume_server_pb.VolumeSyncStatusRequest) (*volume_server_pb.VolumeSyncStatusResponse, error) {
@@ -23,3 +24,64 @@ func (vs *VolumeServer) VolumeSyncStatus(ctx context.Context, req *volume_server
 	return resp, nil
 
 }
+
+func (vs *VolumeServer) VolumeSyncIndex(ctx context.Context, req *volume_server_pb.VolumeSyncIndexRequest) (*volume_server_pb.VolumeSyncIndexResponse, error) {
+
+	resp := &volume_server_pb.VolumeSyncIndexResponse{}
+
+	v := vs.store.GetVolume(storage.VolumeId(req.VolumdId))
+	if v == nil {
+		return nil, fmt.Errorf("Not Found Volume Id %d", req.VolumdId)
+	}
+
+	content, err := v.IndexFileContent()
+
+	if err != nil {
+		glog.Errorf("sync volume %d index: %v", req.VolumdId, err)
+	} else {
+		glog.V(2).Infof("sync volume %d index", req.VolumdId)
+	}
+
+	resp.IndexFileContent = content
+
+	return resp, nil
+
+}
+
+func (vs *VolumeServer) VolumeSyncData(ctx context.Context, req *volume_server_pb.VolumeSyncDataRequest) (*volume_server_pb.VolumeSyncDataResponse, error) {
+
+	resp := &volume_server_pb.VolumeSyncDataResponse{}
+
+	v := vs.store.GetVolume(storage.VolumeId(req.VolumdId))
+	if v == nil {
+		return nil, fmt.Errorf("Not Found Volume Id %d", req.VolumdId)
+	}
+
+	if uint32(v.SuperBlock.CompactRevision) != req.Revision {
+		return nil, fmt.Errorf("Requested Volume Revision is %d, but current revision is %d", req.Revision, v.SuperBlock.CompactRevision)
+	}
+
+	content, err := storage.ReadNeedleBlob(v.DataFile(), int64(req.Offset)*types.NeedlePaddingSize, req.Size, v.Version())
+	if err != nil {
+		return nil, fmt.Errorf("read offset:%d size:%d", req.Offset, req.Size)
+	}
+
+	id, err := types.ParseNeedleId(req.NeedleId)
+	if err != nil {
+		return nil, fmt.Errorf("parsing needle id %s: %v", req.NeedleId, err)
+	}
+	n := new(storage.Needle)
+	n.ParseNeedleHeader(content)
+	if id != n.Id {
+		return nil, fmt.Errorf("Expected file entry id %d, but found %d", id, n.Id)
+	}
+
+	if err != nil {
+		glog.Errorf("sync volume %d data: %v", req.VolumdId, err)
+	}
+
+	resp.FileContent = content
+
+	return resp, nil
+
+}
diff --git a/weed/server/volume_server.go b/weed/server/volume_server.go
index c0f3ef97e..0914e81b0 100644
--- a/weed/server/volume_server.go
+++ b/weed/server/volume_server.go
@@ -47,8 +47,6 @@ func NewVolumeServer(adminMux, publicMux *http.ServeMux, ip string,
 	handleStaticResources(adminMux)
 	adminMux.HandleFunc("/ui/index.html", vs.uiStatusHandler)
 	adminMux.HandleFunc("/status", vs.guard.WhiteList(vs.statusHandler))
-	adminMux.HandleFunc("/admin/sync/index", vs.guard.WhiteList(vs.getVolumeIndexContentHandler))
-	adminMux.HandleFunc("/admin/sync/data", vs.guard.WhiteList(vs.getVolumeDataContentHandler))
 	adminMux.HandleFunc("/stats/counter", vs.guard.WhiteList(statsCounterHandler))
 	adminMux.HandleFunc("/stats/memory", vs.guard.WhiteList(statsMemoryHandler))
 	adminMux.HandleFunc("/stats/disk", vs.guard.WhiteList(vs.statsDiskHandler))
diff --git a/weed/server/volume_server_handlers_admin.go b/weed/server/volume_server_handlers_admin.go
index de165bd3f..85311cdd7 100644
--- a/weed/server/volume_server_handlers_admin.go
+++ b/weed/server/volume_server_handlers_admin.go
@@ -1,12 +1,11 @@
 package weed_server
 
 import (
-	"fmt"
-	"github.com/chrislusf/seaweedfs/weed/stats"
-	"github.com/chrislusf/seaweedfs/weed/storage"
-	"github.com/chrislusf/seaweedfs/weed/util"
 	"net/http"
 	"path/filepath"
+
+	"github.com/chrislusf/seaweedfs/weed/stats"
+	"github.com/chrislusf/seaweedfs/weed/util"
 )
 
 func (vs *VolumeServer) statusHandler(w http.ResponseWriter, r *http.Request) {
@@ -28,16 +27,3 @@ func (vs *VolumeServer) statsDiskHandler(w http.ResponseWriter, r *http.Request)
 	m["DiskStatuses"] = ds
 	writeJsonQuiet(w, r, http.StatusOK, m)
 }
-
-// TODO delete this when volume sync is all moved to grpc
-func (vs *VolumeServer) getVolume(volumeParameterName string, r *http.Request) (*storage.Volume, error) {
-	vid, err := vs.getVolumeId(volumeParameterName, r)
-	if err != nil {
-		return nil, err
-	}
-	v := vs.store.GetVolume(vid)
-	if v == nil {
-		return nil, fmt.Errorf("Not Found Volume Id %d", vid)
-	}
-	return v, nil
-}
diff --git a/weed/server/volume_server_handlers_sync.go b/weed/server/volume_server_handlers_sync.go
deleted file mode 100644
index 9fe22d02b..000000000
--- a/weed/server/volume_server_handlers_sync.go
+++ /dev/null
@@ -1,74 +0,0 @@
-package weed_server
-
-import (
-	"fmt"
-	"net/http"
-
-	"github.com/chrislusf/seaweedfs/weed/storage"
-	"github.com/chrislusf/seaweedfs/weed/storage/types"
-	"github.com/chrislusf/seaweedfs/weed/util"
-)
-
-func (vs *VolumeServer) getVolumeIndexContentHandler(w http.ResponseWriter, r *http.Request) {
-	v, err := vs.getVolume("volume", r)
-	if v == nil {
-		writeJsonError(w, r, http.StatusBadRequest, err)
-		return
-	}
-	content, err := v.IndexFileContent()
-	if err != nil {
-		writeJsonError(w, r, http.StatusInternalServerError, err)
-		return
-	}
-	w.Write(content)
-}
-
-func (vs *VolumeServer) getVolumeDataContentHandler(w http.ResponseWriter, r *http.Request) {
-	v, err := vs.getVolume("volume", r)
-	if v == nil {
-		writeJsonError(w, r, http.StatusBadRequest, fmt.Errorf("Not Found volume: %v", err))
-		return
-	}
-	if int(v.SuperBlock.CompactRevision) != util.ParseInt(r.FormValue("revision"), 0) {
-		writeJsonError(w, r, http.StatusExpectationFailed, fmt.Errorf("Requested Volume Revision is %s, but current revision is %d", r.FormValue("revision"), v.SuperBlock.CompactRevision))
-		return
-	}
-	offset := uint32(util.ParseUint64(r.FormValue("offset"), 0))
-	size := uint32(util.ParseUint64(r.FormValue("size"), 0))
-	content, err := storage.ReadNeedleBlob(v.DataFile(), int64(offset)*types.NeedlePaddingSize, size, v.Version())
-	if err != nil {
-		writeJsonError(w, r, http.StatusInternalServerError, err)
-		return
-	}
-
-	id, err := types.ParseNeedleId(r.FormValue("id"))
-	if err != nil {
-		writeJsonError(w, r, http.StatusBadRequest, err)
-		return
-	}
-	n := new(storage.Needle)
-	n.ParseNeedleHeader(content)
-	if id != n.Id {
-		writeJsonError(w, r, http.StatusNotFound, fmt.Errorf("Expected file entry id %d, but found %d", id, n.Id))
-		return
-	}
-
-	w.Write(content)
-}
-
-func (vs *VolumeServer) getVolumeId(volumeParameterName string, r *http.Request) (storage.VolumeId, error) {
-	volumeIdString := r.FormValue(volumeParameterName)
-
-	if volumeIdString == "" {
-		err := fmt.Errorf("Empty Volume Id: Need to pass in %s=the_volume_id.", volumeParameterName)
-		return 0, err
-	}
-
-	vid, err := storage.NewVolumeId(volumeIdString)
-	if err != nil {
-		err = fmt.Errorf("Volume Id %s is not a valid unsigned integer", volumeIdString)
-		return 0, err
-	}
-
-	return vid, err
-}