support additional header name-value pairs

3 files changed, 66 insertions, 16 deletions

diff --git a/weed/storage/needle.go b/weed/storage/needle.go
index 1d306395e..ea013e290 100644
--- a/weed/storage/needle.go
+++ b/weed/storage/needle.go
@@ -11,6 +11,8 @@ import (
 	"strings"
 	"time"
 
+	"encoding/json"
+
 	"github.com/chrislusf/seaweedfs/weed/glog"
 	"github.com/chrislusf/seaweedfs/weed/images"
 	"github.com/chrislusf/seaweedfs/weed/operation"
@@ -22,6 +24,7 @@ const (
 	NeedleChecksumSize    = 4
 	MaxPossibleVolumeSize = 4 * 1024 * 1024 * 1024 * 8
 	TombstoneFileSize     = math.MaxUint32
+	PairNamePrefix        = "Seaweed-"
 )
 
 /*
@@ -40,6 +43,8 @@ type Needle struct {
 	Name         []byte `comment:"maximum 256 characters"` //version2
 	MimeSize     uint8  //version2
 	Mime         []byte `comment:"maximum 256 characters"` //version2
+	PairsSize    uint16 //version2
+	Pairs        []byte `comment:"additional name value pairs, json format, maximum 64kB"`
 	LastModified uint64 //only store LastModifiedBytesLength bytes, which is 5 bytes to disk
 	Ttl          *TTL
 
@@ -55,8 +60,17 @@ func (n *Needle) String() (str string) {
 }
 
 func ParseUpload(r *http.Request) (
-	fileName string, data []byte, mimeType string, isGzipped bool,
+	fileName string, data []byte, mimeType string, pairs []byte, isGzipped bool,
 	modifiedTime uint64, ttl *TTL, isChunkedFile bool, e error) {
+	pairMap := make(map[string]string)
+	for k, v := range r.Header {
+		if len(v) > 0 && strings.HasPrefix(k, PairNamePrefix) {
+			pairMap[k] = v[0]
+		}
+	}
+	if len(pairMap) != 0 {
+		pairs, _ = json.Marshal(pairMap)
+	}
 	form, fe := r.MultipartReader()
 	if fe != nil {
 		glog.V(0).Infoln("MultipartReader [ERROR]", fe)
@@ -109,19 +123,19 @@ func ParseUpload(r *http.Request) (
 	}
 
 	isChunkedFile, _ = strconv.ParseBool(r.FormValue("cm"))
-	isGzipped = false
+	dotIndex := strings.LastIndex(fileName, ".")
+	ext, mtype := "", ""
+	if dotIndex > 0 {
+		ext = strings.ToLower(fileName[dotIndex:])
+		mtype = mime.TypeByExtension(ext)
+	}
+	contentType := part.Header.Get("Content-Type")
+	if contentType != "" && mtype != contentType {
+		mimeType = contentType //only return mime type if not deductable
+		mtype = contentType
+	}
+
 	if !isChunkedFile {
-		dotIndex := strings.LastIndex(fileName, ".")
-		ext, mtype := "", ""
-		if dotIndex > 0 {
-			ext = strings.ToLower(fileName[dotIndex:])
-			mtype = mime.TypeByExtension(ext)
-		}
-		contentType := part.Header.Get("Content-Type")
-		if contentType != "" && mtype != contentType {
-			mimeType = contentType //only return mime type if not deductable
-			mtype = contentType
-		}
 		if part.Header.Get("Content-Encoding") == "gzip" {
 			isGzipped = true
 		} else if operation.IsGzippable(ext, mtype) {
@@ -144,9 +158,10 @@ func ParseUpload(r *http.Request) (
 	return
 }
 func NewNeedle(r *http.Request, fixJpgOrientation bool) (n *Needle, e error) {
+	var pair []byte
 	fname, mimeType, isGzipped, isChunkedFile := "", "", false, false
 	n = new(Needle)
-	fname, n.Data, mimeType, isGzipped, n.LastModified, n.Ttl, isChunkedFile, e = ParseUpload(r)
+	fname, n.Data, mimeType, pair, isGzipped, n.LastModified, n.Ttl, isChunkedFile, e = ParseUpload(r)
 	if e != nil {
 		return
 	}
@@ -158,6 +173,11 @@ func NewNeedle(r *http.Request, fixJpgOrientation bool) (n *Needle, e error) {
 		n.Mime = []byte(mimeType)
 		n.SetHasMime()
 	}
+	if len(pair) < 65536 {
+		n.Pairs = pair
+		n.PairsSize = uint16(len(pair))
+		n.SetHasPairs()
+	}
 	if isGzipped {
 		n.SetGzipped()
 	}
diff --git a/weed/storage/needle_read_write.go b/weed/storage/needle_read_write.go
index 8baa325df..ff43effb3 100644
--- a/weed/storage/needle_read_write.go
+++ b/weed/storage/needle_read_write.go
@@ -16,6 +16,7 @@ const (
 	FlagHasMime             = 0x04
 	FlagHasLastModifiedDate = 0x08
 	FlagHasTtl              = 0x10
+	FlagHasPairs            = 0x20
 	FlagIsChunkManifest     = 0x80
 	LastModifiedBytesLength = 5
 	TtlBytesLength          = 2
@@ -78,6 +79,9 @@ func (n *Needle) Append(w io.Writer, version Version) (size uint32, actualSize i
 			if n.HasTtl() {
 				n.Size = n.Size + TtlBytesLength
 			}
+			if n.HasPairs() {
+				n.Size += 2 + uint32(n.PairsSize)
+			}
 		} else {
 			n.Size = 0
 		}
@@ -128,6 +132,15 @@ func (n *Needle) Append(w io.Writer, version Version) (size uint32, actualSize i
 					return
 				}
 			}
+			if n.HasPairs() {
+				util.Uint16toBytes(header[0:2], n.PairsSize)
+				if _, err = w.Write(header[0:2]); err != nil {
+					return
+				}
+				if _, err = w.Write(n.Pairs); err != nil {
+					return
+				}
+			}
 		}
 		padding := NeedlePaddingSize - ((NeedleHeaderSize + n.Size + NeedleChecksumSize) % NeedlePaddingSize)
 		util.Uint32toBytes(header[0:NeedleChecksumSize], n.Checksum.Value())
@@ -141,8 +154,9 @@ func (n *Needle) Append(w io.Writer, version Version) (size uint32, actualSize i
 }
 
 func ReadNeedleBlob(r *os.File, offset int64, size uint32) (dataSlice []byte, block *Block, err error) {
-	padding := NeedlePaddingSize - ((NeedleHeaderSize + size + NeedleChecksumSize) % NeedlePaddingSize)
-	readSize := NeedleHeaderSize + size + NeedleChecksumSize + padding
+	NeedleWithoutPaddingSize := NeedleHeaderSize + size + NeedleChecksumSize
+	padding := NeedlePaddingSize - (NeedleWithoutPaddingSize % NeedlePaddingSize)
+	readSize := NeedleWithoutPaddingSize + padding
 	return getBytesForFileBlock(r, offset, int(readSize))
 }
 
@@ -213,6 +227,13 @@ func (n *Needle) readNeedleDataVersion2(bytes []byte) {
 		n.Ttl = LoadTTLFromBytes(bytes[index : index+TtlBytesLength])
 		index = index + TtlBytesLength
 	}
+	if index < lenBytes && n.HasPairs() {
+		n.PairsSize = util.BytesToUint16(bytes[index : index+2])
+		index += 2
+		end := index + int(n.PairsSize)
+		n.Pairs = bytes[index:end]
+		index = end
+	}
 }
 
 func ReadNeedleHeader(r *os.File, version Version, offset int64) (n *Needle, bodyLength uint32, err error) {
@@ -296,3 +317,11 @@ func (n *Needle) IsChunkedManifest() bool {
 func (n *Needle) SetIsChunkManifest() {
 	n.Flags = n.Flags | FlagIsChunkManifest
 }
+
+func (n *Needle) HasPairs() bool {
+	return n.Flags&FlagHasPairs != 0
+}
+
+func (n *Needle) SetHasPairs() {
+	n.Flags = n.Flags | FlagHasPairs
+}
diff --git a/weed/storage/store.go b/weed/storage/store.go
index 614c87ace..37a3904bd 100644
--- a/weed/storage/store.go
+++ b/weed/storage/store.go
@@ -303,6 +303,7 @@ func (s *Store) Write(i VolumeId, n *Needle) (size uint32, err error) {
 			err = fmt.Errorf("Volume %d is read only", i)
 			return
 		}
+		// TODO: count needle size ahead
 		if MaxPossibleVolumeSize >= v.ContentSize()+uint64(size) {
 			size, err = v.writeNeedle(n)
 		} else {