add build option support 5-byte offset

3 files changed, 143 insertions, 53 deletions

diff --git a/weed/storage/types/needle_types.go b/weed/storage/types/needle_types.go
index 86eae03ac..b591dd7c3 100644
--- a/weed/storage/types/needle_types.go
+++ b/weed/storage/types/needle_types.go
@@ -12,10 +12,6 @@ type Offset struct {
 	OffsetLower
 }
 
-type OffsetHigher struct {
-	// b4 byte
-}
-
 type OffsetLower struct {
 	b3 byte
 	b2 byte
@@ -26,12 +22,10 @@ type OffsetLower struct {
 type Cookie uint32
 
 const (
-	OffsetSize            = 4 // + 1
 	SizeSize              = 4 // uint32 size
 	NeedleEntrySize       = CookieSize + NeedleIdSize + SizeSize
 	TimestampSize         = 8 // int64 size
 	NeedlePaddingSize     = 8
-	MaxPossibleVolumeSize = 4 * 1024 * 1024 * 1024 * 8
 	TombstoneFileSize     = math.MaxUint32
 	CookieSize            = 4
 )
@@ -54,50 +48,3 @@ func ParseCookie(cookieString string) (Cookie, error) {
 	}
 	return Cookie(cookie), nil
 }
-
-func OffsetToBytes(bytes []byte, offset Offset) {
-	bytes[3] = offset.b0
-	bytes[2] = offset.b1
-	bytes[1] = offset.b2
-	bytes[0] = offset.b3
-}
-
-// only for testing, will be removed later.
-func Uint32ToOffset(offset uint32) Offset {
-	return Offset{
-		OffsetLower: OffsetLower{
-			b0: byte(offset),
-			b1: byte(offset >> 8),
-			b2: byte(offset >> 16),
-			b3: byte(offset >> 24),
-		},
-	}
-}
-
-func BytesToOffset(bytes []byte) Offset {
-	return Offset{
-		OffsetLower: OffsetLower{
-			b0: bytes[3],
-			b1: bytes[2],
-			b2: bytes[1],
-			b3: bytes[0],
-		},
-	}
-}
-
-func (offset Offset) IsZero() bool {
-	return offset.b0 == 0 && offset.b1 == 0 && offset.b2 == 0 && offset.b3 == 0
-}
-
-func ToOffset(offset int64) Offset {
-	smaller := uint32(offset / int64(NeedlePaddingSize))
-	return Uint32ToOffset(smaller)
-}
-
-func (offset Offset) ToAcutalOffset() (actualOffset int64) {
-	return (int64(offset.b0) + int64(offset.b1)<<8 + int64(offset.b2)<<16 + int64(offset.b3)<<24) * int64(NeedlePaddingSize)
-}
-
-func (offset Offset) String() string {
-	return fmt.Sprintf("%d", int64(offset.b0)+int64(offset.b1)<<8+int64(offset.b2)<<16+int64(offset.b3)<<24)
-}
diff --git a/weed/storage/types/offset_4bytes.go b/weed/storage/types/offset_4bytes.go
new file mode 100644
index 000000000..9acd069d3
--- /dev/null
+++ b/weed/storage/types/offset_4bytes.go
@@ -0,0 +1,63 @@
+// +build !5BytesOffset
+
+package types
+
+import (
+	"fmt"
+)
+
+type OffsetHigher struct {
+	// b4 byte
+}
+
+const (
+	OffsetSize            = 4
+	MaxPossibleVolumeSize = 4 * 1024 * 1024 * 1024 * 8 // 32GB
+)
+
+func OffsetToBytes(bytes []byte, offset Offset) {
+	bytes[3] = offset.b0
+	bytes[2] = offset.b1
+	bytes[1] = offset.b2
+	bytes[0] = offset.b3
+}
+
+// only for testing, will be removed later.
+func Uint32ToOffset(offset uint32) Offset {
+	return Offset{
+		OffsetLower: OffsetLower{
+			b0: byte(offset),
+			b1: byte(offset >> 8),
+			b2: byte(offset >> 16),
+			b3: byte(offset >> 24),
+		},
+	}
+}
+
+func BytesToOffset(bytes []byte) Offset {
+	return Offset{
+		OffsetLower: OffsetLower{
+			b0: bytes[3],
+			b1: bytes[2],
+			b2: bytes[1],
+			b3: bytes[0],
+		},
+	}
+}
+
+func (offset Offset) IsZero() bool {
+	return offset.b0 == 0 && offset.b1 == 0 && offset.b2 == 0 && offset.b3 == 0
+}
+
+func ToOffset(offset int64) Offset {
+	smaller := uint32(offset / int64(NeedlePaddingSize))
+	return Uint32ToOffset(smaller)
+}
+
+func (offset Offset) ToAcutalOffset() (actualOffset int64) {
+	return (int64(offset.b0) + int64(offset.b1)<<8 + int64(offset.b2)<<16 + int64(offset.b3)<<24) * int64(NeedlePaddingSize)
+}
+
+func (offset Offset) String() string {
+	return fmt.Sprintf("%d", int64(offset.b0)+int64(offset.b1)<<8+int64(offset.b2)<<16+int64(offset.b3)<<24)
+}
diff --git a/weed/storage/types/offset_5bytes.go b/weed/storage/types/offset_5bytes.go
new file mode 100644
index 000000000..f57e4f6d4
--- /dev/null
+++ b/weed/storage/types/offset_5bytes.go
@@ -0,0 +1,80 @@
+// +build 5BytesOffset
+
+package types
+
+import (
+	"fmt"
+)
+
+type OffsetHigher struct {
+	b4 byte
+}
+
+const (
+	OffsetSize            = 4 + 1
+	MaxPossibleVolumeSize = 4 * 1024 * 1024 * 1024 * 8 * 256 /* 256 is from the extra byte */ // 8TB
+)
+
+func OffsetToBytes(bytes []byte, offset Offset) {
+	bytes[4] = offset.b4
+	bytes[3] = offset.b0
+	bytes[2] = offset.b1
+	bytes[1] = offset.b2
+	bytes[0] = offset.b3
+}
+
+// only for testing, will be removed later.
+func Uint32ToOffset(offset uint32) Offset {
+	return Offset{
+		OffsetHigher: OffsetHigher{
+			b4: byte(offset >> 32),
+		},
+		OffsetLower: OffsetLower{
+			b0: byte(offset),
+			b1: byte(offset >> 8),
+			b2: byte(offset >> 16),
+			b3: byte(offset >> 24),
+		},
+	}
+}
+
+func BytesToOffset(bytes []byte) Offset {
+	return Offset{
+		OffsetHigher: OffsetHigher{
+			b4: bytes[4],
+		},
+		OffsetLower: OffsetLower{
+			b0: bytes[3],
+			b1: bytes[2],
+			b2: bytes[1],
+			b3: bytes[0],
+		},
+	}
+}
+
+func (offset Offset) IsZero() bool {
+	return offset.b0 == 0 && offset.b1 == 0 && offset.b2 == 0 && offset.b3 == 0 && offset.b4 == 0
+}
+
+func ToOffset(offset int64) Offset {
+	smaller := offset / int64(NeedlePaddingSize)
+	return Offset{
+		OffsetHigher: OffsetHigher{
+			b4: byte(smaller >> 32),
+		},
+		OffsetLower: OffsetLower{
+			b0: byte(smaller),
+			b1: byte(smaller >> 8),
+			b2: byte(smaller >> 16),
+			b3: byte(smaller >> 24),
+		},
+	}
+}
+
+func (offset Offset) ToAcutalOffset() (actualOffset int64) {
+	return (int64(offset.b0) + int64(offset.b1)<<8 + int64(offset.b2)<<16 + int64(offset.b3)<<24 + int64(offset.b4)<<32) * int64(NeedlePaddingSize)
+}
+
+func (offset Offset) String() string {
+	return fmt.Sprintf("%d", int64(offset.b0)+int64(offset.b1)<<8+int64(offset.b2)<<16+int64(offset.b3)<<24+int64(offset.b4)<<32)
+}