tests can compile

7 files changed, 1635 insertions, 0 deletions

diff --git a/weed/data/Makefile b/weed/data/Makefile
new file mode 100644
index 000000000..bc6431a24
--- /dev/null
+++ b/weed/data/Makefile
@@ -0,0 +1,6 @@
+all: gen
+
+.PHONY : gen
+
+gen:
+	protoc columnar.proto --go_out=./columnar_pb --go-grpc_out=./columnar_pb --go_opt=paths=source_relative --go-grpc_opt=paths=source_relative
diff --git a/weed/data/column_uint16.go b/weed/data/column_uint16.go
new file mode 100644
index 000000000..3442df041
--- /dev/null
+++ b/weed/data/column_uint16.go
@@ -0,0 +1,32 @@
+package data
+
+import (
+	"encoding/binary"
+	"fmt"
+	"io"
+)
+
+type ColumnUint16 struct {
+}
+
+const SIZE_Uint16 = 2
+
+func (c *ColumnUint16) Read(buf []byte, readerAt io.ReaderAt, offset int64, i int64) uint16 {
+	if n, err := readerAt.ReadAt(buf, offset+i*SIZE_Uint16); n == SIZE_Uint16 && err == nil {
+		return binary.BigEndian.Uint16(buf)
+	}
+	return 0
+}
+
+func WriteUint16s(buf []byte, data []uint16) (err error) {
+	off := 0
+	size := len(data)
+	if len(buf) < size<<1 {
+		return fmt.Errorf("buf too small")
+	}
+	for _, dat := range data {
+		binary.BigEndian.PutUint16(buf[off:], dat)
+		off += SIZE_Uint16
+	}
+	return nil
+}
diff --git a/weed/data/column_uint32.go b/weed/data/column_uint32.go
new file mode 100644
index 000000000..2b661786a
--- /dev/null
+++ b/weed/data/column_uint32.go
@@ -0,0 +1,32 @@
+package data
+
+import (
+	"encoding/binary"
+	"fmt"
+	"io"
+)
+
+type ColumnUint32 struct {
+}
+
+const SIZE_Uint32 = 4
+
+func (c *ColumnUint32) Read(buf []byte, readerAt io.ReaderAt, offset int64, i int64) uint32 {
+	if n, err := readerAt.ReadAt(buf, offset+i*SIZE_Uint32); n == SIZE_Uint32 && err == nil {
+		return binary.BigEndian.Uint32(buf)
+	}
+	return 0
+}
+
+func WriteUint32s(buf []byte, data []uint32) (err error) {
+	off := 0
+	size := len(data)
+	if len(buf) < size<<2 {
+		return fmt.Errorf("buf too small")
+	}
+	for _, dat := range data {
+		binary.BigEndian.PutUint32(buf[off:], dat)
+		off += SIZE_Uint32
+	}
+	return nil
+}
diff --git a/weed/data/columnar.proto b/weed/data/columnar.proto
new file mode 100644
index 000000000..e6ac692b5
--- /dev/null
+++ b/weed/data/columnar.proto
@@ -0,0 +1,103 @@
+syntax = "proto3";
+
+package columnar_pb;
+
+option go_package = "github.com/seaweedfs/seaweedfs/weed/data/columnar_pb";
+
+message FileId {
+  uint32 volume_id = 1;
+  uint64 file_key = 2;
+  fixed32 cookie = 3;
+}
+
+enum LogicalType {
+  Uint8 = 0;
+  Uint16 = 1;
+  Float32 = 4;
+}
+
+message ColumnUint16 {
+  uint32 base = 1;
+  uint32 min = 3;
+  uint32 max = 4;
+}
+
+message ColumnUint32 {
+  uint32 base = 1;
+  uint32 min = 3;
+  uint32 max = 4;
+}
+
+message ColumnFloat32 {
+  uint32 min = 3;
+  uint32 max = 4;
+}
+
+message ColumnSplit {
+  // The ids of the fields/columns in this file
+  int32 field_id = 1;
+  FileId file_id = 2;
+  int64 row_offset = 3;
+  int32 row_count = 4;
+
+  oneof storage_type {
+    ColumnUint16  meta_uint16 = 8;
+    ColumnUint32  meta_uint32 = 9;
+    ColumnFloat32 meta_float32 = 10;
+  }
+}
+
+message Snapshot {
+  // All fields of the dataset, including the nested fields.
+  repeated Field fields = 1;
+
+  repeated string data_files = 2;
+
+  // Snapshot version number.
+  uint64 version = 3;
+
+}
+
+message DataFile {
+  repeated int32 field_ids = 1;
+  repeated RowGroup row_groups = 2;
+}
+
+message RowGroup {
+  int64 row_offset = 1;
+  int32 row_count = 2;
+  repeated ColumnSplit column_splits = 3;
+}
+
+// Field metadata for a column.
+message Field {
+  enum Type {
+    PARENT = 0;
+    REPEATED = 1;
+    LEAF = 2;
+  }
+  Type type = 1;
+
+  // Fully qualified name.
+  string name = 2;
+  /// Field Id.
+  int32 id = 3;
+  /// Parent Field ID. If not set, this is a top-level column.
+  int32 parent_id = 4;
+
+  // Logical types, support parameterized Arrow Type.
+  LogicalType logical_type = 5;
+  // If this field is nullable.
+  bool nullable = 6;
+}
+
+
+message AnyValue {
+  oneof value {
+    bytes bytes_value = 1;
+    bool bool_value = 2;
+    uint64 int64_value = 3;
+    uint32 int32_value = 4;
+    double double_value = 5;
+  }
+}
diff --git a/weed/data/columnar_pb/columnar.pb.go b/weed/data/columnar_pb/columnar.pb.go
new file mode 100644
index 000000000..3b1bde1cc
--- /dev/null
+++ b/weed/data/columnar_pb/columnar.pb.go
@@ -0,0 +1,1199 @@
+// Code generated by protoc-gen-go. DO NOT EDIT.
+// versions:
+// 	protoc-gen-go v1.28.1
+// 	protoc        v3.21.11
+// source: columnar.proto
+
+package columnar_pb
+
+import (
+	protoreflect "google.golang.org/protobuf/reflect/protoreflect"
+	protoimpl "google.golang.org/protobuf/runtime/protoimpl"
+	reflect "reflect"
+	sync "sync"
+)
+
+const (
+	// Verify that this generated code is sufficiently up-to-date.
+	_ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion)
+	// Verify that runtime/protoimpl is sufficiently up-to-date.
+	_ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20)
+)
+
+type LogicalType int32
+
+const (
+	LogicalType_Uint8   LogicalType = 0
+	LogicalType_Uint16  LogicalType = 1
+	LogicalType_Float32 LogicalType = 4
+)
+
+// Enum value maps for LogicalType.
+var (
+	LogicalType_name = map[int32]string{
+		0: "Uint8",
+		1: "Uint16",
+		4: "Float32",
+	}
+	LogicalType_value = map[string]int32{
+		"Uint8":   0,
+		"Uint16":  1,
+		"Float32": 4,
+	}
+)
+
+func (x LogicalType) Enum() *LogicalType {
+	p := new(LogicalType)
+	*p = x
+	return p
+}
+
+func (x LogicalType) String() string {
+	return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x))
+}
+
+func (LogicalType) Descriptor() protoreflect.EnumDescriptor {
+	return file_columnar_proto_enumTypes[0].Descriptor()
+}
+
+func (LogicalType) Type() protoreflect.EnumType {
+	return &file_columnar_proto_enumTypes[0]
+}
+
+func (x LogicalType) Number() protoreflect.EnumNumber {
+	return protoreflect.EnumNumber(x)
+}
+
+// Deprecated: Use LogicalType.Descriptor instead.
+func (LogicalType) EnumDescriptor() ([]byte, []int) {
+	return file_columnar_proto_rawDescGZIP(), []int{0}
+}
+
+type Field_Type int32
+
+const (
+	Field_PARENT   Field_Type = 0
+	Field_REPEATED Field_Type = 1
+	Field_LEAF     Field_Type = 2
+)
+
+// Enum value maps for Field_Type.
+var (
+	Field_Type_name = map[int32]string{
+		0: "PARENT",
+		1: "REPEATED",
+		2: "LEAF",
+	}
+	Field_Type_value = map[string]int32{
+		"PARENT":   0,
+		"REPEATED": 1,
+		"LEAF":     2,
+	}
+)
+
+func (x Field_Type) Enum() *Field_Type {
+	p := new(Field_Type)
+	*p = x
+	return p
+}
+
+func (x Field_Type) String() string {
+	return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x))
+}
+
+func (Field_Type) Descriptor() protoreflect.EnumDescriptor {
+	return file_columnar_proto_enumTypes[1].Descriptor()
+}
+
+func (Field_Type) Type() protoreflect.EnumType {
+	return &file_columnar_proto_enumTypes[1]
+}
+
+func (x Field_Type) Number() protoreflect.EnumNumber {
+	return protoreflect.EnumNumber(x)
+}
+
+// Deprecated: Use Field_Type.Descriptor instead.
+func (Field_Type) EnumDescriptor() ([]byte, []int) {
+	return file_columnar_proto_rawDescGZIP(), []int{8, 0}
+}
+
+type FileId struct {
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
+
+	VolumeId uint32 `protobuf:"varint,1,opt,name=volume_id,json=volumeId,proto3" json:"volume_id,omitempty"`
+	FileKey  uint64 `protobuf:"varint,2,opt,name=file_key,json=fileKey,proto3" json:"file_key,omitempty"`
+	Cookie   uint32 `protobuf:"fixed32,3,opt,name=cookie,proto3" json:"cookie,omitempty"`
+}
+
+func (x *FileId) Reset() {
+	*x = FileId{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_columnar_proto_msgTypes[0]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
+}
+
+func (x *FileId) String() string {
+	return protoimpl.X.MessageStringOf(x)
+}
+
+func (*FileId) ProtoMessage() {}
+
+func (x *FileId) ProtoReflect() protoreflect.Message {
+	mi := &file_columnar_proto_msgTypes[0]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
+}
+
+// Deprecated: Use FileId.ProtoReflect.Descriptor instead.
+func (*FileId) Descriptor() ([]byte, []int) {
+	return file_columnar_proto_rawDescGZIP(), []int{0}
+}
+
+func (x *FileId) GetVolumeId() uint32 {
+	if x != nil {
+		return x.VolumeId
+	}
+	return 0
+}
+
+func (x *FileId) GetFileKey() uint64 {
+	if x != nil {
+		return x.FileKey
+	}
+	return 0
+}
+
+func (x *FileId) GetCookie() uint32 {
+	if x != nil {
+		return x.Cookie
+	}
+	return 0
+}
+
+type ColumnUint16 struct {
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
+
+	Base uint32 `protobuf:"varint,1,opt,name=base,proto3" json:"base,omitempty"`
+	Min  uint32 `protobuf:"varint,3,opt,name=min,proto3" json:"min,omitempty"`
+	Max  uint32 `protobuf:"varint,4,opt,name=max,proto3" json:"max,omitempty"`
+}
+
+func (x *ColumnUint16) Reset() {
+	*x = ColumnUint16{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_columnar_proto_msgTypes[1]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
+}
+
+func (x *ColumnUint16) String() string {
+	return protoimpl.X.MessageStringOf(x)
+}
+
+func (*ColumnUint16) ProtoMessage() {}
+
+func (x *ColumnUint16) ProtoReflect() protoreflect.Message {
+	mi := &file_columnar_proto_msgTypes[1]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
+}
+
+// Deprecated: Use ColumnUint16.ProtoReflect.Descriptor instead.
+func (*ColumnUint16) Descriptor() ([]byte, []int) {
+	return file_columnar_proto_rawDescGZIP(), []int{1}
+}
+
+func (x *ColumnUint16) GetBase() uint32 {
+	if x != nil {
+		return x.Base
+	}
+	return 0
+}
+
+func (x *ColumnUint16) GetMin() uint32 {
+	if x != nil {
+		return x.Min
+	}
+	return 0
+}
+
+func (x *ColumnUint16) GetMax() uint32 {
+	if x != nil {
+		return x.Max
+	}
+	return 0
+}
+
+type ColumnUint32 struct {
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
+
+	Base uint32 `protobuf:"varint,1,opt,name=base,proto3" json:"base,omitempty"`
+	Min  uint32 `protobuf:"varint,3,opt,name=min,proto3" json:"min,omitempty"`
+	Max  uint32 `protobuf:"varint,4,opt,name=max,proto3" json:"max,omitempty"`
+}
+
+func (x *ColumnUint32) Reset() {
+	*x = ColumnUint32{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_columnar_proto_msgTypes[2]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
+}
+
+func (x *ColumnUint32) String() string {
+	return protoimpl.X.MessageStringOf(x)
+}
+
+func (*ColumnUint32) ProtoMessage() {}
+
+func (x *ColumnUint32) ProtoReflect() protoreflect.Message {
+	mi := &file_columnar_proto_msgTypes[2]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
+}
+
+// Deprecated: Use ColumnUint32.ProtoReflect.Descriptor instead.
+func (*ColumnUint32) Descriptor() ([]byte, []int) {
+	return file_columnar_proto_rawDescGZIP(), []int{2}
+}
+
+func (x *ColumnUint32) GetBase() uint32 {
+	if x != nil {
+		return x.Base
+	}
+	return 0
+}
+
+func (x *ColumnUint32) GetMin() uint32 {
+	if x != nil {
+		return x.Min
+	}
+	return 0
+}
+
+func (x *ColumnUint32) GetMax() uint32 {
+	if x != nil {
+		return x.Max
+	}
+	return 0
+}
+
+type ColumnFloat32 struct {
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
+
+	Min uint32 `protobuf:"varint,3,opt,name=min,proto3" json:"min,omitempty"`
+	Max uint32 `protobuf:"varint,4,opt,name=max,proto3" json:"max,omitempty"`
+}
+
+func (x *ColumnFloat32) Reset() {
+	*x = ColumnFloat32{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_columnar_proto_msgTypes[3]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
+}
+
+func (x *ColumnFloat32) String() string {
+	return protoimpl.X.MessageStringOf(x)
+}
+
+func (*ColumnFloat32) ProtoMessage() {}
+
+func (x *ColumnFloat32) ProtoReflect() protoreflect.Message {
+	mi := &file_columnar_proto_msgTypes[3]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
+}
+
+// Deprecated: Use ColumnFloat32.ProtoReflect.Descriptor instead.
+func (*ColumnFloat32) Descriptor() ([]byte, []int) {
+	return file_columnar_proto_rawDescGZIP(), []int{3}
+}
+
+func (x *ColumnFloat32) GetMin() uint32 {
+	if x != nil {
+		return x.Min
+	}
+	return 0
+}
+
+func (x *ColumnFloat32) GetMax() uint32 {
+	if x != nil {
+		return x.Max
+	}
+	return 0
+}
+
+type ColumnSplit struct {
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
+
+	// The ids of the fields/columns in this file
+	FieldId   int32   `protobuf:"varint,1,opt,name=field_id,json=fieldId,proto3" json:"field_id,omitempty"`
+	FileId    *FileId `protobuf:"bytes,2,opt,name=file_id,json=fileId,proto3" json:"file_id,omitempty"`
+	RowOffset int64   `protobuf:"varint,3,opt,name=row_offset,json=rowOffset,proto3" json:"row_offset,omitempty"`
+	RowCount  int32   `protobuf:"varint,4,opt,name=row_count,json=rowCount,proto3" json:"row_count,omitempty"`
+	// Types that are assignable to StorageType:
+	//
+	//	*ColumnSplit_MetaUint16
+	//	*ColumnSplit_MetaUint32
+	//	*ColumnSplit_MetaFloat32
+	StorageType isColumnSplit_StorageType `protobuf_oneof:"storage_type"`
+}
+
+func (x *ColumnSplit) Reset() {
+	*x = ColumnSplit{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_columnar_proto_msgTypes[4]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
+}
+
+func (x *ColumnSplit) String() string {
+	return protoimpl.X.MessageStringOf(x)
+}
+
+func (*ColumnSplit) ProtoMessage() {}
+
+func (x *ColumnSplit) ProtoReflect() protoreflect.Message {
+	mi := &file_columnar_proto_msgTypes[4]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
+}
+
+// Deprecated: Use ColumnSplit.ProtoReflect.Descriptor instead.
+func (*ColumnSplit) Descriptor() ([]byte, []int) {
+	return file_columnar_proto_rawDescGZIP(), []int{4}
+}
+
+func (x *ColumnSplit) GetFieldId() int32 {
+	if x != nil {
+		return x.FieldId
+	}
+	return 0
+}
+
+func (x *ColumnSplit) GetFileId() *FileId {
+	if x != nil {
+		return x.FileId
+	}
+	return nil
+}
+
+func (x *ColumnSplit) GetRowOffset() int64 {
+	if x != nil {
+		return x.RowOffset
+	}
+	return 0
+}
+
+func (x *ColumnSplit) GetRowCount() int32 {
+	if x != nil {
+		return x.RowCount
+	}
+	return 0
+}
+
+func (m *ColumnSplit) GetStorageType() isColumnSplit_StorageType {
+	if m != nil {
+		return m.StorageType
+	}
+	return nil
+}
+
+func (x *ColumnSplit) GetMetaUint16() *ColumnUint16 {
+	if x, ok := x.GetStorageType().(*ColumnSplit_MetaUint16); ok {
+		return x.MetaUint16
+	}
+	return nil
+}
+
+func (x *ColumnSplit) GetMetaUint32() *ColumnUint32 {
+	if x, ok := x.GetStorageType().(*ColumnSplit_MetaUint32); ok {
+		return x.MetaUint32
+	}
+	return nil
+}
+
+func (x *ColumnSplit) GetMetaFloat32() *ColumnFloat32 {
+	if x, ok := x.GetStorageType().(*ColumnSplit_MetaFloat32); ok {
+		return x.MetaFloat32
+	}
+	return nil
+}
+
+type isColumnSplit_StorageType interface {
+	isColumnSplit_StorageType()
+}
+
+type ColumnSplit_MetaUint16 struct {
+	MetaUint16 *ColumnUint16 `protobuf:"bytes,8,opt,name=meta_uint16,json=metaUint16,proto3,oneof"`
+}
+
+type ColumnSplit_MetaUint32 struct {
+	MetaUint32 *ColumnUint32 `protobuf:"bytes,9,opt,name=meta_uint32,json=metaUint32,proto3,oneof"`
+}
+
+type ColumnSplit_MetaFloat32 struct {
+	MetaFloat32 *ColumnFloat32 `protobuf:"bytes,10,opt,name=meta_float32,json=metaFloat32,proto3,oneof"`
+}
+
+func (*ColumnSplit_MetaUint16) isColumnSplit_StorageType() {}
+
+func (*ColumnSplit_MetaUint32) isColumnSplit_StorageType() {}
+
+func (*ColumnSplit_MetaFloat32) isColumnSplit_StorageType() {}
+
+type Snapshot struct {
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
+
+	// All fields of the dataset, including the nested fields.
+	Fields    []*Field `protobuf:"bytes,1,rep,name=fields,proto3" json:"fields,omitempty"`
+	DataFiles []string `protobuf:"bytes,2,rep,name=data_files,json=dataFiles,proto3" json:"data_files,omitempty"`
+	// Snapshot version number.
+	Version uint64 `protobuf:"varint,3,opt,name=version,proto3" json:"version,omitempty"`
+}
+
+func (x *Snapshot) Reset() {
+	*x = Snapshot{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_columnar_proto_msgTypes[5]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
+}
+
+func (x *Snapshot) String() string {
+	return protoimpl.X.MessageStringOf(x)
+}
+
+func (*Snapshot) ProtoMessage() {}
+
+func (x *Snapshot) ProtoReflect() protoreflect.Message {
+	mi := &file_columnar_proto_msgTypes[5]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
+}
+
+// Deprecated: Use Snapshot.ProtoReflect.Descriptor instead.
+func (*Snapshot) Descriptor() ([]byte, []int) {
+	return file_columnar_proto_rawDescGZIP(), []int{5}
+}
+
+func (x *Snapshot) GetFields() []*Field {
+	if x != nil {
+		return x.Fields
+	}
+	return nil
+}
+
+func (x *Snapshot) GetDataFiles() []string {
+	if x != nil {
+		return x.DataFiles
+	}
+	return nil
+}
+
+func (x *Snapshot) GetVersion() uint64 {
+	if x != nil {
+		return x.Version
+	}
+	return 0
+}
+
+type DataFile struct {
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
+
+	FieldIds  []int32     `protobuf:"varint,1,rep,packed,name=field_ids,json=fieldIds,proto3" json:"field_ids,omitempty"`
+	RowGroups []*RowGroup `protobuf:"bytes,2,rep,name=row_groups,json=rowGroups,proto3" json:"row_groups,omitempty"`
+}
+
+func (x *DataFile) Reset() {
+	*x = DataFile{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_columnar_proto_msgTypes[6]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
+}
+
+func (x *DataFile) String() string {
+	return protoimpl.X.MessageStringOf(x)
+}
+
+func (*DataFile) ProtoMessage() {}
+
+func (x *DataFile) ProtoReflect() protoreflect.Message {
+	mi := &file_columnar_proto_msgTypes[6]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
+}
+
+// Deprecated: Use DataFile.ProtoReflect.Descriptor instead.
+func (*DataFile) Descriptor() ([]byte, []int) {
+	return file_columnar_proto_rawDescGZIP(), []int{6}
+}
+
+func (x *DataFile) GetFieldIds() []int32 {
+	if x != nil {
+		return x.FieldIds
+	}
+	return nil
+}
+
+func (x *DataFile) GetRowGroups() []*RowGroup {
+	if x != nil {
+		return x.RowGroups
+	}
+	return nil
+}
+
+type RowGroup struct {
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
+
+	RowOffset    int64          `protobuf:"varint,1,opt,name=row_offset,json=rowOffset,proto3" json:"row_offset,omitempty"`
+	RowCount     int32          `protobuf:"varint,2,opt,name=row_count,json=rowCount,proto3" json:"row_count,omitempty"`
+	ColumnSplits []*ColumnSplit `protobuf:"bytes,3,rep,name=column_splits,json=columnSplits,proto3" json:"column_splits,omitempty"`
+}
+
+func (x *RowGroup) Reset() {
+	*x = RowGroup{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_columnar_proto_msgTypes[7]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
+}
+
+func (x *RowGroup) String() string {
+	return protoimpl.X.MessageStringOf(x)
+}
+
+func (*RowGroup) ProtoMessage() {}
+
+func (x *RowGroup) ProtoReflect() protoreflect.Message {
+	mi := &file_columnar_proto_msgTypes[7]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
+}
+
+// Deprecated: Use RowGroup.ProtoReflect.Descriptor instead.
+func (*RowGroup) Descriptor() ([]byte, []int) {
+	return file_columnar_proto_rawDescGZIP(), []int{7}
+}
+
+func (x *RowGroup) GetRowOffset() int64 {
+	if x != nil {
+		return x.RowOffset
+	}
+	return 0
+}
+
+func (x *RowGroup) GetRowCount() int32 {
+	if x != nil {
+		return x.RowCount
+	}
+	return 0
+}
+
+func (x *RowGroup) GetColumnSplits() []*ColumnSplit {
+	if x != nil {
+		return x.ColumnSplits
+	}
+	return nil
+}
+
+// Field metadata for a column.
+type Field struct {
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
+
+	Type Field_Type `protobuf:"varint,1,opt,name=type,proto3,enum=columnar_pb.Field_Type" json:"type,omitempty"`
+	// Fully qualified name.
+	Name string `protobuf:"bytes,2,opt,name=name,proto3" json:"name,omitempty"`
+	// / Field Id.
+	Id int32 `protobuf:"varint,3,opt,name=id,proto3" json:"id,omitempty"`
+	// / Parent Field ID. If not set, this is a top-level column.
+	ParentId int32 `protobuf:"varint,4,opt,name=parent_id,json=parentId,proto3" json:"parent_id,omitempty"`
+	// Logical types, support parameterized Arrow Type.
+	LogicalType LogicalType `protobuf:"varint,5,opt,name=logical_type,json=logicalType,proto3,enum=columnar_pb.LogicalType" json:"logical_type,omitempty"`
+	// If this field is nullable.
+	Nullable bool `protobuf:"varint,6,opt,name=nullable,proto3" json:"nullable,omitempty"`
+}
+
+func (x *Field) Reset() {
+	*x = Field{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_columnar_proto_msgTypes[8]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
+}
+
+func (x *Field) String() string {
+	return protoimpl.X.MessageStringOf(x)
+}
+
+func (*Field) ProtoMessage() {}
+
+func (x *Field) ProtoReflect() protoreflect.Message {
+	mi := &file_columnar_proto_msgTypes[8]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
+}
+
+// Deprecated: Use Field.ProtoReflect.Descriptor instead.
+func (*Field) Descriptor() ([]byte, []int) {
+	return file_columnar_proto_rawDescGZIP(), []int{8}
+}
+
+func (x *Field) GetType() Field_Type {
+	if x != nil {
+		return x.Type
+	}
+	return Field_PARENT
+}
+
+func (x *Field) GetName() string {
+	if x != nil {
+		return x.Name
+	}
+	return ""
+}
+
+func (x *Field) GetId() int32 {
+	if x != nil {
+		return x.Id
+	}
+	return 0
+}
+
+func (x *Field) GetParentId() int32 {
+	if x != nil {
+		return x.ParentId
+	}
+	return 0
+}
+
+func (x *Field) GetLogicalType() LogicalType {
+	if x != nil {
+		return x.LogicalType
+	}
+	return LogicalType_Uint8
+}
+
+func (x *Field) GetNullable() bool {
+	if x != nil {
+		return x.Nullable
+	}
+	return false
+}
+
+type AnyValue struct {
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
+
+	// Types that are assignable to Value:
+	//
+	//	*AnyValue_BytesValue
+	//	*AnyValue_BoolValue
+	//	*AnyValue_Int64Value
+	//	*AnyValue_Int32Value
+	//	*AnyValue_DoubleValue
+	Value isAnyValue_Value `protobuf_oneof:"value"`
+}
+
+func (x *AnyValue) Reset() {
+	*x = AnyValue{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_columnar_proto_msgTypes[9]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
+}
+
+func (x *AnyValue) String() string {
+	return protoimpl.X.MessageStringOf(x)
+}
+
+func (*AnyValue) ProtoMessage() {}
+
+func (x *AnyValue) ProtoReflect() protoreflect.Message {
+	mi := &file_columnar_proto_msgTypes[9]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
+}
+
+// Deprecated: Use AnyValue.ProtoReflect.Descriptor instead.
+func (*AnyValue) Descriptor() ([]byte, []int) {
+	return file_columnar_proto_rawDescGZIP(), []int{9}
+}
+
+func (m *AnyValue) GetValue() isAnyValue_Value {
+	if m != nil {
+		return m.Value
+	}
+	return nil
+}
+
+func (x *AnyValue) GetBytesValue() []byte {
+	if x, ok := x.GetValue().(*AnyValue_BytesValue); ok {
+		return x.BytesValue
+	}
+	return nil
+}
+
+func (x *AnyValue) GetBoolValue() bool {
+	if x, ok := x.GetValue().(*AnyValue_BoolValue); ok {
+		return x.BoolValue
+	}
+	return false
+}
+
+func (x *AnyValue) GetInt64Value() uint64 {
+	if x, ok := x.GetValue().(*AnyValue_Int64Value); ok {
+		return x.Int64Value
+	}
+	return 0
+}
+
+func (x *AnyValue) GetInt32Value() uint32 {
+	if x, ok := x.GetValue().(*AnyValue_Int32Value); ok {
+		return x.Int32Value
+	}
+	return 0
+}
+
+func (x *AnyValue) GetDoubleValue() float64 {
+	if x, ok := x.GetValue().(*AnyValue_DoubleValue); ok {
+		return x.DoubleValue
+	}
+	return 0
+}
+
+type isAnyValue_Value interface {
+	isAnyValue_Value()
+}
+
+type AnyValue_BytesValue struct {
+	BytesValue []byte `protobuf:"bytes,1,opt,name=bytes_value,json=bytesValue,proto3,oneof"`
+}
+
+type AnyValue_BoolValue struct {
+	BoolValue bool `protobuf:"varint,2,opt,name=bool_value,json=boolValue,proto3,oneof"`
+}
+
+type AnyValue_Int64Value struct {
+	Int64Value uint64 `protobuf:"varint,3,opt,name=int64_value,json=int64Value,proto3,oneof"`
+}
+
+type AnyValue_Int32Value struct {
+	Int32Value uint32 `protobuf:"varint,4,opt,name=int32_value,json=int32Value,proto3,oneof"`
+}
+
+type AnyValue_DoubleValue struct {
+	DoubleValue float64 `protobuf:"fixed64,5,opt,name=double_value,json=doubleValue,proto3,oneof"`
+}
+
+func (*AnyValue_BytesValue) isAnyValue_Value() {}
+
+func (*AnyValue_BoolValue) isAnyValue_Value() {}
+
+func (*AnyValue_Int64Value) isAnyValue_Value() {}
+
+func (*AnyValue_Int32Value) isAnyValue_Value() {}
+
+func (*AnyValue_DoubleValue) isAnyValue_Value() {}
+
+var File_columnar_proto protoreflect.FileDescriptor
+
+var file_columnar_proto_rawDesc = []byte{
+	0x0a, 0x0e, 0x63, 0x6f, 0x6c, 0x75, 0x6d, 0x6e, 0x61, 0x72, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f,
+	0x12, 0x0b, 0x63, 0x6f, 0x6c, 0x75, 0x6d, 0x6e, 0x61, 0x72, 0x5f, 0x70, 0x62, 0x22, 0x58, 0x0a,
+	0x06, 0x46, 0x69, 0x6c, 0x65, 0x49, 0x64, 0x12, 0x1b, 0x0a, 0x09, 0x76, 0x6f, 0x6c, 0x75, 0x6d,
+	0x65, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x08, 0x76, 0x6f, 0x6c, 0x75,
+	0x6d, 0x65, 0x49, 0x64, 0x12, 0x19, 0x0a, 0x08, 0x66, 0x69, 0x6c, 0x65, 0x5f, 0x6b, 0x65, 0x79,
+	0x18, 0x02, 0x20, 0x01, 0x28, 0x04, 0x52, 0x07, 0x66, 0x69, 0x6c, 0x65, 0x4b, 0x65, 0x79, 0x12,
+	0x16, 0x0a, 0x06, 0x63, 0x6f, 0x6f, 0x6b, 0x69, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x07, 0x52,
+	0x06, 0x63, 0x6f, 0x6f, 0x6b, 0x69, 0x65, 0x22, 0x46, 0x0a, 0x0c, 0x43, 0x6f, 0x6c, 0x75, 0x6d,
+	0x6e, 0x55, 0x69, 0x6e, 0x74, 0x31, 0x36, 0x12, 0x12, 0x0a, 0x04, 0x62, 0x61, 0x73, 0x65, 0x18,
+	0x01, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x04, 0x62, 0x61, 0x73, 0x65, 0x12, 0x10, 0x0a, 0x03, 0x6d,
+	0x69, 0x6e, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x03, 0x6d, 0x69, 0x6e, 0x12, 0x10, 0x0a,
+	0x03, 0x6d, 0x61, 0x78, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x03, 0x6d, 0x61, 0x78, 0x22,
+	0x46, 0x0a, 0x0c, 0x43, 0x6f, 0x6c, 0x75, 0x6d, 0x6e, 0x55, 0x69, 0x6e, 0x74, 0x33, 0x32, 0x12,
+	0x12, 0x0a, 0x04, 0x62, 0x61, 0x73, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x04, 0x62,
+	0x61, 0x73, 0x65, 0x12, 0x10, 0x0a, 0x03, 0x6d, 0x69, 0x6e, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0d,
+	0x52, 0x03, 0x6d, 0x69, 0x6e, 0x12, 0x10, 0x0a, 0x03, 0x6d, 0x61, 0x78, 0x18, 0x04, 0x20, 0x01,
+	0x28, 0x0d, 0x52, 0x03, 0x6d, 0x61, 0x78, 0x22, 0x33, 0x0a, 0x0d, 0x43, 0x6f, 0x6c, 0x75, 0x6d,
+	0x6e, 0x46, 0x6c, 0x6f, 0x61, 0x74, 0x33, 0x32, 0x12, 0x10, 0x0a, 0x03, 0x6d, 0x69, 0x6e, 0x18,
+	0x03, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x03, 0x6d, 0x69, 0x6e, 0x12, 0x10, 0x0a, 0x03, 0x6d, 0x61,
+	0x78, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x03, 0x6d, 0x61, 0x78, 0x22, 0xdf, 0x02, 0x0a,
+	0x0b, 0x43, 0x6f, 0x6c, 0x75, 0x6d, 0x6e, 0x53, 0x70, 0x6c, 0x69, 0x74, 0x12, 0x19, 0x0a, 0x08,
+	0x66, 0x69, 0x65, 0x6c, 0x64, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x05, 0x52, 0x07,
+	0x66, 0x69, 0x65, 0x6c, 0x64, 0x49, 0x64, 0x12, 0x2c, 0x0a, 0x07, 0x66, 0x69, 0x6c, 0x65, 0x5f,
+	0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x13, 0x2e, 0x63, 0x6f, 0x6c, 0x75, 0x6d,
+	0x6e, 0x61, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x46, 0x69, 0x6c, 0x65, 0x49, 0x64, 0x52, 0x06, 0x66,
+	0x69, 0x6c, 0x65, 0x49, 0x64, 0x12, 0x1d, 0x0a, 0x0a, 0x72, 0x6f, 0x77, 0x5f, 0x6f, 0x66, 0x66,
+	0x73, 0x65, 0x74, 0x18, 0x03, 0x20, 0x01, 0x28, 0x03, 0x52, 0x09, 0x72, 0x6f, 0x77, 0x4f, 0x66,
+	0x66, 0x73, 0x65, 0x74, 0x12, 0x1b, 0x0a, 0x09, 0x72, 0x6f, 0x77, 0x5f, 0x63, 0x6f, 0x75, 0x6e,
+	0x74, 0x18, 0x04, 0x20, 0x01, 0x28, 0x05, 0x52, 0x08, 0x72, 0x6f, 0x77, 0x43, 0x6f, 0x75, 0x6e,
+	0x74, 0x12, 0x3c, 0x0a, 0x0b, 0x6d, 0x65, 0x74, 0x61, 0x5f, 0x75, 0x69, 0x6e, 0x74, 0x31, 0x36,
+	0x18, 0x08, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x19, 0x2e, 0x63, 0x6f, 0x6c, 0x75, 0x6d, 0x6e, 0x61,
+	0x72, 0x5f, 0x70, 0x62, 0x2e, 0x43, 0x6f, 0x6c, 0x75, 0x6d, 0x6e, 0x55, 0x69, 0x6e, 0x74, 0x31,
+	0x36, 0x48, 0x00, 0x52, 0x0a, 0x6d, 0x65, 0x74, 0x61, 0x55, 0x69, 0x6e, 0x74, 0x31, 0x36, 0x12,
+	0x3c, 0x0a, 0x0b, 0x6d, 0x65, 0x74, 0x61, 0x5f, 0x75, 0x69, 0x6e, 0x74, 0x33, 0x32, 0x18, 0x09,
+	0x20, 0x01, 0x28, 0x0b, 0x32, 0x19, 0x2e, 0x63, 0x6f, 0x6c, 0x75, 0x6d, 0x6e, 0x61, 0x72, 0x5f,
+	0x70, 0x62, 0x2e, 0x43, 0x6f, 0x6c, 0x75, 0x6d, 0x6e, 0x55, 0x69, 0x6e, 0x74, 0x33, 0x32, 0x48,
+	0x00, 0x52, 0x0a, 0x6d, 0x65, 0x74, 0x61, 0x55, 0x69, 0x6e, 0x74, 0x33, 0x32, 0x12, 0x3f, 0x0a,
+	0x0c, 0x6d, 0x65, 0x74, 0x61, 0x5f, 0x66, 0x6c, 0x6f, 0x61, 0x74, 0x33, 0x32, 0x18, 0x0a, 0x20,
+	0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x63, 0x6f, 0x6c, 0x75, 0x6d, 0x6e, 0x61, 0x72, 0x5f, 0x70,
+	0x62, 0x2e, 0x43, 0x6f, 0x6c, 0x75, 0x6d, 0x6e, 0x46, 0x6c, 0x6f, 0x61, 0x74, 0x33, 0x32, 0x48,
+	0x00, 0x52, 0x0b, 0x6d, 0x65, 0x74, 0x61, 0x46, 0x6c, 0x6f, 0x61, 0x74, 0x33, 0x32, 0x42, 0x0e,
+	0x0a, 0x0c, 0x73, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x5f, 0x74, 0x79, 0x70, 0x65, 0x22, 0x6f,
+	0x0a, 0x08, 0x53, 0x6e, 0x61, 0x70, 0x73, 0x68, 0x6f, 0x74, 0x12, 0x2a, 0x0a, 0x06, 0x66, 0x69,
+	0x65, 0x6c, 0x64, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x63, 0x6f, 0x6c,
+	0x75, 0x6d, 0x6e, 0x61, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x46, 0x69, 0x65, 0x6c, 0x64, 0x52, 0x06,
+	0x66, 0x69, 0x65, 0x6c, 0x64, 0x73, 0x12, 0x1d, 0x0a, 0x0a, 0x64, 0x61, 0x74, 0x61, 0x5f, 0x66,
+	0x69, 0x6c, 0x65, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x09, 0x52, 0x09, 0x64, 0x61, 0x74, 0x61,
+	0x46, 0x69, 0x6c, 0x65, 0x73, 0x12, 0x18, 0x0a, 0x07, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e,
+	0x18, 0x03, 0x20, 0x01, 0x28, 0x04, 0x52, 0x07, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x22,
+	0x5d, 0x0a, 0x08, 0x44, 0x61, 0x74, 0x61, 0x46, 0x69, 0x6c, 0x65, 0x12, 0x1b, 0x0a, 0x09, 0x66,
+	0x69, 0x65, 0x6c, 0x64, 0x5f, 0x69, 0x64, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x05, 0x52, 0x08,
+	0x66, 0x69, 0x65, 0x6c, 0x64, 0x49, 0x64, 0x73, 0x12, 0x34, 0x0a, 0x0a, 0x72, 0x6f, 0x77, 0x5f,
+	0x67, 0x72, 0x6f, 0x75, 0x70, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x15, 0x2e, 0x63,
+	0x6f, 0x6c, 0x75, 0x6d, 0x6e, 0x61, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x52, 0x6f, 0x77, 0x47, 0x72,
+	0x6f, 0x75, 0x70, 0x52, 0x09, 0x72, 0x6f, 0x77, 0x47, 0x72, 0x6f, 0x75, 0x70, 0x73, 0x22, 0x85,
+	0x01, 0x0a, 0x08, 0x52, 0x6f, 0x77, 0x47, 0x72, 0x6f, 0x75, 0x70, 0x12, 0x1d, 0x0a, 0x0a, 0x72,
+	0x6f, 0x77, 0x5f, 0x6f, 0x66, 0x66, 0x73, 0x65, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x03, 0x52,
+	0x09, 0x72, 0x6f, 0x77, 0x4f, 0x66, 0x66, 0x73, 0x65, 0x74, 0x12, 0x1b, 0x0a, 0x09, 0x72, 0x6f,
+	0x77, 0x5f, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x05, 0x52, 0x08, 0x72,
+	0x6f, 0x77, 0x43, 0x6f, 0x75, 0x6e, 0x74, 0x12, 0x3d, 0x0a, 0x0d, 0x63, 0x6f, 0x6c, 0x75, 0x6d,
+	0x6e, 0x5f, 0x73, 0x70, 0x6c, 0x69, 0x74, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x18,
+	0x2e, 0x63, 0x6f, 0x6c, 0x75, 0x6d, 0x6e, 0x61, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x43, 0x6f, 0x6c,
+	0x75, 0x6d, 0x6e, 0x53, 0x70, 0x6c, 0x69, 0x74, 0x52, 0x0c, 0x63, 0x6f, 0x6c, 0x75, 0x6d, 0x6e,
+	0x53, 0x70, 0x6c, 0x69, 0x74, 0x73, 0x22, 0xfa, 0x01, 0x0a, 0x05, 0x46, 0x69, 0x65, 0x6c, 0x64,
+	0x12, 0x2b, 0x0a, 0x04, 0x74, 0x79, 0x70, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x17,
+	0x2e, 0x63, 0x6f, 0x6c, 0x75, 0x6d, 0x6e, 0x61, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x46, 0x69, 0x65,
+	0x6c, 0x64, 0x2e, 0x54, 0x79, 0x70, 0x65, 0x52, 0x04, 0x74, 0x79, 0x70, 0x65, 0x12, 0x12, 0x0a,
+	0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d,
+	0x65, 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x03, 0x20, 0x01, 0x28, 0x05, 0x52, 0x02, 0x69,
+	0x64, 0x12, 0x1b, 0x0a, 0x09, 0x70, 0x61, 0x72, 0x65, 0x6e, 0x74, 0x5f, 0x69, 0x64, 0x18, 0x04,
+	0x20, 0x01, 0x28, 0x05, 0x52, 0x08, 0x70, 0x61, 0x72, 0x65, 0x6e, 0x74, 0x49, 0x64, 0x12, 0x3b,
+	0x0a, 0x0c, 0x6c, 0x6f, 0x67, 0x69, 0x63, 0x61, 0x6c, 0x5f, 0x74, 0x79, 0x70, 0x65, 0x18, 0x05,
+	0x20, 0x01, 0x28, 0x0e, 0x32, 0x18, 0x2e, 0x63, 0x6f, 0x6c, 0x75, 0x6d, 0x6e, 0x61, 0x72, 0x5f,
+	0x70, 0x62, 0x2e, 0x4c, 0x6f, 0x67, 0x69, 0x63, 0x61, 0x6c, 0x54, 0x79, 0x70, 0x65, 0x52, 0x0b,
+	0x6c, 0x6f, 0x67, 0x69, 0x63, 0x61, 0x6c, 0x54, 0x79, 0x70, 0x65, 0x12, 0x1a, 0x0a, 0x08, 0x6e,
+	0x75, 0x6c, 0x6c, 0x61, 0x62, 0x6c, 0x65, 0x18, 0x06, 0x20, 0x01, 0x28, 0x08, 0x52, 0x08, 0x6e,
+	0x75, 0x6c, 0x6c, 0x61, 0x62, 0x6c, 0x65, 0x22, 0x2a, 0x0a, 0x04, 0x54, 0x79, 0x70, 0x65, 0x12,
+	0x0a, 0x0a, 0x06, 0x50, 0x41, 0x52, 0x45, 0x4e, 0x54, 0x10, 0x00, 0x12, 0x0c, 0x0a, 0x08, 0x52,
+	0x45, 0x50, 0x45, 0x41, 0x54, 0x45, 0x44, 0x10, 0x01, 0x12, 0x08, 0x0a, 0x04, 0x4c, 0x45, 0x41,
+	0x46, 0x10, 0x02, 0x22, 0xc2, 0x01, 0x0a, 0x08, 0x41, 0x6e, 0x79, 0x56, 0x61, 0x6c, 0x75, 0x65,
+	0x12, 0x21, 0x0a, 0x0b, 0x62, 0x79, 0x74, 0x65, 0x73, 0x5f, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18,
+	0x01, 0x20, 0x01, 0x28, 0x0c, 0x48, 0x00, 0x52, 0x0a, 0x62, 0x79, 0x74, 0x65, 0x73, 0x56, 0x61,
+	0x6c, 0x75, 0x65, 0x12, 0x1f, 0x0a, 0x0a, 0x62, 0x6f, 0x6f, 0x6c, 0x5f, 0x76, 0x61, 0x6c, 0x75,
+	0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x08, 0x48, 0x00, 0x52, 0x09, 0x62, 0x6f, 0x6f, 0x6c, 0x56,
+	0x61, 0x6c, 0x75, 0x65, 0x12, 0x21, 0x0a, 0x0b, 0x69, 0x6e, 0x74, 0x36, 0x34, 0x5f, 0x76, 0x61,
+	0x6c, 0x75, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x04, 0x48, 0x00, 0x52, 0x0a, 0x69, 0x6e, 0x74,
+	0x36, 0x34, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x12, 0x21, 0x0a, 0x0b, 0x69, 0x6e, 0x74, 0x33, 0x32,
+	0x5f, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0d, 0x48, 0x00, 0x52, 0x0a,
+	0x69, 0x6e, 0x74, 0x33, 0x32, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x12, 0x23, 0x0a, 0x0c, 0x64, 0x6f,
+	0x75, 0x62, 0x6c, 0x65, 0x5f, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x05, 0x20, 0x01, 0x28, 0x01,
+	0x48, 0x00, 0x52, 0x0b, 0x64, 0x6f, 0x75, 0x62, 0x6c, 0x65, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x42,
+	0x07, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x2a, 0x31, 0x0a, 0x0b, 0x4c, 0x6f, 0x67, 0x69,
+	0x63, 0x61, 0x6c, 0x54, 0x79, 0x70, 0x65, 0x12, 0x09, 0x0a, 0x05, 0x55, 0x69, 0x6e, 0x74, 0x38,
+	0x10, 0x00, 0x12, 0x0a, 0x0a, 0x06, 0x55, 0x69, 0x6e, 0x74, 0x31, 0x36, 0x10, 0x01, 0x12, 0x0b,
+	0x0a, 0x07, 0x46, 0x6c, 0x6f, 0x61, 0x74, 0x33, 0x32, 0x10, 0x04, 0x42, 0x36, 0x5a, 0x34, 0x67,
+	0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x73, 0x65, 0x61, 0x77, 0x65, 0x65,
+	0x64, 0x66, 0x73, 0x2f, 0x73, 0x65, 0x61, 0x77, 0x65, 0x65, 0x64, 0x66, 0x73, 0x2f, 0x77, 0x65,
+	0x65, 0x64, 0x2f, 0x64, 0x61, 0x74, 0x61, 0x2f, 0x63, 0x6f, 0x6c, 0x75, 0x6d, 0x6e, 0x61, 0x72,
+	0x5f, 0x70, 0x62, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
+}
+
+var (
+	file_columnar_proto_rawDescOnce sync.Once
+	file_columnar_proto_rawDescData = file_columnar_proto_rawDesc
+)
+
+func file_columnar_proto_rawDescGZIP() []byte {
+	file_columnar_proto_rawDescOnce.Do(func() {
+		file_columnar_proto_rawDescData = protoimpl.X.CompressGZIP(file_columnar_proto_rawDescData)
+	})
+	return file_columnar_proto_rawDescData
+}
+
+var file_columnar_proto_enumTypes = make([]protoimpl.EnumInfo, 2)
+var file_columnar_proto_msgTypes = make([]protoimpl.MessageInfo, 10)
+var file_columnar_proto_goTypes = []interface{}{
+	(LogicalType)(0),      // 0: columnar_pb.LogicalType
+	(Field_Type)(0),       // 1: columnar_pb.Field.Type
+	(*FileId)(nil),        // 2: columnar_pb.FileId
+	(*ColumnUint16)(nil),  // 3: columnar_pb.ColumnUint16
+	(*ColumnUint32)(nil),  // 4: columnar_pb.ColumnUint32
+	(*ColumnFloat32)(nil), // 5: columnar_pb.ColumnFloat32
+	(*ColumnSplit)(nil),   // 6: columnar_pb.ColumnSplit
+	(*Snapshot)(nil),      // 7: columnar_pb.Snapshot
+	(*DataFile)(nil),      // 8: columnar_pb.DataFile
+	(*RowGroup)(nil),      // 9: columnar_pb.RowGroup
+	(*Field)(nil),         // 10: columnar_pb.Field
+	(*AnyValue)(nil),      // 11: columnar_pb.AnyValue
+}
+var file_columnar_proto_depIdxs = []int32{
+	2,  // 0: columnar_pb.ColumnSplit.file_id:type_name -> columnar_pb.FileId
+	3,  // 1: columnar_pb.ColumnSplit.meta_uint16:type_name -> columnar_pb.ColumnUint16
+	4,  // 2: columnar_pb.ColumnSplit.meta_uint32:type_name -> columnar_pb.ColumnUint32
+	5,  // 3: columnar_pb.ColumnSplit.meta_float32:type_name -> columnar_pb.ColumnFloat32
+	10, // 4: columnar_pb.Snapshot.fields:type_name -> columnar_pb.Field
+	9,  // 5: columnar_pb.DataFile.row_groups:type_name -> columnar_pb.RowGroup
+	6,  // 6: columnar_pb.RowGroup.column_splits:type_name -> columnar_pb.ColumnSplit
+	1,  // 7: columnar_pb.Field.type:type_name -> columnar_pb.Field.Type
+	0,  // 8: columnar_pb.Field.logical_type:type_name -> columnar_pb.LogicalType
+	9,  // [9:9] is the sub-list for method output_type
+	9,  // [9:9] is the sub-list for method input_type
+	9,  // [9:9] is the sub-list for extension type_name
+	9,  // [9:9] is the sub-list for extension extendee
+	0,  // [0:9] is the sub-list for field type_name
+}
+
+func init() { file_columnar_proto_init() }
+func file_columnar_proto_init() {
+	if File_columnar_proto != nil {
+		return
+	}
+	if !protoimpl.UnsafeEnabled {
+		file_columnar_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*FileId); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+		file_columnar_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*ColumnUint16); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+		file_columnar_proto_msgTypes[2].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*ColumnUint32); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+		file_columnar_proto_msgTypes[3].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*ColumnFloat32); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+		file_columnar_proto_msgTypes[4].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*ColumnSplit); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+		file_columnar_proto_msgTypes[5].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*Snapshot); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+		file_columnar_proto_msgTypes[6].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*DataFile); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+		file_columnar_proto_msgTypes[7].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*RowGroup); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+		file_columnar_proto_msgTypes[8].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*Field); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+		file_columnar_proto_msgTypes[9].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*AnyValue); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+	}
+	file_columnar_proto_msgTypes[4].OneofWrappers = []interface{}{
+		(*ColumnSplit_MetaUint16)(nil),
+		(*ColumnSplit_MetaUint32)(nil),
+		(*ColumnSplit_MetaFloat32)(nil),
+	}
+	file_columnar_proto_msgTypes[9].OneofWrappers = []interface{}{
+		(*AnyValue_BytesValue)(nil),
+		(*AnyValue_BoolValue)(nil),
+		(*AnyValue_Int64Value)(nil),
+		(*AnyValue_Int32Value)(nil),
+		(*AnyValue_DoubleValue)(nil),
+	}
+	type x struct{}
+	out := protoimpl.TypeBuilder{
+		File: protoimpl.DescBuilder{
+			GoPackagePath: reflect.TypeOf(x{}).PkgPath(),
+			RawDescriptor: file_columnar_proto_rawDesc,
+			NumEnums:      2,
+			NumMessages:   10,
+			NumExtensions: 0,
+			NumServices:   0,
+		},
+		GoTypes:           file_columnar_proto_goTypes,
+		DependencyIndexes: file_columnar_proto_depIdxs,
+		EnumInfos:         file_columnar_proto_enumTypes,
+		MessageInfos:      file_columnar_proto_msgTypes,
+	}.Build()
+	File_columnar_proto = out.File
+	file_columnar_proto_rawDesc = nil
+	file_columnar_proto_goTypes = nil
+	file_columnar_proto_depIdxs = nil
+}
diff --git a/weed/data/datum.go b/weed/data/datum.go
new file mode 100644
index 000000000..5805b691d
--- /dev/null
+++ b/weed/data/datum.go
@@ -0,0 +1,69 @@
+package data
+
+import "fmt"
+
+type Datum interface {
+	Compare(other Datum) (int, error)
+}
+type Datums []Datum
+
+type DUint16 uint16
+type DUint32 uint32
+type dNull struct{}
+
+var (
+	DNull Datum = dNull{}
+)
+
+func (d dNull) Compare(other Datum) (int, error) {
+	if other == DNull {
+		return 0, nil
+	}
+	return -1, nil
+}
+
+func NewDUint16(d DUint16) *DUint16 {
+	return &d
+}
+func NewDUint32(d DUint32) *DUint32 {
+	return &d
+}
+
+func (d *DUint16) Compare(other Datum) (int, error) {
+	if other == DNull {
+		return 1, nil
+	}
+	thisV := *d
+	var otherV DUint16
+	switch t := other.(type) {
+	case *DUint16:
+		otherV = *t
+	default:
+		return 0, fmt.Errorf("unsupported")
+	}
+	if thisV < otherV {
+		return -1, nil
+	}
+	if thisV > otherV {
+		return 1, nil
+	}
+	return 0, nil
+}
+func (d *DUint32) Compare(other Datum) (int, error) {
+	if other == DNull {
+		return 1, nil
+	}
+	thisV := *d
+	var otherV DUint32
+	switch t := other.(type) {
+	case *DUint32:
+		otherV = *t
+	}
+	if thisV < otherV {
+		return -1, nil
+	}
+	if thisV > otherV {
+		return 1, nil
+	}
+	return 0, nil
+}
diff --git a/weed/data/read_test.go b/weed/data/read_test.go
new file mode 100644
index 000000000..83742d675
--- /dev/null
+++ b/weed/data/read_test.go
@@ -0,0 +1,194 @@
+package data
+
+import (
+	"encoding/binary"
+	"fmt"
+	"github.com/seaweedfs/seaweedfs/weed/util"
+	"io"
+	"testing"
+)
+
+func TestRead(t *testing.T) {
+	x := make([]uint16, 128)
+	y := make([]uint32, 128)
+
+	for i := range x {
+		x[i] = uint16(i)
+	}
+	for i := range y {
+		y[i] = uint32(i * 32)
+	}
+
+	xbuf := make([]byte, len(x)*SIZE_Uint16)
+	ybuf := make([]byte, len(x)*SIZE_Uint32)
+
+	WriteUint16s(xbuf, x)
+	WriteUint32s(ybuf, y)
+
+	df := &DataFile{
+		xbuf:      xbuf,
+		ybuf:      ybuf,
+		xLen:      len(xbuf),
+		yLen:      len(ybuf),
+		xReaderAt: util.NewBytesReader(xbuf),
+		yReaderAt: util.NewBytesReader(ybuf),
+	}
+
+	dataLayout := make(map[FieldName]DataLayout)
+	dataLayout["x"] = DataLayout{
+		LayoutType: Uint16,
+		SortType:   Unsorted,
+	}
+	dataLayout["y"] = DataLayout{
+		LayoutType: Uint32,
+		SortType:   Unsorted,
+	}
+
+	rows, err := df.ReadRows("x", dataLayout, Equal, NewDUint16(65))
+	if err != nil {
+		fmt.Printf("err: %v", err)
+		return
+	}
+	for _, row := range rows {
+		fmt.Printf("row %d width %d ", row.index, len(row.Datums))
+		for i, d := range row.Datums {
+			fmt.Printf("%d: %v ", i, d)
+		}
+		fmt.Println()
+	}
+
+}
+
+type Operator int32
+type LayoutType int32
+type SortType int32
+
+const (
+	Equal Operator = 0
+	GreaterThan
+	GreaterOrEqual
+	LessThan
+	LessOrEqual
+
+	Uint16 LayoutType = 0
+	Uint32            = 1
+
+	Unsorted SortType = 0
+	Ascending
+	Descending
+)
+
+type DataFile struct {
+	xbuf      []byte
+	ybuf      []byte
+	xReaderAt io.ReaderAt
+	xLen      int
+	yReaderAt io.ReaderAt
+	yLen      int
+}
+
+type DataLayout struct {
+	LayoutType
+	SortType
+}
+
+type FieldName string
+
+func (d *DataFile) ReadRows(field FieldName, layout map[FieldName]DataLayout, op Operator, operand Datum) (rows []*Row, err error) {
+	if field == "x" {
+		rows, err = pushDownReadRows(d.xReaderAt, d.xLen, layout[field], op, operand)
+		if err != nil {
+			return
+		}
+		err = hydrateRows(d.yReaderAt, d.yLen, layout["y"], rows)
+	}
+	if field == "y" {
+		rows, err = pushDownReadRows(d.yReaderAt, d.yLen, layout[field], op, operand)
+		if err != nil {
+			return
+		}
+		err = hydrateRows(d.xReaderAt, d.xLen, layout["x"], rows)
+	}
+	return
+}
+
+type Row struct {
+	index int
+	Datums
+}
+
+func pushDownReadRows(readerAt io.ReaderAt, dataLen int, layout DataLayout, op Operator, operand Datum) (rows []*Row, err error) {
+	if layout.LayoutType == Uint16 {
+		if layout.SortType == Unsorted {
+			buf := make([]byte, SIZE_Uint16)
+			for i := 0; i < dataLen; i += SIZE_Uint16 {
+				if n, err := readerAt.ReadAt(buf, int64(i)); n == SIZE_Uint16 && err == nil {
+					d := NewDUint16(DUint16(binary.BigEndian.Uint16(buf)))
+					cmp, err := d.Compare(operand)
+					if err != nil {
+						return rows, err
+					}
+					if cmp == 0 && op == Equal {
+						println(1)
+						rows = append(rows, &Row{
+							index:  i / SIZE_Uint16,
+							Datums: []Datum{d},
+						})
+					}
+				}
+			}
+		}
+	}
+	if layout.LayoutType == Uint32 {
+		if layout.SortType == Unsorted {
+			buf := make([]byte, SIZE_Uint32)
+			for i := 0; i < dataLen; i += SIZE_Uint32 {
+				if n, err := readerAt.ReadAt(buf, int64(i)); n == SIZE_Uint32 && err == nil {
+					d := NewDUint32(DUint32(binary.BigEndian.Uint32(buf)))
+					cmp, err := d.Compare(operand)
+					if err != nil {
+						return rows, err
+					}
+					if cmp == 0 && op == Equal {
+						println(2)
+						rows = append(rows, &Row{
+							index:  i / SIZE_Uint32,
+							Datums: []Datum{d},
+						})
+					}
+				}
+			}
+		}
+	}
+	return
+}
+
+func hydrateRows(readerAt io.ReaderAt, dataLen int, layout DataLayout, rows []*Row) (err error) {
+	if layout.LayoutType == Uint16 {
+		if layout.SortType == Unsorted {
+			buf := make([]byte, SIZE_Uint16)
+			for _, row := range rows {
+				if n, err := readerAt.ReadAt(buf, int64(row.index)*SIZE_Uint16); n == SIZE_Uint16 && err == nil {
+					t := binary.BigEndian.Uint16(buf)
+					d := NewDUint16(DUint16(t))
+					println(3, "add", t)
+					row.Datums = append(row.Datums, d)
+				}
+			}
+		}
+	}
+	if layout.LayoutType == Uint32 {
+		if layout.SortType == Unsorted {
+			buf := make([]byte, SIZE_Uint32)
+			for _, row := range rows {
+				if n, err := readerAt.ReadAt(buf, int64(row.index)*SIZE_Uint32); n == SIZE_Uint32 && err == nil {
+					t := binary.BigEndian.Uint32(buf)
+					d := NewDUint32(DUint32(t))
+					println(4, "add", t)
+					row.Datums = append(row.Datums, d)
+				}
+			}
+		}
+	}
+	return
+}