diff options
Diffstat (limited to 'weed/filesys')
| -rw-r--r-- | weed/filesys/dirty_page.go | 8 | ||||
| -rw-r--r-- | weed/filesys/wfs.go | 4 |
2 files changed, 10 insertions, 2 deletions
diff --git a/weed/filesys/dirty_page.go b/weed/filesys/dirty_page.go index da442d6c6..a78fdb47d 100644 --- a/weed/filesys/dirty_page.go +++ b/weed/filesys/dirty_page.go @@ -59,7 +59,13 @@ func (pages *ContinuousDirtyPages) AddPage(ctx context.Context, offset int64, da Offset: offset, Data: data, }) - return nil, nil + + if pages.totalSize() >= pages.f.wfs.chunkSizeLimit { + chunk, err = pages.saveToStorage(ctx) + pages.pages = nil + glog.V(3).Infof("%s/%s add split [%d,%d)", pages.f.dir.Path, pages.f.Name, chunk.Offset, chunk.Offset+int64(chunk.Size)) + } + return } chunk, err = pages.saveToStorage(ctx) diff --git a/weed/filesys/wfs.go b/weed/filesys/wfs.go index b9cb0210b..4b9e20b95 100644 --- a/weed/filesys/wfs.go +++ b/weed/filesys/wfs.go @@ -13,14 +13,16 @@ type WFS struct { listDirectoryEntriesCache *ccache.Cache collection string replication string + chunkSizeLimit int64 } -func NewSeaweedFileSystem(filer string, collection string, replication string) *WFS { +func NewSeaweedFileSystem(filer string, collection string, replication string, chunkSizeLimitMB int) *WFS { return &WFS{ filer: filer, listDirectoryEntriesCache: ccache.New(ccache.Configure().MaxSize(6000).ItemsToPrune(100)), collection: collection, replication: replication, + chunkSizeLimit: int64(chunkSizeLimitMB) * 1024 * 1024, } } |