4 files changed, 119 insertions, 8 deletions
diff --git a/weed/util/bounded_tree/bounded_tree.go b/weed/util/bounded_tree/bounded_tree.go
index 0e8af2520..3a8a22a9c 100644
--- a/weed/util/bounded_tree/bounded_tree.go
+++ b/weed/util/bounded_tree/bounded_tree.go
@@ -70,12 +70,7 @@ func (t *BoundedTree) ensureVisited(n *Node, currentPath util.FullPath, componen
 	} else {
 		// fmt.Printf("ensure %v\n", currentPath)
 
-		filerPath := currentPath
-		if t.baseDir != "/" {
-			filerPath = t.baseDir + filerPath
-		}
-
-		children, err := visitFn(filerPath)
+		children, err := visitFn(currentPath)
 		if err != nil {
 			glog.V(0).Infof("failed to visit %s: %v", currentPath, err)
 			return false, err
diff --git a/weed/util/constants.go b/weed/util/constants.go
index 6001ae78e..1900d3d42 100644
--- a/weed/util/constants.go
+++ b/weed/util/constants.go
@@ -5,7 +5,7 @@ import (
 )
 
 var (
-	VERSION = fmt.Sprintf("%s %d.%02d", sizeLimit, 2, 24)
+	VERSION = fmt.Sprintf("%s %d.%02d", sizeLimit, 2, 26)
 	COMMIT  = ""
 )
 
diff --git a/weed/util/fasthttp_util.go b/weed/util/fasthttp_util.go
new file mode 100644
index 000000000..82575af98
--- /dev/null
+++ b/weed/util/fasthttp_util.go
@@ -0,0 +1,115 @@
+package util
+
+import (
+	"bytes"
+	"fmt"
+	"github.com/valyala/fasthttp"
+	"sync"
+	"time"
+)
+
+var (
+	fastClient = &fasthttp.Client{
+		NoDefaultUserAgentHeader:      true, // Don't send: User-Agent: fasthttp
+		MaxConnsPerHost:               1024,
+		ReadBufferSize:                4096,      // Make sure to set this big enough that your whole request can be read at once.
+		WriteBufferSize:               64 * 1024, // Same but for your response.
+		ReadTimeout:                   time.Second,
+		WriteTimeout:                  time.Second,
+		MaxIdleConnDuration:           time.Minute,
+		DisableHeaderNamesNormalizing: true, // If you set the case on your headers correctly you can enable this.
+	}
+
+	// Put everything in pools to prevent garbage.
+	bytesPool = sync.Pool{
+		New: func() interface{} {
+			b := make([]byte, 0)
+			return &b
+		},
+	}
+
+	responsePool = sync.Pool{
+		New: func() interface{} {
+			return make(chan *fasthttp.Response)
+		},
+	}
+)
+
+func FastGet(url string) ([]byte, bool, error) {
+
+	req := fasthttp.AcquireRequest()
+	res := fasthttp.AcquireResponse()
+	defer fasthttp.ReleaseRequest(req)
+	defer fasthttp.ReleaseResponse(res)
+
+	req.SetRequestURIBytes([]byte(url))
+	req.Header.Add("Accept-Encoding", "gzip")
+
+	err := fastClient.Do(req, res)
+	if err != nil {
+		return nil, true, err
+	}
+
+	var data []byte
+	contentEncoding := res.Header.Peek("Content-Encoding")
+	if bytes.Compare(contentEncoding, []byte("gzip")) == 0 {
+		data, err = res.BodyGunzip()
+	} else {
+		data = res.Body()
+	}
+
+	out := make([]byte, len(data))
+	copy(out, data)
+
+	if res.StatusCode() >= 400 {
+		retryable := res.StatusCode() >= 500
+		return nil, retryable, fmt.Errorf("%s: %d", url, res.StatusCode())
+	}
+	if err != nil {
+		return nil, false, err
+	}
+	return out, false, nil
+}
+
+func FastReadUrlAsStream(fileUrl string, cipherKey []byte, isContentGzipped bool, isFullChunk bool, offset int64, size int, fn func(data []byte)) (retryable bool, err error) {
+
+	if cipherKey != nil {
+		return readEncryptedUrl(fileUrl, cipherKey, isContentGzipped, isFullChunk, offset, size, fn)
+	}
+
+	req := fasthttp.AcquireRequest()
+	res := fasthttp.AcquireResponse()
+	defer fasthttp.ReleaseRequest(req)
+	defer fasthttp.ReleaseResponse(res)
+
+	req.SetRequestURIBytes([]byte(fileUrl))
+
+	if isFullChunk {
+		req.Header.Add("Accept-Encoding", "gzip")
+	} else {
+		req.Header.Add("Range", fmt.Sprintf("bytes=%d-%d", offset, offset+int64(size)-1))
+	}
+
+	if err = fastClient.Do(req, res); err != nil {
+		return true, err
+	}
+
+	if res.StatusCode() >= 400 {
+		retryable = res.StatusCode() >= 500
+		return retryable, fmt.Errorf("%s: %d", fileUrl, res.StatusCode())
+	}
+
+	contentEncoding := res.Header.Peek("Content-Encoding")
+	if bytes.Compare(contentEncoding, []byte("gzip")) == 0 {
+		bodyData, err := res.BodyGunzip()
+		if err != nil {
+			return false, err
+		}
+		fn(bodyData)
+	} else {
+		fn(res.Body())
+	}
+
+	return false, nil
+
+}
diff --git a/weed/util/http_util.go b/weed/util/http_util.go
index 2cd69b01b..eff282bab 100644
--- a/weed/util/http_util.go
+++ b/weed/util/http_util.go
@@ -21,6 +21,7 @@ var (
 
 func init() {
 	Transport = &http.Transport{
+		MaxIdleConns:        1024,
 		MaxIdleConnsPerHost: 1024,
 	}
 	client = &http.Client{
@@ -312,7 +313,7 @@ func ReadUrlAsStream(fileUrl string, cipherKey []byte, isContentGzipped bool, is
 }
 
 func readEncryptedUrl(fileUrl string, cipherKey []byte, isContentCompressed bool, isFullChunk bool, offset int64, size int, fn func(data []byte)) (bool, error) {
-	encryptedData, retryable, err := Get(fileUrl)
+	encryptedData, retryable, err := FastGet(fileUrl)
 	if err != nil {
 		return retryable, fmt.Errorf("fetch %s: %v", fileUrl, err)
 	}