From 83d3bf280002c88a7d72f79ebede25a83aba4d89 Mon Sep 17 00:00:00 2001 From: Ingo Oppermann Date: Fri, 15 Nov 2024 10:18:40 +0100 Subject: [PATCH] Add support for HTTP range requests --- http/handler/filesystem.go | 152 ++++++++++++++++++++++++++++++++- http/middleware/cache/cache.go | 5 ++ http/middleware/session/HLS.go | 4 + io/fs/disk.go | 4 + io/fs/fs.go | 2 +- io/fs/mem.go | 26 ++++-- io/fs/mem_storage.go | 3 +- io/fs/s3.go | 6 +- 8 files changed, 192 insertions(+), 10 deletions(-) diff --git a/http/handler/filesystem.go b/http/handler/filesystem.go index 9e00e37c..075e3232 100644 --- a/http/handler/filesystem.go +++ b/http/handler/filesystem.go @@ -1,10 +1,16 @@ package handler import ( + "errors" + "fmt" + "io" "net/http" + "net/textproto" "path/filepath" "sort" + "strconv" "strings" + "time" "github.com/datarhei/core/v16/http/api" "github.com/datarhei/core/v16/http/fs" @@ -68,12 +74,56 @@ func (h *FSHandler) GetFile(c echo.Context) error { } c.Response().Header().Set(echo.HeaderContentType, mimeType) + c.Response().Header().Set("Accept-Ranges", "bytes") if c.Request().Method == "HEAD" { + c.Response().Header().Set(echo.HeaderContentLength, strconv.FormatInt(stat.Size(), 10)) return c.Blob(http.StatusOK, "application/data", nil) } - return c.Stream(http.StatusOK, "application/data", file) + var streamFile io.Reader = file + status := http.StatusOK + + ifRange := c.Request().Header.Get("If-Range") + if len(ifRange) != 0 { + ifTime, err := time.Parse("Mon, 02 Jan 2006 15:04:05 MST", ifRange) + if err != nil { + return api.Err(http.StatusBadRequest, "", "%s", err) + } + + if ifTime.Unix() != stat.ModTime().Unix() { + c.Request().Header.Del("Range") + } + } + + byteRange := c.Request().Header.Get("Range") + if len(byteRange) != 0 { + ranges, err := parseRange(byteRange, stat.Size()) + if err != nil { + return api.Err(http.StatusRequestedRangeNotSatisfiable, "", "%s", err.Error()) + } + + if len(ranges) > 1 { + return api.Err(http.StatusNotImplemented, "", "multipart range requests are not supported") + } + + if len(ranges) == 1 { + _, err := file.Seek(ranges[0].start, io.SeekStart) + if err != nil { + return api.Err(http.StatusRequestedRangeNotSatisfiable, "", "%s", err.Error()) + } + + c.Response().Header().Set("Content-Range", ranges[0].contentRange(stat.Size())) + streamFile = &io.LimitedReader{ + R: streamFile, + N: ranges[0].length, + } + + status = http.StatusPartialContent + } + } + + return c.Stream(status, "application/data", streamFile) } func (h *FSHandler) PutFile(c echo.Context) error { @@ -177,3 +227,103 @@ func (h *FSHandler) ListFiles(c echo.Context) error { return c.JSON(http.StatusOK, fileinfos) } + +// From: github.com/golang/go/net/http/fs.go@7dc9fcb + +// errNoOverlap is returned by serveContent's parseRange if first-byte-pos of +// all of the byte-range-spec values is greater than the content size. +var errNoOverlap = errors.New("invalid range: failed to overlap") + +// httpRange specifies the byte range to be sent to the client. +type httpRange struct { + start, length int64 +} + +func (r httpRange) contentRange(size int64) string { + return fmt.Sprintf("bytes %d-%d/%d", r.start, r.start+r.length-1, size) +} + +/* +func (r httpRange) mimeHeader(contentType string, size int64) textproto.MIMEHeader { + return textproto.MIMEHeader{ + "Content-Range": {r.contentRange(size)}, + "Content-Type": {contentType}, + } +} +*/ + +// parseRange parses a Range header string as per RFC 7233. +// errNoOverlap is returned if none of the ranges overlap. +func parseRange(s string, size int64) ([]httpRange, error) { + if s == "" { + return nil, nil // header not present + } + const b = "bytes=" + if !strings.HasPrefix(s, b) { + return nil, errors.New("invalid range") + } + var ranges []httpRange + noOverlap := false + for _, ra := range strings.Split(s[len(b):], ",") { + ra = textproto.TrimString(ra) + if ra == "" { + continue + } + start, end, ok := strings.Cut(ra, "-") + if !ok { + return nil, errors.New("invalid range") + } + start, end = textproto.TrimString(start), textproto.TrimString(end) + var r httpRange + if start == "" { + // If no start is specified, end specifies the + // range start relative to the end of the file, + // and we are dealing with + // which has to be a non-negative integer as per + // RFC 7233 Section 2.1 "Byte-Ranges". + if end == "" || end[0] == '-' { + return nil, errors.New("invalid range") + } + i, err := strconv.ParseInt(end, 10, 64) + if i < 0 || err != nil { + return nil, errors.New("invalid range") + } + if i > size { + i = size + } + r.start = size - i + r.length = size - r.start + } else { + i, err := strconv.ParseInt(start, 10, 64) + if err != nil || i < 0 { + return nil, errors.New("invalid range") + } + if i >= size { + // If the range begins after the size of the content, + // then it does not overlap. + noOverlap = true + continue + } + r.start = i + if end == "" { + // If no end is specified, range extends to end of the file. + r.length = size - r.start + } else { + i, err := strconv.ParseInt(end, 10, 64) + if err != nil || r.start > i { + return nil, errors.New("invalid range") + } + if i >= size { + i = size - 1 + } + r.length = i - r.start + 1 + } + } + ranges = append(ranges, r) + } + if noOverlap && len(ranges) == 0 { + // The specified ranges did not overlap with the content. + return nil, errNoOverlap + } + return ranges, nil +} diff --git a/http/middleware/cache/cache.go b/http/middleware/cache/cache.go index a32e35e7..3b25fc17 100644 --- a/http/middleware/cache/cache.go +++ b/http/middleware/cache/cache.go @@ -67,6 +67,11 @@ func NewWithConfig(config Config) echo.MiddlewareFunc { return next(c) } + if len(req.Header.Get("Range")) != 0 { + res.Header().Set("X-Cache", "SKIP RANGEREQ") + return next(c) + } + if obj, expireIn, _ := config.Cache.Get(key); obj == nil { // cache miss writer := res.Writer diff --git a/http/middleware/session/HLS.go b/http/middleware/session/HLS.go index d475e14d..4e40e443 100644 --- a/http/middleware/session/HLS.go +++ b/http/middleware/session/HLS.go @@ -219,6 +219,10 @@ func (h *hls) handleEgress(c echo.Context, next echo.HandlerFunc) error { } } + // Remove any Range request headers, because the rewrite will mess up any lengths. + req.Header.Del("Range") + req.Header.Del("If-Range") + rewrite = true } diff --git a/io/fs/disk.go b/io/fs/disk.go index 88352c72..b6d352d1 100644 --- a/io/fs/disk.go +++ b/io/fs/disk.go @@ -121,6 +121,10 @@ func (f *diskFile) Read(p []byte) (int, error) { return f.file.Read(p) } +func (f *diskFile) Seek(offset int64, whence int) (int64, error) { + return f.file.Seek(offset, whence) +} + // diskFilesystem implements the Filesystem interface type diskFilesystem struct { metadata map[string]string diff --git a/io/fs/fs.go b/io/fs/fs.go index 9f3b8661..d7102783 100644 --- a/io/fs/fs.go +++ b/io/fs/fs.go @@ -31,7 +31,7 @@ type FileInfo interface { // File provides access to a single file. type File interface { - io.ReadCloser + io.ReadSeekCloser // Name returns the Name of the file. Name() string diff --git a/io/fs/mem.go b/io/fs/mem.go index d0bcba63..01c3e3dc 100644 --- a/io/fs/mem.go +++ b/io/fs/mem.go @@ -67,6 +67,7 @@ func (f *memFileInfo) IsDir() bool { type memFile struct { memFileInfo data *bytes.Buffer // Contents of the file + r io.ReadSeeker } func (f *memFile) Name() string { @@ -86,21 +87,33 @@ func (f *memFile) Stat() (FileInfo, error) { } func (f *memFile) Read(p []byte) (int, error) { - if f.data == nil { + if f.r == nil { return 0, io.EOF } - return f.data.Read(p) + return f.r.Read(p) +} + +func (f memFile) Seek(offset int64, whence int) (int64, error) { + if f.r == nil { + return 0, io.EOF + } + + return f.r.Seek(offset, whence) } func (f *memFile) Close() error { - if f.data == nil { - return io.EOF + var err error = nil + + if f.r == nil { + err = io.EOF } + f.r = nil + f.data = nil - return nil + return err } type memFilesystem struct { @@ -258,7 +271,8 @@ func (fs *memFilesystem) Open(path string) File { if file.data != nil { newFile.lastMod = file.lastMod - newFile.data = bytes.NewBuffer(file.data.Bytes()) + newFile.data = file.data + newFile.r = bytes.NewReader(file.data.Bytes()) newFile.size = int64(newFile.data.Len()) } diff --git a/io/fs/mem_storage.go b/io/fs/mem_storage.go index a7bbc2b7..191ed71b 100644 --- a/io/fs/mem_storage.go +++ b/io/fs/mem_storage.go @@ -61,7 +61,8 @@ func (m *memStorage) LoadAndCopy(key string) (*memFile, bool) { } if v.data != nil { - f.data = bytes.NewBuffer(v.data.Bytes()) + f.data = &bytes.Buffer{} + f.data.Write(v.data.Bytes()) } return f, true diff --git a/io/fs/s3.go b/io/fs/s3.go index 22c66d05..298e905c 100644 --- a/io/fs/s3.go +++ b/io/fs/s3.go @@ -622,7 +622,7 @@ func (f *s3FileInfo) IsDir() bool { } type s3File struct { - data io.ReadCloser + data io.ReadSeekCloser name string size int64 lastModified time.Time @@ -632,6 +632,10 @@ func (f *s3File) Read(p []byte) (int, error) { return f.data.Read(p) } +func (f *s3File) Seek(offset int64, whence int) (int64, error) { + return f.data.Seek(offset, whence) +} + func (f *s3File) Close() error { return f.data.Close() }