public/cache-proxy
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

remove logrus and unused deps

Browse Source
This commit is contained in:
guochao
2024-12-25 11:00:02 +08:00
parent a73321aa04
commit 018c07d944
4 changed files with 127 additions and 396 deletions
Download Patch File Download Diff File Expand all files Collapse all files

175
cmd/proxy/main.go
View File

@ -6,6 +6,7 @@ import (
"flag"
"fmt"
"io"
"log/slog"
"net/http"
"os"
"path/filepath"
@ -17,7 +18,6 @@ import (
"github.com/getsentry/sentry-go"
"github.com/sirupsen/logrus"
"gopkg.in/yaml.v3"
)
@ -45,7 +45,8 @@ func (upstream Upstream) GetPath(orig string) (string, bool, error) {
}
type LocalStorage struct {
Path string `yaml:"path"`
Path string `yaml:"path"`
TemporaryFilePattern string `yaml:"temporary-file-pattern"`
}
type Accel struct {
@ -118,11 +119,11 @@ OUTER:
offset += written
}
time.Sleep(time.Millisecond)
logrus.WithFields(logrus.Fields{
"start": offset,
"end": memoryObject.Buffer.Len(),
"n": memoryObject.Buffer.Len() - offset,
}).Trace("remain bytes")
slog.With(
"start", offset,
"end", memoryObject.Buffer.Len(),
"n", memoryObject.Buffer.Len()-offset,
).Debug("remain bytes")
_, err := w.Write(memoryObject.Buffer.Bytes()[offset:])
return err
@ -156,7 +157,8 @@ func configFromFile(path string) (*Config, error) {
Storage: Storage{
Type: "local",
Local: &LocalStorage{
Path: "./data",
Path: "./data",
TemporaryFilePattern: "temp.*",
},
Accel: Accel{
ResponseWithHeaders: []string{"X-Sendfile", "X-Accel-Redirect"},
@ -315,17 +317,14 @@ func (server *Server) streamOnline(w http.ResponseWriter, r *http.Request, mtime
}
if err := memoryObject.StreamTo(w, memoryObject.wg); err != nil {
logrus.WithError(err).Warn("failed to stream response with existing memory object")
slog.With("error", err).Warn("failed to stream response with existing memory object")
}
}
} else {
logrus.WithField("mtime", mtime).Trace("checking fastest upstream")
slog.With("mtime", mtime).Debug("checking fastest upstream")
selectedIdx, response, chunks, err := server.fastesUpstream(r, mtime)
logrus.WithFields(logrus.Fields{
"upstreamIdx": selectedIdx,
}).Trace("fastest upstream")
if chunks == nil && mtime != zeroTime {
logrus.WithFields(logrus.Fields{"upstreamIdx": selectedIdx, "key": key}).Trace("not modified. using local version")
slog.With("upstreamIdx", selectedIdx, "key", key).Debug("not modified. using local version")
if w != nil {
server.serveFile(w, r, key)
}
@ -333,22 +332,26 @@ func (server *Server) streamOnline(w http.ResponseWriter, r *http.Request, mtime
}
if err != nil {
logrus.WithError(err).Warn("failed to select fastest upstream")
slog.With("error", err).Warn("failed to select fastest upstream")
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
if selectedIdx == -1 || response == nil || chunks == nil {
logrus.Trace("no upstream is selected")
slog.Debug("no upstream is selected")
http.NotFound(w, r)
return
}
if response.StatusCode == http.StatusNotModified {
logrus.WithField("upstreamIdx", selectedIdx).Trace("not modified. using local version")
slog.With("upstreamIdx", selectedIdx).Debug("not modified. using local version")
os.Chtimes(key, zeroTime, time.Now())
server.serveFile(w, r, key)
return
}
slog.With(
"upstreamIdx", selectedIdx,
).Debug("found fastest upstream")
buffer := &bytes.Buffer{}
ctx, cancel := context.WithCancel(r.Context())
defer cancel()
@ -382,7 +385,7 @@ func (server *Server) streamOnline(w http.ResponseWriter, r *http.Request, mtime
for chunk := range chunks {
if chunk.error != nil {
err = chunk.error
logrus.WithError(err).Warn("failed to read from upstream")
slog.With("error", err).Warn("failed to read from upstream")
}
if chunk.buffer == nil {
break
@ -395,41 +398,69 @@ func (server *Server) streamOnline(w http.ResponseWriter, r *http.Request, mtime
memoryObject.wg.Wait()
if err != nil {
logrus.WithError(err).WithField("upstreamIdx", selectedIdx).Error("something happened during download. will not cache this response")
slog.With("error", err, "upstreamIdx", selectedIdx).Error("something happened during download. will not cache this response")
}
go func() {
defer func() {
server.lu.Lock()
defer server.lu.Unlock()
delete(server.o, r.URL.Path)
slog.Debug("memory object released")
}()
if err == nil {
logrus.Trace("preparing to release memory object")
slog.Debug("preparing to release memory object")
mtime := zeroTime
lastModifiedHeader := response.Header.Get("Last-Modified")
if lastModified, err := time.Parse(time.RFC1123, lastModifiedHeader); err != nil {
logrus.WithError(err).WithFields(logrus.Fields{
"value": lastModifiedHeader,
"url": response.Request.URL,
}).Trace("failed to parse last modified header value")
slog.With(
"error", err,
"value", lastModifiedHeader,
"url", response.Request.URL,
).Debug("failed to parse last modified header value. set modified time to now")
} else {
slog.With(
"header", lastModifiedHeader,
"value", lastModified,
"url", response.Request.URL,
).Debug("found modified time")
mtime = lastModified
}
if err := os.MkdirAll(server.Storage.Local.Path, 0755); err != nil {
logrus.Warn(err)
slog.With("error", err).Warn("failed to create local storage path")
}
fp, err := os.CreateTemp(server.Storage.Local.Path, "temp.*")
name := fp.Name()
if server.Config.Storage.Local.TemporaryFilePattern == "" {
if err := os.WriteFile(key, buffer.Bytes(), 0644); err != nil {
slog.With("error", err).Warn("failed to write file")
os.Remove(key)
}
return
}
fp, err := os.CreateTemp(server.Storage.Local.Path, server.Storage.Local.TemporaryFilePattern)
if err != nil {
logrus.WithFields(logrus.Fields{
"key": key,
"path": server.Storage.Local.Path,
"pattern": "temp.*",
}).WithError(err).Warn("ftime.Time{}ailed to create template file")
} else if _, err := fp.Write(buffer.Bytes()); err != nil {
slog.With(
"key", key,
"path", server.Storage.Local.Path,
"pattern", server.Storage.Local.TemporaryFilePattern,
"error", err,
).Warn("failed to create template file")
return
}
name := fp.Name()
if _, err := fp.Write(buffer.Bytes()); err != nil {
fp.Close()
os.Remove(name)
logrus.WithError(err).Warn("failed to write into template file")
slog.With("error", err).Warn("failed to write into template file")
} else if err := fp.Close(); err != nil {
os.Remove(name)
logrus.WithError(err).Warn("failed to close template file")
slog.With("error", err).Warn("failed to close template file")
} else {
os.Chtimes(name, zeroTime, mtime)
dirname := filepath.Dir(key)
@ -438,12 +469,6 @@ func (server *Server) streamOnline(w http.ResponseWriter, r *http.Request, mtime
os.Rename(name, key)
}
}
server.lu.Lock()
defer server.lu.Unlock()
delete(server.o, r.URL.Path)
logrus.Trace("memory object released")
}()
}
@ -457,7 +482,6 @@ func (server *Server) fastesUpstream(r *http.Request, lastModified time.Time) (r
selectedOnce := &sync.Once{}
wg := &sync.WaitGroup{}
wg.Add(len(server.Upstreams))
logrus.WithField("size", len(server.Upstreams)).Trace("wg")
defer close(selectedCh)
for idx := range server.Upstreams {
@ -465,19 +489,19 @@ func (server *Server) fastesUpstream(r *http.Request, lastModified time.Time) (r
ctx, cancel := context.WithCancel(context.Background())
cancelFuncs[idx] = cancel
logger := logrus.WithField("upstreamIdx", idx)
logger := slog.With("upstreamIdx", idx)
go func() {
defer wg.Done()
response, ch, err := server.tryUpstream(ctx, idx, r, lastModified)
if err == context.Canceled { // others returned
logger.Trace("context canceled")
logger.Debug("context canceled")
return
}
if err != nil {
if err != context.Canceled && err != context.DeadlineExceeded {
logger.WithError(err).Warn("upstream has error")
logger.With("error", err).Warn("upstream has error")
}
return
}
@ -496,24 +520,22 @@ func (server *Server) fastesUpstream(r *http.Request, lastModified time.Time) (r
for cancelIdx, cancel := range cancelFuncs {
if cancelIdx == idx {
logrus.WithField("upstreamIdx", cancelIdx).Trace("selected thus not canceled")
slog.With("upstreamIdx", cancelIdx).Debug("selected thus not canceled")
continue
}
logrus.WithField("upstreamIdx", cancelIdx).Trace("not selected and thus canceled")
slog.With("upstreamIdx", cancelIdx).Debug("not selected and thus canceled")
cancel()
}
logger.Trace("upstream is selected")
logger.Debug("upstream is selected")
})
logger.Trace("voted")
return
logger.Debug("voted")
}()
}
wg.Wait()
logrus.Trace("all upstream tried")
slog.Debug("all upstream returned")
resultIdx = -1
select {
@ -528,13 +550,13 @@ func (server *Server) fastesUpstream(r *http.Request, lastModified time.Time) (r
func (server *Server) tryUpstream(ctx context.Context, upstreamIdx int, r *http.Request, lastModified time.Time) (response *http.Response, chunks chan Chunk, err error) {
upstream := server.Upstreams[upstreamIdx]
logger := logrus.WithField("upstreamIdx", upstreamIdx)
logger := slog.With("upstreamIdx", upstreamIdx)
newpath, matched, err := upstream.GetPath(r.URL.Path)
logger.WithFields(logrus.Fields{
"path": newpath,
"matched": matched,
}).Trace("trying upstream")
logger.With(
"path", newpath,
"matched", matched,
).Debug("trying upstream")
if err != nil {
return nil, nil, err
}
@ -552,9 +574,9 @@ func (server *Server) tryUpstream(ctx context.Context, upstreamIdx int, r *http.
return nil, nil, err
}
if lastModified != zeroTime {
logger.WithFields(logrus.Fields{
"mtime": lastModified.Format(time.RFC1123),
}).Trace("check modified since")
logger.With(
"mtime", lastModified.Format(time.RFC1123),
).Debug("check modified since")
request.Header.Set("If-Modified-Since", lastModified.Format(time.RFC1123))
}
@ -567,7 +589,7 @@ func (server *Server) tryUpstream(ctx context.Context, upstreamIdx int, r *http.
if err != nil {
return nil, nil, err
}
logrus.WithField("status", response.StatusCode).Trace("responded")
slog.With("status", response.StatusCode).Debug("responded")
if response.StatusCode == http.StatusNotModified {
return response, nil, nil
}
@ -575,10 +597,10 @@ func (server *Server) tryUpstream(ctx context.Context, upstreamIdx int, r *http.
return nil, nil, nil
}
if response.StatusCode < 200 || response.StatusCode >= 500 {
logrus.WithFields(logrus.Fields{
"url": newurl,
"status": response.StatusCode,
}).Warn("unexpected status")
slog.With(
"url", newurl,
"status", response.StatusCode,
).Warn("unexpected status")
return response, nil, fmt.Errorf("unexpected status(url=%v): %v: %v", newurl, response.StatusCode, response)
}
@ -587,7 +609,6 @@ func (server *Server) tryUpstream(ctx context.Context, upstreamIdx int, r *http.
ch := make(chan Chunk, 1024)
buffer := make([]byte, server.Misc.FirstChunkBytes)
start := time.Now()
n, err := io.ReadAtLeast(response.Body, buffer, len(buffer))
if err != nil {
@ -595,7 +616,6 @@ func (server *Server) tryUpstream(ctx context.Context, upstreamIdx int, r *http.
return response, nil, err
}
}
logger.WithField("duration", time.Now().Sub(start)).Tracef("first %v bytes", n)
ch <- Chunk{buffer: buffer[:n]}
go func() {
@ -609,7 +629,7 @@ func (server *Server) tryUpstream(ctx context.Context, upstreamIdx int, r *http.
currentOffset += int64(n)
}
if response.ContentLength > 0 && currentOffset == response.ContentLength && err == io.EOF || err == io.ErrUnexpectedEOF {
logger.Trace("done")
logger.Debug("done")
return
}
if err != nil {
@ -646,26 +666,25 @@ func init() {
func main() {
flag.Parse()
if lvl, err := logrus.ParseLevel(logLevel); err != nil {
logrus.WithError(err).Panic("failed to parse log level")
lvl := slog.LevelInfo
if err := lvl.UnmarshalText([]byte(logLevel)); err != nil {
slog.With("error", err).Error("failed to parse log level")
os.Exit(-1)
} else {
logrus.SetLevel(lvl)
slog.SetLogLoggerLevel(lvl)
}
if sentrydsn != "" {
if err := sentry.Init(sentry.ClientOptions{
Dsn: sentrydsn,
}); err != nil {
logrus.WithField("dsn", sentrydsn).WithError(err).Panic("failed to setup sentry")
slog.With("dsn", sentrydsn, "error", err).Error("failed to setup sentry")
os.Exit(-1)
}
defer sentry.Flush(time.Second * 3)
}
logrus.SetFormatter(&logrus.TextFormatter{
FullTimestamp: true,
TimestampFormat: "2006-01-02T15:04:05.000",
})
config, err := configFromFile(configFilePath)
if err != nil {
panic(err)
@ -684,6 +703,6 @@ func main() {
}
http.HandleFunc("GET /{path...}", server.handleRequest)
logrus.WithFields(logrus.Fields{"addr": ":8881"}).Info("serving app")
slog.With("addr", ":8881").Info("serving app")
http.ListenAndServe(":8881", nil)
}