move x-accel into local storage configuration

.gitea/workflows/container-build.yaml

@@ -3,22 +3,25 @@ name: build container
 run-name: build container on ${{ gitea.actor }}
 on: [push]
 
-env:
-  GOPROXY: ${{ vars.GOPROXY }}
-
 jobs:
   build-container:
-    runs-on: bookworm
+    runs-on: docker
     steps:
     - name: Setup apt mirror
       run: sed -i "s,deb.debian.org,${{ vars.DEBIAN_MIRROR }},g" ${{ vars.DEBIAN_APT_SOURCES }}
       if:  ${{ vars.DEBIAN_MIRROR && vars.DEBIAN_APT_SOURCES }}
     - name: Setup debian environment
       run: apt update && apt install -y podman podman-compose nodejs
+    - name: Setup cache for podman
+      uses: actions/cache@v4
+      with:
+        path: |
+          /var/lib/containers
+        key: podman-storage
     - name: Check out repository code
       uses: actions/checkout@v4
     - name: Build container
-      run: REGISTRY=${{ github.server_url}}; REGISTRY=${REGISTRY##https://}; REGISTRY=${REGISTRY##http://}; podman build -t $REGISTRY/${{ github.repository }} .
+      run: REGISTRY=${{ github.server_url}}; REGISTRY=${REGISTRY##https://}; REGISTRY=${REGISTRY##http://}; podman build --build-arg GOPROXY=${{ vars.GOPROXY }} -t $REGISTRY/${{ github.repository }} .
     - name: Login to Container Registry
       run:  echo "${{ secrets.ACTION_PACKAGE_WRITE_TOKEN }}" | podman login ${{ github.server_url }} -u ${{ github.repository_owner }} --password-stdin
     - name: Push Container Image

.gitea/workflows/go-test.yaml

@@ -7,22 +7,33 @@ on:
       - go.mod
       - go.lock
       - go.work
+      - .gitea/workflows/go-test.yaml
 env:
   GOPROXY: ${{ vars.GOPROXY }}
 
 jobs:
   test:
-    runs-on: bookworm
+    runs-on: docker
     steps:
     - name: Setup apt mirror
       run:    sed -i "s,deb.debian.org,${{ vars.DEBIAN_MIRROR }},g" ${{ vars.DEBIAN_APT_SOURCES }}
       if:   ${{ vars.DEBIAN_MIRROR && vars.DEBIAN_APT_SOURCES }}
     - name: Setup Debian environment
       run:    apt update && apt install -y nodejs
+    - name: Setup cache for golang toolchain
+      uses: actions/cache@v4
+      with:
+        path: |
+          /opt/hostedtoolcache/go
+          /root/go/pkg/mod
+          /root/.cache/go-build
+        key: ${{ runner.os }}-golang
     - name: Setup Golang
       uses:   actions/setup-go@v5
       with:
         go-version: 1.24.0
+        cache-dependency-path: |
+          go.sum
     - name: Check out repository code
       uses:   actions/checkout@v4
     - name: Run tests

.gitignore

@@ -2,3 +2,6 @@
 
 data
 __debug*
+
+.env
+.envrc

Dockerfile

@@ -1,10 +1,12 @@
-FROM docker.io/library/golang:1.23-alpine
-ARG  GOPROXY
+FROM docker.io/library/golang:1.24-alpine
+ARG  GOPROXY=direct
+ARG  TAGS=""
+ARG  ALPINE_MIRROR=https://mirrors.ustc.edu.cn
 WORKDIR /src
 COPY go.mod go.sum ./
-RUN  apk add git && env GOPROXY=${GOPROXY:-direct} go mod download
+RUN  sed -i "s,https://dl-cdn.alpinelinux.org,${ALPINE_MIRROR}," /etc/apk/repositories; apk add git && env GOPROXY=${GOPROXY:-direct} go mod download
 ADD . /src
-RUN  go build -o /cache-proxy ./cmd/proxy
+RUN  go build -o /cache-proxy -tags "${TAGS}" ./cmd/proxy
 
 FROM docker.io/library/alpine:3.21 AS runtime
 COPY --from=0 /cache-proxy /bin/cache-proxy

cmd/proxy/debug.go

@@ -0,0 +1,17 @@
+//go:build pprof
+
+package main
+
+import (
+	"flag"
+	"os"
+)
+
+func init() {
+	flag.BoolVar(&pprofEnabled, "pprof", true, "")
+
+	if v, ok := os.LookupEnv("SENTRY_DSN"); ok {
+		sentrydsn = v
+	}
+	flag.StringVar(&sentrydsn, "sentry", sentrydsn, "sentry dsn to report errors")
+}

cmd/proxy/main.go

@@ -4,6 +4,7 @@ import (
 	"flag"
 	"log/slog"
 	"net/http"
+	"net/http/pprof"
 	"os"
 	"path/filepath"
 	"time"
@@ -22,6 +23,7 @@ var (
 	configFilePath = "config.yaml"
 	logLevel       = "info"
 	sentrydsn      = ""
+	pprofEnabled   = false
 )
 
 func init() {
@@ -31,12 +33,8 @@ func init() {
 	if v, ok := os.LookupEnv("LOG_LEVEL"); ok {
 		logLevel = v
 	}
-	if v, ok := os.LookupEnv("SENTRY_DSN"); ok {
-		sentrydsn = v
-	}
 	flag.StringVar(&configFilePath, "config", configFilePath, "path to config file")
 	flag.StringVar(&logLevel, "log-level", logLevel, "log level. (trace, debug, info, warn, error)")
-	flag.StringVar(&sentrydsn, "sentry", sentrydsn, "sentry dsn to report errors")
 }
 
 func configFromFile(path string) (*cacheproxy.Config, error) {
@@ -57,9 +55,9 @@ func configFromFile(path string) (*cacheproxy.Config, error) {
 			Local: &cacheproxy.LocalStorage{
 				Path:                 "./data",
 				TemporaryFilePattern: "temp.*",
-			},
-			Accel: cacheproxy.Accel{
-				ResponseWithHeaders: []string{"X-Sendfile", "X-Accel-Redirect"},
+				Accel: cacheproxy.Accel{
+					RespondWithHeaders: []string{"X-Sendfile", "X-Accel-Redirect"},
+				},
 			},
 		},
 		Misc: cacheproxy.MiscConfig{
@@ -124,6 +122,14 @@ func main() {
 
 	mux.HandleFunc("GET /", server.HandleRequestWithCache)
 
+	if pprofEnabled {
+		mux.HandleFunc("GET /debug/pprof/", pprof.Index)
+		mux.HandleFunc("GET /debug/pprof/cmdline", pprof.Cmdline)
+		mux.HandleFunc("GET /debug/pprof/profile", pprof.Profile)
+		mux.HandleFunc("GET /debug/pprof/symbol", pprof.Symbol)
+		mux.HandleFunc("GET /debug/pprof/trace", pprof.Trace)
+	}
+
 	slog.With("addr", ":8881").Info("serving app")
 	if err := http.ListenAndServe(":8881", middleware.Use(mux,
 		recover.Recover(),

compose.release.yaml

@@ -0,0 +1,20 @@
+services:
+  cache-proxy:
+    image: ${REPOSITORY:-git.jeffthecoder.xyz}/${OWNER:-public}/cache-proxy:${VERSION:-latest}
+    build:
+      args:
+        GOPROXY: ${GOPROXY:-direct}
+        ALPINE_MIRROR: ${ALPINE_MIRROR:-https://mirrors.ustc.edu.cn}
+
+    restart: always
+
+    volumes:
+      - ./config.yaml:/config.yaml:ro
+      - ./data:/data
+    ports:
+      - 8881:8881
+    environment:
+      - LOG_LEVEL=info
+    env_file:
+      - path: .env
+        required: false

compose.yaml

@@ -1,11 +1,12 @@
 services:
   cache-proxy:
-    image: 100.64.0.2:3000/guochao/cache-proxy
+    image: ${REPOSITORY:-git.jeffthecoder.xyz}/${OWNER:-public}/cache-proxy:${VERSION:-latest}
     build:
       args:
         GOPROXY: ${GOPROXY:-direct}
+        ALPINE_MIRROR: ${ALPINE_MIRROR:-https://mirrors.ustc.edu.cn}
+        TAGS: pprof
 
-    pull_policy: always
     restart: always
 
     volumes:
@@ -14,8 +15,8 @@ services:
     ports:
       - 8881:8881
     environment:
-      - HTTPS_PROXY=http://100.64.0.23:7890
-      - HTTP_PROXY=http://100.64.0.23:7890
-      - ALL_PROXY=http://100.64.0.23:7890
-      - SENTRY_DSN=http://3b7336602c77427d96318074cd86ee04@100.64.0.2:8000/2
+      - SENTRY_DSN=${SENTRY_DSN}
       - LOG_LEVEL=debug
+    env_file:
+      - path: .env
+        required: false

config.go

@@ -36,17 +36,18 @@ func (upstream Upstream) GetPath(orig string) (string, bool, error) {
 type LocalStorage struct {
 	Path                 string `yaml:"path"`
 	TemporaryFilePattern string `yaml:"temporary-file-pattern"`
+
+	Accel Accel `yaml:"accel"`
 }
 
 type Accel struct {
-	EnableByHeader      string   `yaml:"enable-by-header"`
-	ResponseWithHeaders []string `yaml:"response-with-headers"`
+	EnableByHeader     string   `yaml:"enable-by-header"`
+	RespondWithHeaders []string `yaml:"respond-with-headers"`
 }
 
 type Storage struct {
 	Type  string        `yaml:"type"`
 	Local *LocalStorage `yaml:"local"`
-	Accel Accel         `yaml:"accel"`
 }
 
 type CachePolicyOnPath struct {

config.yaml

@@ -1,7 +1,7 @@
 upstream:
 - server: https://mirrors.aliyun.com
   match:
-    match: /(debian|ubuntu|ubuntu-releases|alpine|archlinux|kali|manjaro|msys2|almalinux|rocky|centos|centos-stream|centos-vault|fedora|epel|gnu)/(.*)
+    match: /(debian|ubuntu|ubuntu-releases|alpine|archlinux|kali|manjaro|msys2|almalinux|rocky|centos|centos-stream|centos-vault|fedora|epel|gnu|pypi)/(.*)
     replace: '/$1/$2'
 - server: https://mirrors.tencent.com
   match:
@@ -9,7 +9,7 @@ upstream:
     replace: '/$1/$2'
 - server: https://mirrors.ustc.edu.cn
   match:
-    match: /(debian|ubuntu|ubuntu-releases|alpine|archlinux|kali|manjaro|msys2|almalinux|rocky|centos|centos-stream|centos-vault|fedora|epel|elrepo|remi|rpmfusion|tailscale|gnu|rust-static)/(.*)
+    match: /(debian|ubuntu|ubuntu-releases|alpine|archlinux|kali|manjaro|msys2|almalinux|rocky|centos|centos-stream|centos-vault|fedora|epel|elrepo|remi|rpmfusion|tailscale|gnu|rust-static|pypi)/(.*)
     replace: '/$1/$2'
 - server: https://packages.microsoft.com/repos/code
   match:
@@ -88,21 +88,21 @@ upstream:
   allowed-redirect: "^https?://cf-builds.garudalinux.org/.*/chaotic-aur/.*$"
 
 misc:
-  first-chunk-bytes: 31457280 # 1024*1024*30, all upstreams with 200 response will wait for the first chunks to select a upstream by bandwidth, instead of by latency. defaults to 50M
-  # chunk-bytes: 1048576 # 1024*1024
+  first-chunk-bytes: 31457280 ## 1024*1024*30, all upstreams with 200 response will wait for the first chunks to select a upstream by bandwidth, instead of by latency. defaults to 50M
+  # chunk-bytes: 1048576 ## 1024*1024
 
 cache:
   timeout: 1h
   policies:
-  - match: '.*\.(rpm|deb|apk|iso)$' # rpm/deb/apk/iso won't change, create only
+  - match: '.*\.(rpm|deb|apk|iso)$' ## rpm/deb/apk/iso won't change, create only
     refresh-after: never
-  - match: '^/.*-security/.*' # mostly ubuntu/debian security
+  - match: '^/.*-security/.*' ## mostly ubuntu/debian security
     refresh-after: 1h
-  - match: '^/archlinux-localaur/.*' # archlinux local repo
+  - match: '^/archlinux-localaur/.*' ## archlinux local repo
     refresh-after: never
-  - match: '^/(archlinux.*|chaotic-aur)/*.tar.*' # archlinux packages
+  - match: '^/(archlinux.*|chaotic-aur)/*.tar.*' ## archlinux packages
     refresh-after: never
-  - match: '/chaotic-aur/.*\.db$' # archlinux chaotic-aur database
+  - match: '/chaotic-aur/.*\.db$' ## archlinux chaotic-aur database
     refresh-after: 24h
   - match: '/centos/7'
     refresh-after: never
@@ -110,20 +110,23 @@ cache:
     refresh-after: never
 
 storage:
-  type: local # ignored
+  type: local ## ignored for now
   local:
-    path: ./data # defaults to ./data
+    path: ./data ## defaults to ./data
   
-  accel:
-    # example nginx config: 
-    ## location /i {
-    ##   internal;
-    ##   alias /path/to/data;
-    ## }
-    ## location / {
-    ##   proxy_pass 127.0.0.1:8881;
-    ##   proxy_set_header X-Accel-Path /i;
-    ## }
-    ## 
+    accel:
+      ## example nginx config: 
+      ## location /i {
+      ##   internal;
+      ##   alias /path/to/data;
+      ## }
+      ## location / {
+      ##   proxy_pass 127.0.0.1:8881;
+      ##   proxy_set_header X-Accel-Path /i;
+      ## }
+      ## 
+      ## then cache proxy will respond to backend a header to indicate sendfile(join(X-Accel-Path, relpath))
+      # enable-by-header: "X-Accel-Path"
 
-    # enable-by-header: "X-Accel-Path"
+      ## respond with different headers to 
+      # respond-with-headers: [X-Sendfile, X-Accel-Redirect] 

server.go

@@ -7,6 +7,7 @@ import (
 	"fmt"
 	"io"
 	"log/slog"
+	"net"
 	"net/http"
 	"os"
 	"path/filepath"
@@ -106,7 +107,7 @@ type Chunk struct {
 }
 
 func (server *Server) serveFile(w http.ResponseWriter, r *http.Request, path string) {
-	if location := r.Header.Get(server.Storage.Accel.EnableByHeader); server.Storage.Accel.EnableByHeader != "" && location != "" {
+	if location := r.Header.Get(server.Storage.Local.Accel.EnableByHeader); server.Storage.Local.Accel.EnableByHeader != "" && location != "" {
 		relPath, err := filepath.Rel(server.Storage.Local.Path, path)
 		if err != nil {
 			http.Error(w, err.Error(), http.StatusBadRequest)
@@ -114,7 +115,7 @@ func (server *Server) serveFile(w http.ResponseWriter, r *http.Request, path str
 		}
 		accelPath := filepath.Join(location, relPath)
 
-		for _, headerKey := range server.Storage.Accel.ResponseWithHeaders {
+		for _, headerKey := range server.Storage.Local.Accel.RespondWithHeaders {
 			w.Header().Set(headerKey, accelPath)
 		}
 
@@ -332,7 +333,30 @@ func (server *Server) streamOnline(w http.ResponseWriter, r *http.Request, mtime
 		}
 
 		if err != nil {
-			slog.With("error", err, "upstreamIdx", selectedIdx).Error("something happened during download. will not cache this response")
+			logger := slog.With("upstreamIdx", selectedIdx)
+			logger.Error("something happened during download. will not cache this response. setting lingering to reset the connection.")
+			hijacker, ok := w.(http.Hijacker)
+			if !ok {
+				logger.Warn("response writer is not a hijacker. failed to set lingering")
+				return
+			}
+			conn, _, err := hijacker.Hijack()
+			if err != nil {
+				logger.With("error", err).Warn("hijack failed. failed to set lingering")
+				return
+			}
+			defer conn.Close()
+
+			tcpConn, ok := conn.(*net.TCPConn)
+			if !ok {
+				logger.With("error", err).Warn("connection is not a *net.TCPConn. failed to set lingering")
+				return
+			}
+			if err := tcpConn.SetLinger(0); err != nil {
+				logger.With("error", err).Warn("failed to set lingering")
+				return
+			}
+			logger.Debug("connection set to linger. it will be reset once the conn.Close is called")
 		}
 		go func() {
 			defer func() {