+28 -21

.forgejo/workflows/ci.yaml

··· 5 5 pull_request: 6 6 workflow_dispatch: 7 7 8 8 + env: 9 9 + FORGE: codeberg.org 10 10 + 8 11 jobs: 9 12 check: 10 10 - runs-on: codeberg-small-lazy 13 13 + runs-on: debian-trixie 11 14 container: 12 12 - image: docker.io/library/node:24-trixie-slim@sha256:45babd1b4ce0349fb12c4e24bf017b90b96d52806db32e001e3013f341bef0fe 15 15 + image: docker.io/library/node:24-trixie-slim@sha256:b05474903f463ce4064c09986525e6588c3e66c51b69be9c93a39fb359f883ce 13 16 steps: 14 17 - name: Check out source code 15 15 - uses: https://code.forgejo.org/actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5 18 18 + uses: https://code.forgejo.org/actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 16 19 - name: Set up toolchain 17 17 - uses: https://code.forgejo.org/actions/setup-go@44694675825211faa026b3c33043df3e48a5fa00 # v6 20 20 + uses: https://code.forgejo.org/actions/setup-go@4dc6199c7b1a012772edbd06daecab0f50c9053c # v6.1.0 18 21 with: 19 22 go-version: '>=1.25.0' 20 23 - name: Install dependencies ··· 25 28 - name: Build service 26 29 run: | 27 30 go build 31 31 + - name: Run tests 32 32 + run: | 33 33 + go test ./... 28 34 - name: Run static analysis 29 35 run: | 30 30 - go vet 31 31 - staticcheck 36 36 + go vet ./... 37 37 + staticcheck ./... 32 38 33 39 release: 34 40 # IMPORTANT: This workflow step will not work without the Releases unit enabled! 35 41 if: ${{ forge.ref == 'refs/heads/main' || startsWith(forge.event.ref, 'refs/tags/v') }} 36 42 needs: [check] 37 37 - runs-on: codeberg-medium-lazy 43 43 + runs-on: debian-trixie 38 44 container: 39 39 - image: docker.io/library/node:24-trixie-slim@sha256:ef4ca6d078dd18322059a1f051225f7bbfc2bb60c16cbb5d8a1ba2cc8964fe8a 45 45 + image: docker.io/library/node:24-trixie-slim@sha256:b05474903f463ce4064c09986525e6588c3e66c51b69be9c93a39fb359f883ce 40 46 steps: 41 47 - name: Check out source code 42 42 - uses: https://code.forgejo.org/actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5 48 48 + uses: https://code.forgejo.org/actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 43 49 - name: Set up toolchain 44 44 - uses: https://code.forgejo.org/actions/setup-go@44694675825211faa026b3c33043df3e48a5fa00 # v6 50 50 + uses: https://code.forgejo.org/actions/setup-go@4dc6199c7b1a012772edbd06daecab0f50c9053c # v6.1.0 45 51 with: 46 52 go-version: '>=1.25.0' 47 53 - name: Install dependencies ··· 58 64 build linux arm64 59 65 build darwin arm64 60 66 - name: Create release 61 61 - uses: https://code.forgejo.org/actions/forgejo-release@v2.7.3 67 67 + uses: https://code.forgejo.org/actions/forgejo-release@fc0488c944626f9265d87fbc4dd6c08f78014c63 # v2.7.3 62 68 with: 63 69 tag: ${{ startsWith(forge.event.ref, 'refs/tags/v') && forge.ref_name || 'latest' }} 64 70 release-dir: assets ··· 69 75 package: 70 76 if: ${{ forge.ref == 'refs/heads/main' || startsWith(forge.event.ref, 'refs/tags/v') }} 71 77 needs: [check] 72 72 - runs-on: codeberg-medium-lazy 78 78 + runs-on: debian-trixie 73 79 container: 74 74 - image: docker.io/library/node:24-trixie-slim@sha256:ef4ca6d078dd18322059a1f051225f7bbfc2bb60c16cbb5d8a1ba2cc8964fe8a 80 80 + image: docker.io/library/node:24-trixie-slim@sha256:b05474903f463ce4064c09986525e6588c3e66c51b69be9c93a39fb359f883ce 75 81 steps: 76 82 - name: Install dependencies 77 83 run: | 78 84 apt-get -y update 79 79 - apt-get -y install buildah ca-certificates 85 85 + apt-get -y install ca-certificates buildah qemu-user-binfmt 80 86 - name: Check out source code 81 81 - uses: https://code.forgejo.org/actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5 87 87 + uses: https://code.forgejo.org/actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 88 88 + - name: Authenticate with Docker 89 89 + run: | 90 90 + buildah login --authfile=/tmp/authfile-${FORGE}.json \ 91 91 + -u ${{ vars.PACKAGES_USER }} -p ${{ secrets.PACKAGES_TOKEN }} ${FORGE} 82 92 - name: Build container 83 93 run: | 84 94 printf '[storage]\ndriver="vfs"\nrunroot="/run/containers/storage"\ngraphroot="/var/lib/containers/storage"\n' | tee /etc/containers/storage.conf 85 85 - buildah build --arch=amd64 --tag=container:${VER}-amd64 . 86 86 - buildah build --arch=arm64 --tag=container:${VER}-arm64 . 95 95 + buildah build ${CACHE} --arch=amd64 --tag=container:${VER}-amd64 96 96 + buildah build ${CACHE} --arch=arm64 --tag=container:${VER}-arm64 87 97 buildah manifest create container:${VER} \ 88 98 container:${VER}-amd64 \ 89 99 container:${VER}-arm64 90 100 env: 91 101 BUILDAH_ISOLATION: chroot 92 102 VER: ${{ startsWith(forge.event.ref, 'refs/tags/v') && forge.ref_name || 'latest' }} 103 103 + CACHE: ${{ format('--authfile=/tmp/authfile-{0}.json --layers --cache-from {0}/{1}/cache --cache-to {0}/{1}/cache', env.FORGE, forge.repository) }} 93 104 - if: ${{ forge.repository == 'git-pages/git-pages' }} 94 105 name: Push container to Codeberg 95 106 run: | 96 96 - buildah login --authfile=/tmp/authfile-${FORGE}.json \ 97 97 - -u ${{ vars.PACKAGES_USER }} -p ${{ secrets.PACKAGES_TOKEN }} ${FORGE} 98 107 buildah manifest push --authfile=/tmp/authfile-${FORGE}.json \ 99 108 --all container:${VER} "docker://${FORGE}/${{ forge.repository }}:${VER/v/}" 100 109 env: 101 101 - BUILDAH_ISOLATION: chroot 102 102 - FORGE: codeberg.org 103 110 VER: ${{ startsWith(forge.event.ref, 'refs/tags/v') && forge.ref_name || 'latest' }}

+1

.gitignore

··· 4 4 /data 5 5 /config*.toml* 6 6 /git-pages 7 7 + /site

+11 -14

Dockerfile

··· 1 1 # Install CA certificates. 2 2 - FROM docker.io/library/alpine:latest@sha256:4b7ce07002c69e8f3d704a9c5d6fd3053be500b7f1c69fc0d80990c2ad8dd412 AS ca-certificates-builder 2 2 + FROM docker.io/library/alpine:3 AS ca-certificates-builder 3 3 RUN apk --no-cache add ca-certificates 4 4 5 5 # Build supervisor. 6 6 - FROM docker.io/library/golang:1.25-alpine@sha256:aee43c3ccbf24fdffb7295693b6e33b21e01baec1b2a55acc351fde345e9ec34 AS supervisor-builder 6 6 + FROM docker.io/library/golang:1.25-alpine@sha256:ac09a5f469f307e5da71e766b0bd59c9c49ea460a528cc3e6686513d64a6f1fb AS supervisor-builder 7 7 RUN apk --no-cache add git 8 8 WORKDIR /build 9 9 RUN git clone https://github.com/ochinchina/supervisord . && \ 10 10 git checkout 16cb640325b3a4962b2ba17d68fb5c2b1e1b6b3c 11 11 - RUN GOBIN=/usr/bin go install -ldflags "-s -w" && \ 12 12 - go clean -cache -modcache 11 11 + RUN GOBIN=/usr/bin go install -ldflags "-s -w" 13 12 14 13 # Build Caddy with S3 storage backend. 15 15 - FROM docker.io/library/caddy:2.10.2-builder@sha256:53f91ad7c5f1ab9a607953199b7c1e10920c570ae002aef913d68ed7464fb19f AS caddy-builder 14 14 + FROM docker.io/library/caddy:2.10.2-builder@sha256:6644af24bde2b4dbb07eb57637051abd2aa713e9787fa1eb544c3f31a0620898 AS caddy-builder 16 15 RUN xcaddy build ${CADDY_VERSION} \ 17 17 - --with=github.com/ss098/certmagic-s3@v0.0.0-20250922022452-8af482af5f39 && \ 18 18 - go clean -cache -modcache 16 16 + --with=github.com/ss098/certmagic-s3@v0.0.0-20250922022452-8af482af5f39 19 17 20 18 # Build git-pages. 21 21 - FROM docker.io/library/golang:1.25-alpine@sha256:aee43c3ccbf24fdffb7295693b6e33b21e01baec1b2a55acc351fde345e9ec34 AS git-pages-builder 19 19 + FROM docker.io/library/golang:1.25-alpine@sha256:ac09a5f469f307e5da71e766b0bd59c9c49ea460a528cc3e6686513d64a6f1fb AS git-pages-builder 22 20 RUN apk --no-cache add git 23 21 WORKDIR /build 24 22 COPY go.mod go.sum ./ 25 23 RUN go mod download 26 24 COPY *.go ./ 27 25 COPY src/ ./src/ 28 28 - RUN go build -ldflags "-s -w" -o git-pages . && \ 29 29 - go clean -cache -modcache 26 26 + RUN go build -ldflags "-s -w" -o git-pages . 30 27 31 28 # Compose git-pages and Caddy. 32 32 - FROM docker.io/library/busybox:1.37.0-musl@sha256:ef13e7482851632be3faf5bd1d28d4727c0810901d564b35416f309975a12a30 29 29 + FROM docker.io/library/busybox:1.37.0-musl@sha256:03db190ed4c1ceb1c55d179a0940e2d71d42130636a780272629735893292223 33 30 COPY --from=ca-certificates-builder /etc/ssl/cert.pem /etc/ssl/cert.pem 34 31 COPY --from=supervisor-builder /usr/bin/supervisord /bin/supervisord 35 32 COPY --from=caddy-builder /usr/bin/caddy /bin/caddy ··· 39 36 RUN mkdir /app/data 40 37 COPY conf/supervisord.conf /app/supervisord.conf 41 38 COPY conf/Caddyfile /app/Caddyfile 42 42 - COPY conf/config.example.toml /app/config.toml 39 39 + COPY conf/config.docker.toml /app/config.toml 43 40 44 41 # Caddy ports: 45 42 EXPOSE 80/tcp 443/tcp 443/udp ··· 49 46 # While the default command is to run git-pages standalone, the intended configuration 50 47 # is to use it with Caddy and store both site data and credentials to an S3-compatible 51 48 # object store. 52 52 - # * In a standalone configuration, the default, git-caddy listens on port 3000 (http). 53 53 - # * In a combined configuration, supervisord launches both git-caddy and Caddy, and 49 49 + # * In a standalone configuration, the default, git-pages listens on port 3000 (http). 50 50 + # * In a combined configuration, supervisord launches both git-pages and Caddy, and 54 51 # Caddy listens on ports 80 (http) and 443 (https). 55 52 CMD ["git-pages"] 56 53 # CMD ["supervisord"]

-14

LICENSE-0BSD.txt

··· 1 1 - Copyright (C) git-pages contributors 2 2 - Copyright (C) Catherine 'whitequark' 3 3 - 4 4 - Permission to use, copy, modify, and/or distribute this software for 5 5 - any purpose with or without fee is hereby granted. 6 6 - 7 7 - THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES 8 8 - WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF 9 9 - MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR 10 10 - ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES 11 11 - WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN 12 12 - AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT 13 13 - OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. 14 14 -

+14

LICENSE.txt

··· 1 1 + Copyright (C) git-pages contributors 2 2 + Copyright (C) Catherine 'whitequark' 3 3 + 4 4 + Permission to use, copy, modify, and/or distribute this software for 5 5 + any purpose with or without fee is hereby granted. 6 6 + 7 7 + THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES 8 8 + WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF 9 9 + MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR 10 10 + ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES 11 11 + WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN 12 12 + AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT 13 13 + OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. 14 14 +

+38 -11

README.md

··· 1 1 git-pages 2 2 ========= 3 3 4 4 - _git-pages_ is a static site server for use with Git forges (i.e. a GitHub Pages replacement). It is written with efficiency in mind, scaling horizontally to any number of deployed sites and concurrent requests and serving sites up to hundreds of megabytes in size, while being equally suitable for single-user deployments. 4 4 + _git-pages_ is a static site server for use with Git forges (i.e. a GitHub Pages replacement). It is written with efficiency in mind, scaling horizontally to any number of machines and serving sites up to multiple gigabytes in size, while being equally suitable for small single-user deployments. 5 5 + 6 6 + It is implemented in Go and has no other mandatory dependencies, although it is designed to be used together with the [Caddy server][caddy] for TLS termination. Site data may be stored on the filesystem or in an [Amazon S3](https://aws.amazon.com/s3/) compatible object store. 5 7 6 6 - It is implemented in Go and has no other mandatory dependencies, although it is designed to be used together with the [Caddy server][caddy] (for TLS termination) and an [Amazon S3](https://aws.amazon.com/s3/) compatible object store (for horizontal scalability of storage). 8 8 + The included Docker container provides everything needed to deploy a Pages service, including zero-configuration on-demand provisioning of TLS certificates from [Let's Encrypt](https://letsencrypt.org/), and runs on any commodity cloud infrastructure. 7 9 8 8 - The included Docker container provides everything needed to deploy a Pages service, including zero-configuration on-demand provisioning of TLS certificates from [Let's Encrypt](https://letsencrypt.org/), and runs on any commodity cloud infrastructure. There is also a first-party deployment of _git-pages_ at [grebedoc.dev](https://grebedoc.dev). 10 10 + > [!TIP] 11 11 + > If you want to publish a site using _git-pages_ to an existing service like Codeberg Pages or [Grebedoc][grebedoc], consider using the [CLI tool][git-pages-cli] or the [Forgejo Action][git-pages-action]. 9 12 10 13 [caddy]: https://caddyserver.com/ 14 14 + [git-pages-cli]: https://codeberg.org/git-pages/git-pages-cli 15 15 + [git-pages-action]: https://codeberg.org/git-pages/action 16 16 + [codeberg-pages]: https://codeberg.page 17 17 + [grebedoc]: https://grebedoc.dev 11 18 12 19 13 20 Quickstart ··· 32 39 33 40 The `pages` branch of the repository is now available at http://localhost:3000/! 34 41 35 35 - [git-pages-cli]: https://codeberg.org/git-pages/git-pages-cli 36 36 - 37 42 38 43 Deployment 39 44 ---------- ··· 63 68 - Site URLs that have a path starting with `.git-pages/...` are reserved for _git-pages_ itself. 64 69 - The `.git-pages/health` URL returns `ok` with the `Last-Modified:` header set to the manifest modification time. 65 70 - The `.git-pages/manifest.json` URL returns a [ProtoJSON](https://protobuf.dev/programming-guides/json/) representation of the deployed site manifest with the `Last-Modified:` header set to the manifest modification time. It enumerates site structure, redirect rules, and errors that were not severe enough to abort publishing. Note that **the manifest JSON format is not stable and will change without notice**. 66 66 - - **With feature `archive-site`:** The `.git-pages/archive.tar` URL returns a tar archive of all site contents, including `_redirects` and `_headers` files (reconstructed from the manifest), with the `Last-Modified:` header set to the manifest modification time. Compression can be enabled using the `Accept-Encoding:` HTTP header (only). 71 71 + - The `.git-pages/archive.tar` URL returns a tar archive of all site contents, including `_redirects` and `_headers` files (reconstructed from the manifest), with the `Last-Modified:` header set to the manifest modification time. Compression can be enabled using the `Accept-Encoding:` HTTP header (only). 67 72 * In response to a `PUT` or `POST` request, the server updates a site with new content. The URL of the request must be the root URL of the site that is being published. 68 73 - If the `PUT` method receives an `application/x-www-form-urlencoded` body, it contains a repository URL to be shallowly cloned. The `Branch` header contains the branch to be checked out; the `pages` branch is used if the header is absent. 69 74 - If the `PUT` method receives an `application/x-tar`, `application/x-tar+gzip`, `application/x-tar+zstd`, or `application/zip` body, it contains an archive to be extracted. 70 75 - The `POST` method requires an `application/json` body containing a Forgejo/Gitea/Gogs/GitHub webhook event payload. Requests where the `ref` key contains anything other than `refs/heads/pages` are ignored, and only the `pages` branch is used. The `repository.clone_url` key contains a repository URL to be shallowly cloned. 71 76 - If the received contents is empty, performs the same action as `DELETE`. 77 77 + * In response to a `PATCH` request, the server partially updates a site with new content. The URL of the request must be the root URL of the site that is being published. 78 78 + - The request must have a `application/x-tar`, `application/x-tar+gzip`, or `application/x-tar+zstd` body, whose contents is *merged* with the existing site contents as follows: 79 79 + - A character device entry with major 0 and minor 0 is treated as a "whiteout marker" (following [unionfs][whiteout]): it causes any existing file or directory with the same name to be deleted. 80 80 + - A directory entry replaces any existing file or directory with the same name (if any), recursively removing the old contents. 81 81 + - A file or symlink entry replaces any existing file or directory with the same name (if any). 82 82 + - If there is no `Create-Parents:` header or a `Create-Parents: no` header is present, the parent path of an entry must exist and refer to a directory. 83 83 + - If a `Create-Parents: yes` header is present, any missing segments in the parent path of an entry will be created (like `mkdir -p`). Any existing segments refer to directories. 84 84 + - The request must have a `Atomic: yes` or `Atomic: no` header. Not every backend configuration makes it possible to perform atomic compare-and-swap operations; on backends without atomic CAS support, `Atomic: yes` requests will fail, while `Atomic: no` requests will provide a best-effort approximation. 85 85 + - If a `PATCH` request loses a race against another content update request, it may return `409 Conflict`. This is true regardless of the `Atomic:` header value. Whenever this happens, resubmit the request as-is. 86 86 + - If the site has no contents after the update is applied, performs the same action as `DELETE`. 72 87 * In response to a `DELETE` request, the server unpublishes a site. The URL of the request must be the root URL of the site that is being unpublished. Site data remains stored for an indeterminate period of time, but becomes completely inaccessible. 88 88 + * If a `Dry-Run: yes` header is provided with a `PUT`, `PATCH`, `DELETE`, or `POST` request, only the authorization checks are run; no destructive updates are made. 73 89 * All updates to site content are atomic (subject to consistency guarantees of the storage backend). That is, there is an instantaneous moment during an update before which the server will return the old content and after which it will return the new content. 74 90 * Files with a certain name, when placed in the root of a site, have special functions: 75 91 - [Netlify `_redirects`][_redirects] file can be used to specify HTTP redirect and rewrite rules. The _git-pages_ implementation currently does not support placeholders, query parameters, or conditions, and may differ from Netlify in other minor ways. If you find that a supported `_redirects` file feature does not work the same as on Netlify, please file an issue. (Note that _git-pages_ does not perform URL normalization; `/foo` and `/foo/` are *not* the same, unlike with Netlify.) 76 92 - [Netlify `_headers`][_headers] file can be used to specify custom HTTP response headers (if allowlisted by configuration). In particular, this is useful to enable [CORS requests][cors]. The _git-pages_ implementation may differ from Netlify in minor ways; if you find that a `_headers` file feature does not work the same as on Netlify, please file an issue. 93 93 + * Incremental updates can be made using `PUT` or `PATCH` requests where the body contains an archive (both tar and zip are supported). 94 94 + - Any archive entry that is a symlink to `/git/pages/<git-sha256>` is replaced with an existing manifest entry for the same site whose git blob hash matches `<git-sha256>`. If there is no existing manifest entry with the specified git hash, the update fails with a `422 Unprocessable Entity`. 95 95 + - For this error response only, if the negotiated content type is `application/vnd.git-pages.unresolved`, the response will contain the `<git-sha256>` of each unresolved reference, one per line. 96 96 + * Support for SHA-256 Git hashes is [limited by go-git][go-git-sha256]; once go-git implements the required features, _git-pages_ will automatically gain support for SHA-256 Git hashes. Note that shallow clones (used by _git-pages_ to conserve bandwidth if available) aren't supported yet in the Git protocol as of 2025. 77 97 78 98 [_redirects]: https://docs.netlify.com/manage/routing/redirects/overview/ 79 99 [_headers]: https://docs.netlify.com/manage/routing/headers/ 80 100 [cors]: https://developer.mozilla.org/en-US/docs/Web/HTTP/Guides/CORS 101 101 + [go-git-sha256]: https://github.com/go-git/go-git/issues/706 102 102 + [whiteout]: https://docs.kernel.org/filesystems/overlayfs.html#whiteouts-and-opaque-directories 81 103 82 104 83 105 Authorization ··· 85 107 86 108 DNS is the primary authorization method, using either TXT records or wildcard matching. In certain cases, git forge authorization is used in addition to DNS. 87 109 88 88 - The authorization flow for content updates (`PUT`, `DELETE`, `POST` requests) proceeds sequentially in the following order, with the first of multiple applicable rule taking precedence: 110 110 + The authorization flow for content updates (`PUT`, `PATCH`, `DELETE`, `POST` requests) proceeds sequentially in the following order, with the first of multiple applicable rule taking precedence: 89 111 90 112 1. **Development Mode:** If the environment variable `PAGES_INSECURE` is set to a truthful value like `1`, the request is authorized. 91 91 - 2. **DNS Challenge:** If the method is `PUT`, `DELETE`, `POST`, and a well-formed `Authorization:` header is provided containing a `<token>`, and a TXT record lookup at `_git-pages-challenge.<host>` returns a record whose concatenated value equals `SHA256("<host> <token>")`, the request is authorized. 113 113 + 2. **DNS Challenge:** If the method is `PUT`, `PATCH`, `DELETE`, `POST`, and a well-formed `Authorization:` header is provided containing a `<token>`, and a TXT record lookup at `_git-pages-challenge.<host>` returns a record whose concatenated value equals `SHA256("<host> <token>")`, the request is authorized. 92 114 - **`Pages` scheme:** Request includes an `Authorization: Pages <token>` header. 93 115 - **`Basic` scheme:** Request includes an `Authorization: Basic <basic>` header, where `<basic>` is equal to `Base64("Pages:<token>")`. (Useful for non-Forgejo forges.) 94 94 - 3. **DNS Allowlist:** If the method is `PUT` or `POST`, and a TXT record lookup at `_git-pages-repository.<host>` returns a set of well-formed absolute URLs, and (for `PUT` requests) the body contains a repository URL, and the requested clone URLs is contained in this set of URLs, the request is authorized. 116 116 + 3. **DNS Allowlist:** If the method is `PUT` or `POST`, and the request URL is `scheme://<user>.<host>/`, and a TXT record lookup at `_git-pages-repository.<host>` returns a set of well-formed absolute URLs, and (for `PUT` requests) the body contains a repository URL, and the requested clone URLs is contained in this set of URLs, the request is authorized. 95 117 4. **Wildcard Match (content):** If the method is `POST`, and a `[[wildcard]]` configuration section exists where the suffix of a hostname (compared label-wise) is equal to `[[wildcard]].domain`, and (for `PUT` requests) the body contains a repository URL, and the requested clone URL is a *matching* clone URL, the request is authorized. 96 118 - **Index repository:** If the request URL is `scheme://<user>.<host>/`, a *matching* clone URL is computed by templating `[[wildcard]].clone-url` with `<user>` and `<project>`, where `<project>` is computed by templating each element of `[[wildcard]].index-repos` with `<user>`, and `[[wildcard]]` is the section where the match occurred. 97 119 - **Project repository:** If the request URL is `scheme://<user>.<host>/<project>/`, a *matching* clone URL is computed by templating `[[wildcard]].clone-url` with `<user>` and `<project>`, and `[[wildcard]]` is the section where the match occurred. 98 98 - 5. **Forge Authorization:** If the method is `PUT`, and the body contains an archive, and a `[[wildcard]]` configuration section exists where the suffix of a hostname (compared label-wise) is equal to `[[wildcard]].domain`, and `[[wildcard]].authorization` is non-empty, and the request includes a `Forge-Authorization:` header, and the header (when forwarded as `Authorization:`) grants push permissions to a repository at the *matching* clone URL (as defined above) as determined by an API call to the forge, the request is authorized. (This enables publishing a site for a private repository.) 120 120 + 5. **Forge Authorization:** If the method is `PUT` or `PATCH`, and the body contains an archive, and a `[[wildcard]]` configuration section exists where the suffix of a hostname (compared label-wise) is equal to `[[wildcard]].domain`, and `[[wildcard]].authorization` is non-empty, and the request includes a `Forge-Authorization:` header, and the header (when forwarded as `Authorization:`) grants push permissions to a repository at the *matching* clone URL (as defined above) as determined by an API call to the forge, the request is authorized. (This enables publishing a site for a private repository.) 99 121 5. **Default Deny:** Otherwise, the request is not authorized. 100 122 101 123 The authorization flow for metadata retrieval (`GET` requests with site paths starting with `.git-pages/`) in the following order, with the first of multiple applicable rule taking precedence: ··· 115 137 * If `SENTRY_DSN` environment variable is set, panics are reported to Sentry. 116 138 * If `SENTRY_DSN` and `SENTRY_LOGS=1` environment variables are set, logs are uploaded to Sentry. 117 139 * If `SENTRY_DSN` and `SENTRY_TRACING=1` environment variables are set, traces are uploaded to Sentry. 140 140 + * Optional syslog integration allows transmitting application logs to a syslog daemon. When present, the `SYSLOG_ADDR` environment variable enables the integration, and the value is used to configure the syslog destination. The value must follow the format `family/address` and is usually one of the following: 141 141 + * a Unix datagram socket: `unixgram//dev/log`; 142 142 + * TLS over TCP: `tcp+tls/host:port`; 143 143 + * plain TCP: `tcp/host:post`; 144 144 + * UDP: `udp/host:port`. 118 145 119 146 120 147 Architecture (v2) ··· 160 187 License 161 188 ------- 162 189 163 163 - [0-clause BSD](LICENSE-0BSD.txt) 190 190 + [0-clause BSD](LICENSE.txt)

-6

conf/Caddyfile

··· 25 25 on_demand 26 26 } 27 27 28 28 - # initial PUT/POST for a new domain has to happen over HTTP 29 29 - @upgrade `method('GET') && protocol('http')` 30 30 - redir @upgrade https://{host}{uri} 301 31 31 - 32 28 reverse_proxy http://localhost:3000 33 33 - header Alt-Svc `h3=":443"; persist=1, h2=":443"; persist=1` 34 34 - encode 35 29 }

+4

conf/config.docker.toml

··· 1 1 + [server] 2 2 + pages = "tcp/:3000" 3 3 + caddy = "tcp/:3001" 4 4 + metrics = "tcp/:3002"

+15 -6

conf/config.example.toml

··· 5 5 6 6 [server] 7 7 # Use "-" to disable the handler. 8 8 - pages = "tcp/:3000" 9 9 - caddy = "tcp/:3001" 10 10 - metrics = "tcp/:3002" 8 8 + pages = "tcp/localhost:3000" 9 9 + caddy = "tcp/localhost:3001" 10 10 + metrics = "tcp/localhost:3002" 11 11 12 12 [[wildcard]] # non-default section 13 13 domain = "codeberg.page" ··· 15 15 index-repos = ["<user>.codeberg.page", "pages"] 16 16 index-repo-branch = "main" 17 17 authorization = "forgejo" 18 18 - fallback-proxy-to = "https://codeberg.page" 18 18 + 19 19 + [fallback] # non-default section 20 20 + proxy-to = "https://codeberg.page" 21 21 + insecure = false 19 22 20 23 [storage] 21 24 type = "fs" ··· 23 26 [storage.fs] 24 27 root = "./data" 25 28 26 26 - [storage.s3] # non-default bucket configuration 29 29 + [storage.s3] # non-default section 27 30 endpoint = "play.min.io" 28 31 access-key-id = "Q3AM3UQ867SPQQA43P2F" 29 32 secret-access-key = "zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG" ··· 47 50 update-timeout = "60s" 48 51 max-heap-size-ratio = 0.5 # * RAM_size 49 52 forbidden-domains = [] 50 50 - # allowed-repository-url-prefixes = <nil> 53 53 + allowed-repository-url-prefixes = [] 51 54 allowed-custom-headers = ["X-Clacks-Overhead"] 55 55 + 56 56 + [audit] 57 57 + node-id = 0 58 58 + collect = false 59 59 + include-ip = "" 60 60 + notify-url = "" 52 61 53 62 [observability] 54 63 slow-response-threshold = "500ms"

+24

flake.lock

··· 18 18 "type": "github" 19 19 } 20 20 }, 21 21 + "gomod2nix": { 22 22 + "inputs": { 23 23 + "flake-utils": [ 24 24 + "flake-utils" 25 25 + ], 26 26 + "nixpkgs": [ 27 27 + "nixpkgs" 28 28 + ] 29 29 + }, 30 30 + "locked": { 31 31 + "lastModified": 1763982521, 32 32 + "narHash": "sha256-ur4QIAHwgFc0vXiaxn5No/FuZicxBr2p0gmT54xZkUQ=", 33 33 + "owner": "nix-community", 34 34 + "repo": "gomod2nix", 35 35 + "rev": "02e63a239d6eabd595db56852535992c898eba72", 36 36 + "type": "github" 37 37 + }, 38 38 + "original": { 39 39 + "owner": "nix-community", 40 40 + "repo": "gomod2nix", 41 41 + "type": "github" 42 42 + } 43 43 + }, 21 44 "nix-filter": { 22 45 "locked": { 23 46 "lastModified": 1757882181, ··· 52 75 "root": { 53 76 "inputs": { 54 77 "flake-utils": "flake-utils", 78 78 + "gomod2nix": "gomod2nix", 55 79 "nix-filter": "nix-filter", 56 80 "nixpkgs": "nixpkgs" 57 81 }

+19 -4

flake.nix

··· 3 3 nixpkgs.url = "github:NixOS/nixpkgs/nixpkgs-unstable"; 4 4 flake-utils.url = "github:numtide/flake-utils"; 5 5 nix-filter.url = "github:numtide/nix-filter"; 6 6 + 7 7 + gomod2nix = { 8 8 + url = "github:nix-community/gomod2nix"; 9 9 + inputs.nixpkgs.follows = "nixpkgs"; 10 10 + inputs.flake-utils.follows = "flake-utils"; 11 11 + }; 6 12 }; 7 13 8 14 outputs = ··· 11 17 nixpkgs, 12 18 flake-utils, 13 19 nix-filter, 14 14 - }: 20 20 + ... 21 21 + }@inputs: 15 22 flake-utils.lib.eachDefaultSystem ( 16 23 system: 17 24 let 18 18 - pkgs = nixpkgs.legacyPackages.${system}; 25 25 + pkgs = import nixpkgs { 26 26 + inherit system; 27 27 + 28 28 + overlays = [ 29 29 + inputs.gomod2nix.overlays.default 30 30 + ]; 31 31 + }; 19 32 20 20 - git-pages = pkgs.buildGo125Module { 33 33 + git-pages = pkgs.buildGoApplication { 21 34 pname = "git-pages"; 22 35 version = "0"; 23 36 ··· 43 56 "-s -w" 44 57 ]; 45 58 46 46 - vendorHash = "sha256-oVXELOXbRTzzU8pUGNE4K552thlZXGAX7qpv6ETwz6o="; 59 59 + go = pkgs.go_1_25; 60 60 + modules = ./gomod2nix.toml; 47 61 }; 48 62 in 49 63 { ··· 56 70 57 71 packages = with pkgs; [ 58 72 caddy 73 73 + gomod2nix 59 74 ]; 60 75 }; 61 76

+30 -18

go.mod

··· 3 3 go 1.25.0 4 4 5 5 require ( 6 6 - codeberg.org/git-pages/go-headers v1.1.0 6 6 + codeberg.org/git-pages/go-headers v1.1.1 7 7 + codeberg.org/git-pages/go-slog-syslog v0.0.0-20251207093707-892f654e80b7 7 8 github.com/KimMachineGun/automemlimit v0.7.5 8 9 github.com/c2h5oh/datasize v0.0.0-20231215233829-aa82cc1e6500 9 10 github.com/creasty/defaults v1.8.0 10 10 - github.com/getsentry/sentry-go v0.36.2 11 11 - github.com/getsentry/sentry-go/slog v0.36.2 12 12 - github.com/go-git/go-billy/v6 v6.0.0-20251026101908-623011986e70 13 13 - github.com/go-git/go-git/v6 v6.0.0-20251029213217-0bbfc0875edd 14 14 - github.com/klauspost/compress v1.18.1 15 15 - github.com/maypok86/otter/v2 v2.2.1 16 16 - github.com/minio/minio-go/v7 v7.0.95 11 11 + github.com/dghubble/trie v0.1.0 12 12 + github.com/fatih/color v1.18.0 13 13 + github.com/getsentry/sentry-go v0.40.0 14 14 + github.com/getsentry/sentry-go/slog v0.40.0 15 15 + github.com/go-git/go-billy/v6 v6.0.0-20251217170237-e9738f50a3cd 16 16 + github.com/go-git/go-git/v6 v6.0.0-20251231065035-29ae690a9f19 17 17 + github.com/jpillora/backoff v1.0.0 18 18 + github.com/kankanreno/go-snowflake v1.2.0 19 19 + github.com/klauspost/compress v1.18.2 20 20 + github.com/maypok86/otter/v2 v2.3.0 21 21 + github.com/minio/minio-go/v7 v7.0.97 17 22 github.com/pelletier/go-toml/v2 v2.2.4 18 23 github.com/pquerna/cachecontrol v0.2.0 19 24 github.com/prometheus/client_golang v1.23.2 20 20 - github.com/samber/slog-multi v1.5.0 25 25 + github.com/samber/slog-multi v1.7.0 21 26 github.com/tj/go-redirects v0.0.0-20200911105812-fd1ba1020b37 22 27 github.com/valyala/fasttemplate v1.2.2 23 23 - google.golang.org/protobuf v1.36.10 28 28 + golang.org/x/net v0.48.0 29 29 + google.golang.org/protobuf v1.36.11 24 30 ) 25 31 26 32 require ( ··· 29 35 github.com/beorn7/perks v1.0.1 // indirect 30 36 github.com/cespare/xxhash/v2 v2.3.0 // indirect 31 37 github.com/cloudflare/circl v1.6.1 // indirect 32 32 - github.com/cyphar/filepath-securejoin v0.5.0 // indirect 38 38 + github.com/cyphar/filepath-securejoin v0.6.1 // indirect 39 39 + github.com/davecgh/go-spew v1.1.1 // indirect 33 40 github.com/dustin/go-humanize v1.0.1 // indirect 34 41 github.com/emirpasic/gods v1.18.1 // indirect 35 42 github.com/go-git/gcfg/v2 v2.0.2 // indirect 36 43 github.com/go-ini/ini v1.67.0 // indirect 37 37 - github.com/goccy/go-json v0.10.5 // indirect 38 44 github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8 // indirect 39 45 github.com/google/uuid v1.6.0 // indirect 40 46 github.com/kevinburke/ssh_config v1.4.0 // indirect 41 47 github.com/klauspost/cpuid/v2 v2.3.0 // indirect 42 42 - github.com/minio/crc64nvme v1.0.2 // indirect 48 48 + github.com/klauspost/crc32 v1.3.0 // indirect 49 49 + github.com/leodido/go-syslog/v4 v4.3.0 // indirect 50 50 + github.com/mattn/go-colorable v0.1.13 // indirect 51 51 + github.com/mattn/go-isatty v0.0.20 // indirect 52 52 + github.com/minio/crc64nvme v1.1.0 // indirect 43 53 github.com/minio/md5-simd v1.1.2 // indirect 44 54 github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect 45 55 github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58 // indirect 46 56 github.com/philhofer/fwd v1.2.0 // indirect 47 57 github.com/pjbgf/sha1cd v0.5.0 // indirect 48 58 github.com/pkg/errors v0.9.1 // indirect 59 59 + github.com/pmezard/go-difflib v1.0.0 // indirect 49 60 github.com/prometheus/client_model v0.6.2 // indirect 50 61 github.com/prometheus/common v0.66.1 // indirect 51 62 github.com/prometheus/procfs v0.16.1 // indirect 52 63 github.com/rs/xid v1.6.0 // indirect 53 53 - github.com/samber/lo v1.51.0 // indirect 64 64 + github.com/samber/lo v1.52.0 // indirect 54 65 github.com/samber/slog-common v0.19.0 // indirect 55 66 github.com/sergi/go-diff v1.4.0 // indirect 67 67 + github.com/stretchr/testify v1.11.1 // indirect 56 68 github.com/tinylib/msgp v1.3.0 // indirect 57 69 github.com/tj/assert v0.0.3 // indirect 58 70 github.com/valyala/bytebufferpool v1.0.0 // indirect 59 71 go.yaml.in/yaml/v2 v2.4.2 // indirect 60 60 - golang.org/x/crypto v0.43.0 // indirect 61 61 - golang.org/x/net v0.46.0 // indirect 62 62 - golang.org/x/sys v0.37.0 // indirect 63 63 - golang.org/x/text v0.30.0 // indirect 72 72 + golang.org/x/crypto v0.46.0 // indirect 73 73 + golang.org/x/sys v0.39.0 // indirect 74 74 + golang.org/x/text v0.32.0 // indirect 75 75 + gopkg.in/yaml.v3 v3.0.1 // indirect 64 76 )

+59 -44

go.sum

··· 1 1 - codeberg.org/git-pages/go-headers v1.0.0 h1:hvGU97hQdXaT5HwCpZJWQdg7akvtOBCSUNL4u2a5uTs= 2 2 - codeberg.org/git-pages/go-headers v1.0.0/go.mod h1:N4gwH0U3YPwmuyxqH7xBA8j44fTPX+vOEP7ejJVBPts= 3 3 - codeberg.org/git-pages/go-headers v1.1.0 h1:rk7/SOSsn+XuL7PUQZFYUaWKHEaj6K8mXmUV9rF2VxE= 4 4 - codeberg.org/git-pages/go-headers v1.1.0/go.mod h1:N4gwH0U3YPwmuyxqH7xBA8j44fTPX+vOEP7ejJVBPts= 1 1 + codeberg.org/git-pages/go-headers v1.1.1 h1:fpIBELKo66Z2k+gCeYl5mCEXVQ99Lmx1iup1nbo2shE= 2 2 + codeberg.org/git-pages/go-headers v1.1.1/go.mod h1:N4gwH0U3YPwmuyxqH7xBA8j44fTPX+vOEP7ejJVBPts= 3 3 + codeberg.org/git-pages/go-slog-syslog v0.0.0-20251207093707-892f654e80b7 h1:+rkrAxhNZo/eKEcKOqVOsF6ohAPv5amz0JLburOeRjs= 4 4 + codeberg.org/git-pages/go-slog-syslog v0.0.0-20251207093707-892f654e80b7/go.mod h1:8NPSXbYcVb71qqNM5cIgn1/uQgMisLbu2dVD1BNxsUw= 5 5 github.com/KimMachineGun/automemlimit v0.7.5 h1:RkbaC0MwhjL1ZuBKunGDjE/ggwAX43DwZrJqVwyveTk= 6 6 github.com/KimMachineGun/automemlimit v0.7.5/go.mod h1:QZxpHaGOQoYvFhv/r4u3U0JTC2ZcOwbSr11UZF46UBM= 7 7 github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY= ··· 22 22 github.com/cloudflare/circl v1.6.1/go.mod h1:uddAzsPgqdMAYatqJ0lsjX1oECcQLIlRpzZh3pJrofs= 23 23 github.com/creasty/defaults v1.8.0 h1:z27FJxCAa0JKt3utc0sCImAEb+spPucmKoOdLHvHYKk= 24 24 github.com/creasty/defaults v1.8.0/go.mod h1:iGzKe6pbEHnpMPtfDXZEr0NVxWnPTjb1bbDy08fPzYM= 25 25 - github.com/cyphar/filepath-securejoin v0.5.0 h1:hIAhkRBMQ8nIeuVwcAoymp7MY4oherZdAxD+m0u9zaw= 26 26 - github.com/cyphar/filepath-securejoin v0.5.0/go.mod h1:Sdj7gXlvMcPZsbhwhQ33GguGLDGQL7h7bg04C/+u9jI= 25 25 + github.com/cyphar/filepath-securejoin v0.6.1 h1:5CeZ1jPXEiYt3+Z6zqprSAgSWiggmpVyciv8syjIpVE= 26 26 + github.com/cyphar/filepath-securejoin v0.6.1/go.mod h1:A8hd4EnAeyujCJRrICiOWqjS1AX0a9kM5XL+NwKoYSc= 27 27 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 28 28 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= 29 29 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 30 30 + github.com/dghubble/trie v0.1.0 h1:kJnjBLFFElBwS60N4tkPvnLhnpcDxbBjIulgI8CpNGM= 31 31 + github.com/dghubble/trie v0.1.0/go.mod h1:sOmnzfBNH7H92ow2292dDFWNsVQuh/izuD7otCYb1ak= 30 32 github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= 31 33 github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= 32 32 - github.com/elazarl/goproxy v1.7.2 h1:Y2o6urb7Eule09PjlhQRGNsqRfPmYI3KKQLFpCAV3+o= 33 33 - github.com/elazarl/goproxy v1.7.2/go.mod h1:82vkLNir0ALaW14Rc399OTTjyNREgmdL2cVoIbS6XaE= 34 34 github.com/emirpasic/gods v1.18.1 h1:FXtiHYKDGKCW2KzwZKx0iC0PQmdlorYgdFG9jPXJ1Bc= 35 35 github.com/emirpasic/gods v1.18.1/go.mod h1:8tpGGwCnJ5H4r6BWwaV6OrWmMoPhUl5jm/FMNAnJvWQ= 36 36 - github.com/getsentry/sentry-go v0.36.2 h1:uhuxRPTrUy0dnSzTd0LrYXlBYygLkKY0hhlG5LXarzM= 37 37 - github.com/getsentry/sentry-go v0.36.2/go.mod h1:p5Im24mJBeruET8Q4bbcMfCQ+F+Iadc4L48tB1apo2c= 38 38 - github.com/getsentry/sentry-go/slog v0.36.2 h1:PM27JHFE3lsE8fgI/cOueEOtjiktnC3Za2o5oL9PbJQ= 39 39 - github.com/getsentry/sentry-go/slog v0.36.2/go.mod h1:aVFAxnpA3FEtZeSBhBFAnWOlqhiLjaaoOZ0bmBN9IHo= 36 36 + github.com/fatih/color v1.18.0 h1:S8gINlzdQ840/4pfAwic/ZE0djQEH3wM94VfqLTZcOM= 37 37 + github.com/fatih/color v1.18.0/go.mod h1:4FelSpRwEGDpQ12mAdzqdOukCy4u8WUtOY6lkT/6HfU= 38 38 + github.com/getsentry/sentry-go v0.40.0 h1:VTJMN9zbTvqDqPwheRVLcp0qcUcM+8eFivvGocAaSbo= 39 39 + github.com/getsentry/sentry-go v0.40.0/go.mod h1:eRXCoh3uvmjQLY6qu63BjUZnaBu5L5WhMV1RwYO8W5s= 40 40 + github.com/getsentry/sentry-go/slog v0.40.0 h1:uR2EPL9w6uHw3XB983IAqzqM9mP+fjJpNY9kfob3/Z8= 41 41 + github.com/getsentry/sentry-go/slog v0.40.0/go.mod h1:ArRaP+0rsbnJGyvZwYDo/vDQT/YBbOQeOlO+DGW+F9s= 40 42 github.com/gliderlabs/ssh v0.3.8 h1:a4YXD1V7xMF9g5nTkdfnja3Sxy1PVDCj1Zg4Wb8vY6c= 41 43 github.com/gliderlabs/ssh v0.3.8/go.mod h1:xYoytBv1sV0aL3CavoDuJIQNURXkkfPA/wxQ1pL1fAU= 42 44 github.com/go-errors/errors v1.4.2 h1:J6MZopCL4uSllY1OfXM374weqZFFItUbrImctkmUxIA= 43 45 github.com/go-errors/errors v1.4.2/go.mod h1:sIVyrIiJhuEF+Pj9Ebtd6P/rEYROXFi3BopGUQ5a5Og= 44 46 github.com/go-git/gcfg/v2 v2.0.2 h1:MY5SIIfTGGEMhdA7d7JePuVVxtKL7Hp+ApGDJAJ7dpo= 45 47 github.com/go-git/gcfg/v2 v2.0.2/go.mod h1:/lv2NsxvhepuMrldsFilrgct6pxzpGdSRC13ydTLSLs= 46 46 - github.com/go-git/go-billy/v6 v6.0.0-20251026101908-623011986e70 h1:TWpNrg9JPxp0q+KG0hoFGBulPIP/kMK1b0mDqjdEB/s= 47 47 - github.com/go-git/go-billy/v6 v6.0.0-20251026101908-623011986e70/go.mod h1:TpCYxdQ0tWZkrnAkd7yqK+z1C8RKcyjcaYAJNAcnUnM= 48 48 - github.com/go-git/go-git-fixtures/v5 v5.1.1 h1:OH8i1ojV9bWfr0ZfasfpgtUXQHQyVS8HXik/V1C099w= 49 49 - github.com/go-git/go-git-fixtures/v5 v5.1.1/go.mod h1:Altk43lx3b1ks+dVoAG2300o5WWUnktvfY3VI6bcaXU= 50 50 - github.com/go-git/go-git/v6 v6.0.0-20251029213217-0bbfc0875edd h1:pn6+tR4O8McyqEr2MbQwqcySovpG8jDd11F/jQ6aAfA= 51 51 - github.com/go-git/go-git/v6 v6.0.0-20251029213217-0bbfc0875edd/go.mod h1:z9pQiXCfyOZIs/8qa5zmozzbcsDPtGN91UD7+qeX3hk= 48 48 + github.com/go-git/go-billy/v6 v6.0.0-20251217170237-e9738f50a3cd h1:Gd/f9cGi/3h1JOPaa6er+CkKUGyGX2DBJdFbDKVO+R0= 49 49 + github.com/go-git/go-billy/v6 v6.0.0-20251217170237-e9738f50a3cd/go.mod h1:d3XQcsHu1idnquxt48kAv+h+1MUiYKLH/e7LAzjP+pI= 50 50 + github.com/go-git/go-git-fixtures/v5 v5.1.2-0.20251229094738-4b14af179146 h1:xYfxAopYyL44ot6dMBIb1Z1njFM0ZBQ99HdIB99KxLs= 51 51 + github.com/go-git/go-git-fixtures/v5 v5.1.2-0.20251229094738-4b14af179146/go.mod h1:QE/75B8tBSLNGyUUbA9tw3EGHoFtYOtypa2h8YJxsWI= 52 52 + github.com/go-git/go-git/v6 v6.0.0-20251231065035-29ae690a9f19 h1:0lz2eJScP8v5YZQsrEw+ggWC5jNySjg4bIZo5BIh6iI= 53 53 + github.com/go-git/go-git/v6 v6.0.0-20251231065035-29ae690a9f19/go.mod h1:L+Evfcs7EdTqxwv854354cb6+++7TFL3hJn3Wy4g+3w= 52 54 github.com/go-ini/ini v1.67.0 h1:z6ZrTEZqSWOTyH2FlglNbNgARyHG8oLW9gMELqKr06A= 53 55 github.com/go-ini/ini v1.67.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8= 54 54 - github.com/goccy/go-json v0.10.5 h1:Fq85nIqj+gXn/S5ahsiTlK3TmC85qgirsdTP/+DeaC4= 55 55 - github.com/goccy/go-json v0.10.5/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M= 56 56 github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8 h1:f+oWsMOmNPc8JmEHVZIycC7hBoQxHH9pNKQORJNozsQ= 57 57 github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8/go.mod h1:wcDNUvekVysuuOpQKo3191zZyTpiI6se1N1ULghS0sw= 58 58 github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= 59 59 github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= 60 60 github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= 61 61 github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= 62 62 + github.com/jpillora/backoff v1.0.0 h1:uvFg412JmmHBHw7iwprIxkPMI+sGQ4kzOWsMeHnm2EA= 63 63 + github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= 64 64 + github.com/kankanreno/go-snowflake v1.2.0 h1:Zx2SctsH5pivIj9vyhwyDyQS23jcDJx4iT49Bjv81kk= 65 65 + github.com/kankanreno/go-snowflake v1.2.0/go.mod h1:6CZ+10PeVsFXKZUTYyJzPiRIjn1IXbInaWLCX/LDJ0g= 62 66 github.com/kevinburke/ssh_config v1.4.0 h1:6xxtP5bZ2E4NF5tuQulISpTO2z8XbtH8cg1PWkxoFkQ= 63 67 github.com/kevinburke/ssh_config v1.4.0/go.mod h1:q2RIzfka+BXARoNexmF9gkxEX7DmvbW9P4hIVx2Kg4M= 64 64 - github.com/klauspost/compress v1.18.1 h1:bcSGx7UbpBqMChDtsF28Lw6v/G94LPrrbMbdC3JH2co= 65 65 - github.com/klauspost/compress v1.18.1/go.mod h1:ZQFFVG+MdnR0P+l6wpXgIL4NTtwiKIdBnrBd8Nrxr+0= 68 68 + github.com/klauspost/compress v1.18.2 h1:iiPHWW0YrcFgpBYhsA6D1+fqHssJscY/Tm/y2Uqnapk= 69 69 + github.com/klauspost/compress v1.18.2/go.mod h1:R0h/fSBs8DE4ENlcrlib3PsXS61voFxhIs2DeRhCvJ4= 66 70 github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= 67 71 github.com/klauspost/cpuid/v2 v2.3.0 h1:S4CRMLnYUhGeDFDqkGriYKdfoFlDnMtqTiI/sFzhA9Y= 68 72 github.com/klauspost/cpuid/v2 v2.3.0/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0= 73 73 + github.com/klauspost/crc32 v1.3.0 h1:sSmTt3gUt81RP655XGZPElI0PelVTZ6YwCRnPSupoFM= 74 74 + github.com/klauspost/crc32 v1.3.0/go.mod h1:D7kQaZhnkX/Y0tstFGf8VUzv2UofNGqCjnC3zdHB0Hw= 69 75 github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= 70 76 github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= 71 77 github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= ··· 75 81 github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= 76 82 github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= 77 83 github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= 78 78 - github.com/maypok86/otter/v2 v2.2.1 h1:hnGssisMFkdisYcvQ8L019zpYQcdtPse+g0ps2i7cfI= 79 79 - github.com/maypok86/otter/v2 v2.2.1/go.mod h1:1NKY9bY+kB5jwCXBJfE59u+zAwOt6C7ni1FTlFFMqVs= 80 80 - github.com/minio/crc64nvme v1.0.2 h1:6uO1UxGAD+kwqWWp7mBFsi5gAse66C4NXO8cmcVculg= 81 81 - github.com/minio/crc64nvme v1.0.2/go.mod h1:eVfm2fAzLlxMdUGc0EEBGSMmPwmXD5XiNRpnu9J3bvg= 84 84 + github.com/leodido/go-syslog/v4 v4.3.0 h1:bbSpI/41bYK9iSdlYzcwvlxuLOE8yi4VTFmedtnghdA= 85 85 + github.com/leodido/go-syslog/v4 v4.3.0/go.mod h1:eJ8rUfDN5OS6dOkCOBYlg2a+hbAg6pJa99QXXgMrd98= 86 86 + github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= 87 87 + github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= 88 88 + github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= 89 89 + github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= 90 90 + github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= 91 91 + github.com/maypok86/otter/v2 v2.3.0 h1:8H8AVVFUSzJwIegKwv1uF5aGitTY+AIrtktg7OcLs8w= 92 92 + github.com/maypok86/otter/v2 v2.3.0/go.mod h1:XgIdlpmL6jYz882/CAx1E4C1ukfgDKSaw4mWq59+7l8= 93 93 + github.com/minio/crc64nvme v1.1.0 h1:e/tAguZ+4cw32D+IO/8GSf5UVr9y+3eJcxZI2WOO/7Q= 94 94 + github.com/minio/crc64nvme v1.1.0/go.mod h1:eVfm2fAzLlxMdUGc0EEBGSMmPwmXD5XiNRpnu9J3bvg= 82 95 github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34= 83 96 github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM= 84 84 - github.com/minio/minio-go/v7 v7.0.95 h1:ywOUPg+PebTMTzn9VDsoFJy32ZuARN9zhB+K3IYEvYU= 85 85 - github.com/minio/minio-go/v7 v7.0.95/go.mod h1:wOOX3uxS334vImCNRVyIDdXX9OsXDm89ToynKgqUKlo= 97 97 + github.com/minio/minio-go/v7 v7.0.97 h1:lqhREPyfgHTB/ciX8k2r8k0D93WaFqxbJX36UZq5occ= 98 98 + github.com/minio/minio-go/v7 v7.0.97/go.mod h1:re5VXuo0pwEtoNLsNuSr0RrLfT/MBtohwdaSmPPSRSk= 86 99 github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= 87 100 github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= 88 101 github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58 h1:onHthvaw9LFnH4t2DcNVpwGmV9E1BkGknEliJkfwQj0= ··· 113 126 github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= 114 127 github.com/rs/xid v1.6.0 h1:fV591PaemRlL6JfRxGDEPl69wICngIQ3shQtzfy2gxU= 115 128 github.com/rs/xid v1.6.0/go.mod h1:7XoLgs4eV+QndskICGsho+ADou8ySMSjJKDIan90Nz0= 116 116 - github.com/samber/lo v1.51.0 h1:kysRYLbHy/MB7kQZf5DSN50JHmMsNEdeY24VzJFu7wI= 117 117 - github.com/samber/lo v1.51.0/go.mod h1:4+MXEGsJzbKGaUEQFKBq2xtfuznW9oz/WrgyzMzRoM0= 129 129 + github.com/samber/lo v1.52.0 h1:Rvi+3BFHES3A8meP33VPAxiBZX/Aws5RxrschYGjomw= 130 130 + github.com/samber/lo v1.52.0/go.mod h1:4+MXEGsJzbKGaUEQFKBq2xtfuznW9oz/WrgyzMzRoM0= 118 131 github.com/samber/slog-common v0.19.0 h1:fNcZb8B2uOLooeYwFpAlKjkQTUafdjfqKcwcC89G9YI= 119 132 github.com/samber/slog-common v0.19.0/go.mod h1:dTz+YOU76aH007YUU0DffsXNsGFQRQllPQh9XyNoA3M= 120 120 - github.com/samber/slog-multi v1.5.0 h1:UDRJdsdb0R5vFQFy3l26rpX3rL3FEPJTJ2yKVjoiT1I= 121 121 - github.com/samber/slog-multi v1.5.0/go.mod h1:im2Zi3mH/ivSY5XDj6LFcKToRIWPw1OcjSVSdXt+2d0= 133 133 + github.com/samber/slog-multi v1.7.0 h1:GKhbkxU3ujkyMsefkuz4qvE6EcgtSuqjFisPnfdzVLI= 134 134 + github.com/samber/slog-multi v1.7.0/go.mod h1:qTqzmKdPpT0h4PFsTN5rYRgLwom1v+fNGuIrl1Xnnts= 122 135 github.com/sergi/go-diff v1.4.0 h1:n/SP9D5ad1fORl+llWyN+D6qoUETXNZARKjyY2/KVCw= 123 136 github.com/sergi/go-diff v1.4.0/go.mod h1:A0bzQcvG0E7Rwjx0REVgAGH58e96+X0MeOfepqsbeW4= 124 137 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= ··· 140 153 go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= 141 154 go.yaml.in/yaml/v2 v2.4.2 h1:DzmwEr2rDGHl7lsFgAHxmNz/1NlQ7xLIrlN2h5d1eGI= 142 155 go.yaml.in/yaml/v2 v2.4.2/go.mod h1:081UH+NErpNdqlCXm3TtEran0rJZGxAYx9hb/ELlsPU= 143 143 - golang.org/x/crypto v0.43.0 h1:dduJYIi3A3KOfdGOHX8AVZ/jGiyPa3IbBozJ5kNuE04= 144 144 - golang.org/x/crypto v0.43.0/go.mod h1:BFbav4mRNlXJL4wNeejLpWxB7wMbc79PdRGhWKncxR0= 145 145 - golang.org/x/net v0.46.0 h1:giFlY12I07fugqwPuWJi68oOnpfqFnJIJzaIIm2JVV4= 146 146 - golang.org/x/net v0.46.0/go.mod h1:Q9BGdFy1y4nkUwiLvT5qtyhAnEHgnQ/zd8PfU6nc210= 147 147 - golang.org/x/sys v0.37.0 h1:fdNQudmxPjkdUTPnLn5mdQv7Zwvbvpaxqs831goi9kQ= 148 148 - golang.org/x/sys v0.37.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= 149 149 - golang.org/x/term v0.36.0 h1:zMPR+aF8gfksFprF/Nc/rd1wRS1EI6nDBGyWAvDzx2Q= 150 150 - golang.org/x/term v0.36.0/go.mod h1:Qu394IJq6V6dCBRgwqshf3mPF85AqzYEzofzRdZkWss= 151 151 - golang.org/x/text v0.30.0 h1:yznKA/E9zq54KzlzBEAWn1NXSQ8DIp/NYMy88xJjl4k= 152 152 - golang.org/x/text v0.30.0/go.mod h1:yDdHFIX9t+tORqspjENWgzaCVXgk0yYnYuSZ8UzzBVM= 153 153 - google.golang.org/protobuf v1.36.10 h1:AYd7cD/uASjIL6Q9LiTjz8JLcrh/88q5UObnmY3aOOE= 154 154 - google.golang.org/protobuf v1.36.10/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= 156 156 + golang.org/x/crypto v0.46.0 h1:cKRW/pmt1pKAfetfu+RCEvjvZkA9RimPbh7bhFjGVBU= 157 157 + golang.org/x/crypto v0.46.0/go.mod h1:Evb/oLKmMraqjZ2iQTwDwvCtJkczlDuTmdJXoZVzqU0= 158 158 + golang.org/x/net v0.48.0 h1:zyQRTTrjc33Lhh0fBgT/H3oZq9WuvRR5gPC70xpDiQU= 159 159 + golang.org/x/net v0.48.0/go.mod h1:+ndRgGjkh8FGtu1w1FGbEC31if4VrNVMuKTgcAAnQRY= 160 160 + golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 161 161 + golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 162 162 + golang.org/x/sys v0.39.0 h1:CvCKL8MeisomCi6qNZ+wbb0DN9E5AATixKsvNtMoMFk= 163 163 + golang.org/x/sys v0.39.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= 164 164 + golang.org/x/term v0.38.0 h1:PQ5pkm/rLO6HnxFR7N2lJHOZX6Kez5Y1gDSJla6jo7Q= 165 165 + golang.org/x/term v0.38.0/go.mod h1:bSEAKrOT1W+VSu9TSCMtoGEOUcKxOKgl3LE5QEF/xVg= 166 166 + golang.org/x/text v0.32.0 h1:ZD01bjUt1FQ9WJ0ClOL5vxgxOI/sVCNgX1YtKwcY0mU= 167 167 + golang.org/x/text v0.32.0/go.mod h1:o/rUWzghvpD5TXrTIBuJU77MTaN0ljMWE47kxGJQ7jY= 168 168 + google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE= 169 169 + google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= 155 170 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 156 171 gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 157 172 gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=

+204

gomod2nix.toml

··· 1 1 + schema = 3 2 2 + 3 3 + [mod] 4 4 + [mod."codeberg.org/git-pages/go-headers"] 5 5 + version = "v1.1.1" 6 6 + hash = "sha256-qgL7l1FHXxcBWhBnBLEI0yENd6P+frvwlKxEAXLA3VY=" 7 7 + [mod."codeberg.org/git-pages/go-slog-syslog"] 8 8 + version = "v0.0.0-20251207093707-892f654e80b7" 9 9 + hash = "sha256-ye+DBIyxqTEOViYRrQPWyGJCaLmyKSDwH5btlqDPizM=" 10 10 + [mod."github.com/KimMachineGun/automemlimit"] 11 11 + version = "v0.7.5" 12 12 + hash = "sha256-lH/ip9j2hbYUc2W/XIYve/5TScQPZtEZe3hu76CY//k=" 13 13 + [mod."github.com/Microsoft/go-winio"] 14 14 + version = "v0.6.2" 15 15 + hash = "sha256-tVNWDUMILZbJvarcl/E7tpSnkn7urqgSHa2Eaka5vSU=" 16 16 + [mod."github.com/ProtonMail/go-crypto"] 17 17 + version = "v1.3.0" 18 18 + hash = "sha256-TUG+C4MyeWglOmiwiW2/NUVurFHXLgEPRd3X9uQ1NGI=" 19 19 + [mod."github.com/beorn7/perks"] 20 20 + version = "v1.0.1" 21 21 + hash = "sha256-h75GUqfwJKngCJQVE5Ao5wnO3cfKD9lSIteoLp/3xJ4=" 22 22 + [mod."github.com/c2h5oh/datasize"] 23 23 + version = "v0.0.0-20231215233829-aa82cc1e6500" 24 24 + hash = "sha256-8MqL7xCvE6fIjanz2jwkaLP1OE5kLu62TOcQx452DHQ=" 25 25 + [mod."github.com/cespare/xxhash/v2"] 26 26 + version = "v2.3.0" 27 27 + hash = "sha256-7hRlwSR+fos1kx4VZmJ/7snR7zHh8ZFKX+qqqqGcQpY=" 28 28 + [mod."github.com/cloudflare/circl"] 29 29 + version = "v1.6.1" 30 30 + hash = "sha256-Dc69V12eIFnJoUNmwg6VKXHfAMijbAeEVSDe8AiOaLo=" 31 31 + [mod."github.com/creasty/defaults"] 32 32 + version = "v1.8.0" 33 33 + hash = "sha256-I1LE1cfOhMS5JxB7+fWTKieefw2Gge1UhIZh+A6pa6s=" 34 34 + [mod."github.com/cyphar/filepath-securejoin"] 35 35 + version = "v0.6.1" 36 36 + hash = "sha256-obqip8c1c9mjXFznyXF8aDnpcMw7ttzv+e28anCa/v0=" 37 37 + [mod."github.com/davecgh/go-spew"] 38 38 + version = "v1.1.1" 39 39 + hash = "sha256-nhzSUrE1fCkN0+RL04N4h8jWmRFPPPWbCuDc7Ss0akI=" 40 40 + [mod."github.com/dghubble/trie"] 41 41 + version = "v0.1.0" 42 42 + hash = "sha256-hVh7uYylpMCCSPcxl70hJTmzSwaA1MxBmJFBO5Xdncc=" 43 43 + [mod."github.com/dustin/go-humanize"] 44 44 + version = "v1.0.1" 45 45 + hash = "sha256-yuvxYYngpfVkUg9yAmG99IUVmADTQA0tMbBXe0Fq0Mc=" 46 46 + [mod."github.com/emirpasic/gods"] 47 47 + version = "v1.18.1" 48 48 + hash = "sha256-hGDKddjLj+5dn2woHtXKUdd49/3xdsqnhx7VEdCu1m4=" 49 49 + [mod."github.com/fatih/color"] 50 50 + version = "v1.18.0" 51 51 + hash = "sha256-pP5y72FSbi4j/BjyVq/XbAOFjzNjMxZt2R/lFFxGWvY=" 52 52 + [mod."github.com/getsentry/sentry-go"] 53 53 + version = "v0.40.0" 54 54 + hash = "sha256-mJ+EzM8WRzJ2Yp7ithDJNceU4+GbzQyi46yc8J8d13Y=" 55 55 + [mod."github.com/getsentry/sentry-go/slog"] 56 56 + version = "v0.40.0" 57 57 + hash = "sha256-uc9TpKiWMEpRbxwV2uGQeq1DDdZi+APOgu2StVzzEkw=" 58 58 + [mod."github.com/go-git/gcfg/v2"] 59 59 + version = "v2.0.2" 60 60 + hash = "sha256-icqMDeC/tEg/3979EuEN67Ml5KjdDA0R3QvR6iLLrSI=" 61 61 + [mod."github.com/go-git/go-billy/v6"] 62 62 + version = "v6.0.0-20251217170237-e9738f50a3cd" 63 63 + hash = "sha256-b2yunYcPUiLTU+Rr8qTBdsDEfsIhZDYmyqKW5udmpFY=" 64 64 + [mod."github.com/go-git/go-git/v6"] 65 65 + version = "v6.0.0-20251224103503-78aff6aa5ea9" 66 66 + hash = "sha256-kYjDqH0NZ+sxQnj5K8xKfO2WOVKtQ/7tWcqY6KYqAZE=" 67 67 + [mod."github.com/go-ini/ini"] 68 68 + version = "v1.67.0" 69 69 + hash = "sha256-V10ahGNGT+NLRdKUyRg1dos5RxLBXBk1xutcnquc/+4=" 70 70 + [mod."github.com/golang/groupcache"] 71 71 + version = "v0.0.0-20241129210726-2c02b8208cf8" 72 72 + hash = "sha256-AdLZ3dJLe/yduoNvZiXugZxNfmwJjNQyQGsIdzYzH74=" 73 73 + [mod."github.com/google/uuid"] 74 74 + version = "v1.6.0" 75 75 + hash = "sha256-VWl9sqUzdOuhW0KzQlv0gwwUQClYkmZwSydHG2sALYw=" 76 76 + [mod."github.com/jpillora/backoff"] 77 77 + version = "v1.0.0" 78 78 + hash = "sha256-uxHg68NN8hrwPCrPfLYYprZHf7dMyEoPoF46JFx0IHU=" 79 79 + [mod."github.com/kankanreno/go-snowflake"] 80 80 + version = "v1.2.0" 81 81 + hash = "sha256-713xGEqjwaUGIu2EHII5sldWmcquFpxZmte/7R/O6LA=" 82 82 + [mod."github.com/kevinburke/ssh_config"] 83 83 + version = "v1.4.0" 84 84 + hash = "sha256-UclxB7Ll1FZCgU2SrGkiGdr4CoSRJ127MNnZtxKTsvg=" 85 85 + [mod."github.com/klauspost/compress"] 86 86 + version = "v1.18.2" 87 87 + hash = "sha256-mRa+6qEi5joqQao13ZFogmq67rOQzHCVbCCjKA+HKEc=" 88 88 + [mod."github.com/klauspost/cpuid/v2"] 89 89 + version = "v2.3.0" 90 90 + hash = "sha256-50JhbQyT67BK38HIdJihPtjV7orYp96HknI2VP7A9Yc=" 91 91 + [mod."github.com/klauspost/crc32"] 92 92 + version = "v1.3.0" 93 93 + hash = "sha256-RsS/MDJbVzVB+i74whqABgwZJWMw+AutF6HhJBVgbag=" 94 94 + [mod."github.com/leodido/go-syslog/v4"] 95 95 + version = "v4.3.0" 96 96 + hash = "sha256-fCJ2rgrrPR/Ey/PoAsJhd8Sl8mblAnnMAmBuoWFBTgg=" 97 97 + [mod."github.com/mattn/go-colorable"] 98 98 + version = "v0.1.13" 99 99 + hash = "sha256-qb3Qbo0CELGRIzvw7NVM1g/aayaz4Tguppk9MD2/OI8=" 100 100 + [mod."github.com/mattn/go-isatty"] 101 101 + version = "v0.0.20" 102 102 + hash = "sha256-qhw9hWtU5wnyFyuMbKx+7RB8ckQaFQ8D+8GKPkN3HHQ=" 103 103 + [mod."github.com/maypok86/otter/v2"] 104 104 + version = "v2.3.0" 105 105 + hash = "sha256-ELzmi/s2WqDeUmzSGnfx+ys2Hs28XHqF7vlEzyRotIA=" 106 106 + [mod."github.com/minio/crc64nvme"] 107 107 + version = "v1.1.0" 108 108 + hash = "sha256-OwlE70X91WO4HdbpGsOaB4w12Qrk0duCpfLeAskiqY8=" 109 109 + [mod."github.com/minio/md5-simd"] 110 110 + version = "v1.1.2" 111 111 + hash = "sha256-vykcXvy2VBBAXnJott/XsGTT0gk2UL36JzZKfJ1KAUY=" 112 112 + [mod."github.com/minio/minio-go/v7"] 113 113 + version = "v7.0.97" 114 114 + hash = "sha256-IwF14tWVYjBi28jUG9iFYd4Lpbc7Fvyy0zRzEZ82UEE=" 115 115 + [mod."github.com/munnerz/goautoneg"] 116 116 + version = "v0.0.0-20191010083416-a7dc8b61c822" 117 117 + hash = "sha256-79URDDFenmGc9JZu+5AXHToMrtTREHb3BC84b/gym9Q=" 118 118 + [mod."github.com/pbnjay/memory"] 119 119 + version = "v0.0.0-20210728143218-7b4eea64cf58" 120 120 + hash = "sha256-QI+F1oPLOOtwNp8+m45OOoSfYFs3QVjGzE0rFdpF/IA=" 121 121 + [mod."github.com/pelletier/go-toml/v2"] 122 122 + version = "v2.2.4" 123 123 + hash = "sha256-8qQIPldbsS5RO8v/FW/se3ZsAyvLzexiivzJCbGRg2Q=" 124 124 + [mod."github.com/philhofer/fwd"] 125 125 + version = "v1.2.0" 126 126 + hash = "sha256-cGx2/0QQay46MYGZuamFmU0TzNaFyaO+J7Ddzlr/3dI=" 127 127 + [mod."github.com/pjbgf/sha1cd"] 128 128 + version = "v0.5.0" 129 129 + hash = "sha256-11XBkhdciQGsQ7jEMZ6PgphRKjruTSc7ZxfOwDuPCr8=" 130 130 + [mod."github.com/pkg/errors"] 131 131 + version = "v0.9.1" 132 132 + hash = "sha256-mNfQtcrQmu3sNg/7IwiieKWOgFQOVVe2yXgKBpe/wZw=" 133 133 + [mod."github.com/pmezard/go-difflib"] 134 134 + version = "v1.0.0" 135 135 + hash = "sha256-/FtmHnaGjdvEIKAJtrUfEhV7EVo5A/eYrtdnUkuxLDA=" 136 136 + [mod."github.com/pquerna/cachecontrol"] 137 137 + version = "v0.2.0" 138 138 + hash = "sha256-tuTERCFfwmqPepw/rs5cyv9fArCD30BqgjZqwMV+vzQ=" 139 139 + [mod."github.com/prometheus/client_golang"] 140 140 + version = "v1.23.2" 141 141 + hash = "sha256-3GD4fBFa1tJu8MS4TNP6r2re2eViUE+kWUaieIOQXCg=" 142 142 + [mod."github.com/prometheus/client_model"] 143 143 + version = "v0.6.2" 144 144 + hash = "sha256-q6Fh6v8iNJN9ypD47LjWmx66YITa3FyRjZMRsuRTFeQ=" 145 145 + [mod."github.com/prometheus/common"] 146 146 + version = "v0.66.1" 147 147 + hash = "sha256-bqHPaV9IV70itx63wqwgy2PtxMN0sn5ThVxDmiD7+Tk=" 148 148 + [mod."github.com/prometheus/procfs"] 149 149 + version = "v0.16.1" 150 150 + hash = "sha256-OBCvKlLW2obct35p0L9Q+1ZrxZjpTmbgHMP2rng9hpo=" 151 151 + [mod."github.com/rs/xid"] 152 152 + version = "v1.6.0" 153 153 + hash = "sha256-rJB7h3KuH1DPp5n4dY3MiGnV1Y96A10lf5OUl+MLkzU=" 154 154 + [mod."github.com/samber/lo"] 155 155 + version = "v1.52.0" 156 156 + hash = "sha256-xgMsPJv3rydHH10NZU8wz/DhK2VbbR8ymivOg1ChTp0=" 157 157 + [mod."github.com/samber/slog-common"] 158 158 + version = "v0.19.0" 159 159 + hash = "sha256-OYXVbZML7c3mFClVy8GEnNoWW+4OfcBsxWDtKh1u7B8=" 160 160 + [mod."github.com/samber/slog-multi"] 161 161 + version = "v1.6.0" 162 162 + hash = "sha256-uebbTcvsBP2LdOUIjDptES+HZOXxThnIt3+FKL0qJy4=" 163 163 + [mod."github.com/sergi/go-diff"] 164 164 + version = "v1.4.0" 165 165 + hash = "sha256-rs9NKpv/qcQEMRg7CmxGdP4HGuFdBxlpWf9LbA9wS4k=" 166 166 + [mod."github.com/stretchr/testify"] 167 167 + version = "v1.11.1" 168 168 + hash = "sha256-sWfjkuKJyDllDEtnM8sb/pdLzPQmUYWYtmeWz/5suUc=" 169 169 + [mod."github.com/tinylib/msgp"] 170 170 + version = "v1.3.0" 171 171 + hash = "sha256-PnpndO7k5Yl036vhWJGDsrcz0jsTX8sUiTqm/D3rAVw=" 172 172 + [mod."github.com/tj/assert"] 173 173 + version = "v0.0.3" 174 174 + hash = "sha256-4xhmZcHpUafabaXejE9ucVnGxG/txomvKzBg6cbkusg=" 175 175 + [mod."github.com/tj/go-redirects"] 176 176 + version = "v0.0.0-20200911105812-fd1ba1020b37" 177 177 + hash = "sha256-GpYpxdT4F7PkwGXLo7cYVcIRJrzd1sKHtFDH+bRb6Tk=" 178 178 + [mod."github.com/valyala/bytebufferpool"] 179 179 + version = "v1.0.0" 180 180 + hash = "sha256-I9FPZ3kCNRB+o0dpMwBnwZ35Fj9+ThvITn8a3Jr8mAY=" 181 181 + [mod."github.com/valyala/fasttemplate"] 182 182 + version = "v1.2.2" 183 183 + hash = "sha256-gp+lNXE8zjO+qJDM/YbS6V43HFsYP6PKn4ux1qa5lZ0=" 184 184 + [mod."go.yaml.in/yaml/v2"] 185 185 + version = "v2.4.2" 186 186 + hash = "sha256-oC8RWdf1zbMYCtmR0ATy/kCkhIwPR9UqFZSMOKLVF/A=" 187 187 + [mod."golang.org/x/crypto"] 188 188 + version = "v0.46.0" 189 189 + hash = "sha256-I8N/spcw3/h0DFA+V1WK38HctckWIB9ep93DEVCALxU=" 190 190 + [mod."golang.org/x/net"] 191 191 + version = "v0.48.0" 192 192 + hash = "sha256-oZpddsiJwWCH3Aipa+XXpy7G/xHY5fEagUSok7T0bXE=" 193 193 + [mod."golang.org/x/sys"] 194 194 + version = "v0.39.0" 195 195 + hash = "sha256-dxTBu/JAWUkPbjFIXXRFdhQWyn+YyEpIC+tWqGo0Y6U=" 196 196 + [mod."golang.org/x/text"] 197 197 + version = "v0.32.0" 198 198 + hash = "sha256-9PXtWBKKY9rG4AgjSP4N+I1DhepXhy8SF/vWSIDIoWs=" 199 199 + [mod."google.golang.org/protobuf"] 200 200 + version = "v1.36.11" 201 201 + hash = "sha256-7W+6jntfI/awWL3JP6yQedxqP5S9o3XvPgJ2XxxsIeE=" 202 202 + [mod."gopkg.in/yaml.v3"] 203 203 + version = "v3.0.1" 204 204 + hash = "sha256-FqL9TKYJ0XkNwJFnq9j0VvJ5ZUU1RvH/52h/f5bkYAU="

+3 -1

renovate.json

··· 14 14 "lockFileMaintenance": { 15 15 "enabled": true, 16 16 "automerge": false 17 17 - } 17 17 + }, 18 18 + "semanticCommits": "disabled", 19 19 + "commitMessagePrefix": "[Renovate]" 18 20 }

+393

src/audit.go

··· 1 1 + package git_pages 2 2 + 3 3 + import ( 4 4 + "cmp" 5 5 + "context" 6 6 + "fmt" 7 7 + "io" 8 8 + "net/http" 9 9 + "os" 10 10 + "os/exec" 11 11 + "path" 12 12 + "path/filepath" 13 13 + "strconv" 14 14 + "strings" 15 15 + "time" 16 16 + 17 17 + exponential "github.com/jpillora/backoff" 18 18 + "github.com/kankanreno/go-snowflake" 19 19 + "github.com/prometheus/client_golang/prometheus" 20 20 + "github.com/prometheus/client_golang/prometheus/promauto" 21 21 + "google.golang.org/protobuf/encoding/protojson" 22 22 + "google.golang.org/protobuf/proto" 23 23 + timestamppb "google.golang.org/protobuf/types/known/timestamppb" 24 24 + ) 25 25 + 26 26 + var ( 27 27 + auditNotifyOkCount = promauto.NewCounter(prometheus.CounterOpts{ 28 28 + Name: "git_pages_audit_notify_ok", 29 29 + Help: "Count of successful audit notifications", 30 30 + }) 31 31 + auditNotifyErrorCount = promauto.NewCounter(prometheus.CounterOpts{ 32 32 + Name: "git_pages_audit_notify_error", 33 33 + Help: "Count of failed audit notifications", 34 34 + }) 35 35 + ) 36 36 + 37 37 + type principalKey struct{} 38 38 + 39 39 + var PrincipalKey = principalKey{} 40 40 + 41 41 + func WithPrincipal(ctx context.Context) context.Context { 42 42 + principal := &Principal{} 43 43 + return context.WithValue(ctx, PrincipalKey, principal) 44 44 + } 45 45 + 46 46 + func GetPrincipal(ctx context.Context) *Principal { 47 47 + if principal, ok := ctx.Value(PrincipalKey).(*Principal); ok { 48 48 + return principal 49 49 + } 50 50 + return nil 51 51 + } 52 52 + 53 53 + type AuditID int64 54 54 + 55 55 + func GenerateAuditID() AuditID { 56 56 + inner, err := snowflake.NextID() 57 57 + if err != nil { 58 58 + panic(err) 59 59 + } 60 60 + return AuditID(inner) 61 61 + } 62 62 + 63 63 + func ParseAuditID(repr string) (AuditID, error) { 64 64 + inner, err := strconv.ParseInt(repr, 16, 64) 65 65 + if err != nil { 66 66 + return AuditID(0), err 67 67 + } 68 68 + return AuditID(inner), nil 69 69 + } 70 70 + 71 71 + func (id AuditID) String() string { 72 72 + return fmt.Sprintf("%016x", int64(id)) 73 73 + } 74 74 + 75 75 + func (id AuditID) CompareTime(when time.Time) int { 76 76 + idMillis := int64(id) >> (snowflake.MachineIDLength + snowflake.SequenceLength) 77 77 + whenMillis := when.UTC().UnixNano() / 1e6 78 78 + return cmp.Compare(idMillis, whenMillis) 79 79 + } 80 80 + 81 81 + func EncodeAuditRecord(record *AuditRecord) (data []byte) { 82 82 + data, err := proto.MarshalOptions{Deterministic: true}.Marshal(record) 83 83 + if err != nil { 84 84 + panic(err) 85 85 + } 86 86 + return 87 87 + } 88 88 + 89 89 + func DecodeAuditRecord(data []byte) (record *AuditRecord, err error) { 90 90 + record = &AuditRecord{} 91 91 + err = proto.Unmarshal(data, record) 92 92 + return 93 93 + } 94 94 + 95 95 + func (record *AuditRecord) GetAuditID() AuditID { 96 96 + return AuditID(record.GetId()) 97 97 + } 98 98 + 99 99 + func (record *AuditRecord) DescribePrincipal() string { 100 100 + var items []string 101 101 + if record.Principal != nil { 102 102 + if record.Principal.GetIpAddress() != "" { 103 103 + items = append(items, record.Principal.GetIpAddress()) 104 104 + } 105 105 + if record.Principal.GetCliAdmin() { 106 106 + items = append(items, "<cli-admin>") 107 107 + } 108 108 + } 109 109 + if len(items) > 0 { 110 110 + return strings.Join(items, ";") 111 111 + } else { 112 112 + return "<unknown>" 113 113 + } 114 114 + } 115 115 + 116 116 + func (record *AuditRecord) DescribeResource() string { 117 117 + desc := "<unknown>" 118 118 + if record.Domain != nil && record.Project != nil { 119 119 + desc = path.Join(*record.Domain, *record.Project) 120 120 + } else if record.Domain != nil { 121 121 + desc = *record.Domain 122 122 + } 123 123 + return desc 124 124 + } 125 125 + 126 126 + type AuditRecordScope int 127 127 + 128 128 + const ( 129 129 + AuditRecordComplete AuditRecordScope = iota 130 130 + AuditRecordNoManifest 131 131 + ) 132 132 + 133 133 + func AuditRecordJSON(record *AuditRecord, scope AuditRecordScope) []byte { 134 134 + switch scope { 135 135 + case AuditRecordComplete: 136 136 + // as-is 137 137 + case AuditRecordNoManifest: 138 138 + // trim the manifest 139 139 + newRecord := &AuditRecord{} 140 140 + proto.Merge(newRecord, record) 141 141 + newRecord.Manifest = nil 142 142 + record = newRecord 143 143 + } 144 144 + 145 145 + json, err := protojson.MarshalOptions{ 146 146 + Multiline: true, 147 147 + EmitDefaultValues: true, 148 148 + }.Marshal(record) 149 149 + if err != nil { 150 150 + panic(err) 151 151 + } 152 152 + return json 153 153 + } 154 154 + 155 155 + // This function receives `id` and `record` separately because the record itself may have its 156 156 + // ID missing or mismatched. While this is very unlikely, using the actual primary key as 157 157 + // the filename is more robust. 158 158 + func ExtractAuditRecord(ctx context.Context, id AuditID, record *AuditRecord, dest string) error { 159 159 + const mode = 0o400 // readable by current user, not writable 160 160 + 161 161 + err := os.WriteFile(filepath.Join(dest, fmt.Sprintf("%s-event.json", id)), 162 162 + AuditRecordJSON(record, AuditRecordNoManifest), mode) 163 163 + if err != nil { 164 164 + return err 165 165 + } 166 166 + 167 167 + if record.Manifest != nil { 168 168 + err = os.WriteFile(filepath.Join(dest, fmt.Sprintf("%s-manifest.json", id)), 169 169 + ManifestJSON(record.Manifest), mode) 170 170 + if err != nil { 171 171 + return err 172 172 + } 173 173 + 174 174 + archive, err := os.OpenFile(filepath.Join(dest, fmt.Sprintf("%s-archive.tar", id)), 175 175 + os.O_CREATE|os.O_TRUNC|os.O_WRONLY, mode) 176 176 + if err != nil { 177 177 + return err 178 178 + } 179 179 + defer archive.Close() 180 180 + 181 181 + err = CollectTar(ctx, archive, record.Manifest, ManifestMetadata{}) 182 182 + if err != nil { 183 183 + return err 184 184 + } 185 185 + } 186 186 + 187 187 + return nil 188 188 + } 189 189 + 190 190 + func AuditEventProcessor(command string, args []string) (http.Handler, error) { 191 191 + var err error 192 192 + 193 193 + // Resolve the command to an absolute path, as it will be run from a different current 194 194 + // directory, which would break e.g. `git-pages -audit-server tcp/:3004 ./handler.sh`. 195 195 + if command, err = exec.LookPath(command); err != nil { 196 196 + return nil, err 197 197 + } 198 198 + if command, err = filepath.Abs(command); err != nil { 199 199 + return nil, err 200 200 + } 201 201 + 202 202 + router := http.NewServeMux() 203 203 + router.Handle("GET /", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 204 204 + // Go will cancel the request context if the client drops the connection. We don't want 205 205 + // that to interrupt processing. However, we also want the client (not the server) to 206 206 + // handle retries, so instead of spawning a goroutine to process the event, we do this 207 207 + // within the HTTP handler. If an error is returned, the notify goroutine in the worker 208 208 + // will retry the HTTP request (with backoff) until it succeeds. 209 209 + // 210 210 + // This is a somewhat idiosyncratic design and it's not clear that this is the best 211 211 + // possible approach (e.g. if the worker gets restarted and the event processing fails, 212 212 + // it will not be retried), but it should do the job for now. It is expected that 213 213 + // some form of observability is used to highlight event processor errors. 214 214 + ctx := context.WithoutCancel(r.Context()) 215 215 + 216 216 + id, err := ParseAuditID(r.URL.RawQuery) 217 217 + if err != nil { 218 218 + logc.Printf(ctx, "audit process err: malformed query\n") 219 219 + http.Error(w, "malformed query", http.StatusBadRequest) 220 220 + return 221 221 + } else { 222 222 + logc.Printf(ctx, "audit process %s", id) 223 223 + } 224 224 + 225 225 + record, err := backend.QueryAuditLog(ctx, id) 226 226 + if err != nil { 227 227 + logc.Printf(ctx, "audit process err: missing record\n") 228 228 + http.Error(w, "missing record", http.StatusNotFound) 229 229 + return 230 230 + } 231 231 + 232 232 + args := append(args, id.String(), record.GetEvent().String()) 233 233 + cmd := exec.CommandContext(ctx, command, args...) 234 234 + if cmd.Dir, err = os.MkdirTemp("", "auditRecord"); err != nil { 235 235 + panic(fmt.Errorf("mkdtemp: %w", err)) 236 236 + } 237 237 + defer os.RemoveAll(cmd.Dir) 238 238 + 239 239 + if err = ExtractAuditRecord(ctx, id, record, cmd.Dir); err != nil { 240 240 + logc.Printf(ctx, "audit process %s err: %s\n", id, err) 241 241 + http.Error(w, err.Error(), http.StatusInternalServerError) 242 242 + return 243 243 + } 244 244 + 245 245 + output, err := cmd.CombinedOutput() 246 246 + if err != nil { 247 247 + logc.Printf(ctx, "audit process %s err: %s; %s\n", id, err, string(output)) 248 248 + w.WriteHeader(http.StatusServiceUnavailable) 249 249 + if len(output) == 0 { 250 250 + fmt.Fprintln(w, err.Error()) 251 251 + } 252 252 + } else { 253 253 + logc.Printf(ctx, "audit process %s ok: %s\n", id, string(output)) 254 254 + w.WriteHeader(http.StatusOK) 255 255 + } 256 256 + w.Write(output) 257 257 + })) 258 258 + return router, nil 259 259 + } 260 260 + 261 261 + type auditedBackend struct { 262 262 + Backend 263 263 + } 264 264 + 265 265 + var _ Backend = (*auditedBackend)(nil) 266 266 + 267 267 + func NewAuditedBackend(backend Backend) Backend { 268 268 + return &auditedBackend{backend} 269 269 + } 270 270 + 271 271 + // This function does not retry appending audit records; as such, if it returns an error, 272 272 + // this error must interrupt whatever operation it was auditing. A corollary is that it is 273 273 + // possible that appending an audit record succeeds but the audited operation fails. 274 274 + // This is considered fine since the purpose of auditing is to record end user intent, not 275 275 + // to be a 100% accurate reflection of performed actions. When in doubt, the audit records 276 276 + // should be examined together with the application logs. 277 277 + func (audited *auditedBackend) appendNewAuditRecord(ctx context.Context, record *AuditRecord) (err error) { 278 278 + if config.Audit.Collect { 279 279 + id := GenerateAuditID() 280 280 + record.Id = proto.Int64(int64(id)) 281 281 + record.Timestamp = timestamppb.Now() 282 282 + record.Principal = GetPrincipal(ctx) 283 283 + 284 284 + err = audited.Backend.AppendAuditLog(ctx, id, record) 285 285 + if err != nil { 286 286 + err = fmt.Errorf("audit: %w", err) 287 287 + } else { 288 288 + var subject string 289 289 + if record.Project == nil { 290 290 + subject = *record.Domain 291 291 + } else { 292 292 + subject = path.Join(*record.Domain, *record.Project) 293 293 + } 294 294 + logc.Printf(ctx, "audit %s ok: %s %s\n", subject, id, record.Event.String()) 295 295 + 296 296 + // Send a notification to the audit server, if configured, and try to make sure 297 297 + // it is delivered by retrying with exponential backoff on errors. 298 298 + notifyAudit(context.WithoutCancel(ctx), id) 299 299 + } 300 300 + } 301 301 + return 302 302 + } 303 303 + 304 304 + func notifyAudit(ctx context.Context, id AuditID) { 305 305 + if config.Audit.NotifyURL != nil { 306 306 + notifyURL := config.Audit.NotifyURL.URL 307 307 + notifyURL.RawQuery = id.String() 308 308 + 309 309 + // See also the explanation in `AuditEventProcessor` above. 310 310 + go func() { 311 311 + backoff := exponential.Backoff{ 312 312 + Jitter: true, 313 313 + Min: time.Second * 1, 314 314 + Max: time.Second * 60, 315 315 + } 316 316 + for { 317 317 + resp, err := http.Get(notifyURL.String()) 318 318 + var body []byte 319 319 + if err == nil { 320 320 + defer resp.Body.Close() 321 321 + body, _ = io.ReadAll(resp.Body) 322 322 + } 323 323 + if err == nil && resp.StatusCode == http.StatusOK { 324 324 + logc.Printf(ctx, "audit notify %s ok: %s\n", id, string(body)) 325 325 + auditNotifyOkCount.Inc() 326 326 + break 327 327 + } else { 328 328 + sleepFor := backoff.Duration() 329 329 + if err != nil { 330 330 + logc.Printf(ctx, "audit notify %s err: %s (retry in %s)", 331 331 + id, err, sleepFor) 332 332 + } else { 333 333 + logc.Printf(ctx, "audit notify %s fail: %s (retry in %s); %s", 334 334 + id, resp.Status, sleepFor, string(body)) 335 335 + } 336 336 + auditNotifyErrorCount.Inc() 337 337 + time.Sleep(sleepFor) 338 338 + } 339 339 + } 340 340 + }() 341 341 + } 342 342 + } 343 343 + 344 344 + func (audited *auditedBackend) CommitManifest( 345 345 + ctx context.Context, name string, manifest *Manifest, opts ModifyManifestOptions, 346 346 + ) (err error) { 347 347 + domain, project, ok := strings.Cut(name, "/") 348 348 + if !ok { 349 349 + panic("malformed manifest name") 350 350 + } 351 351 + audited.appendNewAuditRecord(ctx, &AuditRecord{ 352 352 + Event: AuditEvent_CommitManifest.Enum(), 353 353 + Domain: proto.String(domain), 354 354 + Project: proto.String(project), 355 355 + Manifest: manifest, 356 356 + }) 357 357 + 358 358 + return audited.Backend.CommitManifest(ctx, name, manifest, opts) 359 359 + } 360 360 + 361 361 + func (audited *auditedBackend) DeleteManifest( 362 362 + ctx context.Context, name string, opts ModifyManifestOptions, 363 363 + ) (err error) { 364 364 + domain, project, ok := strings.Cut(name, "/") 365 365 + if !ok { 366 366 + panic("malformed manifest name") 367 367 + } 368 368 + audited.appendNewAuditRecord(ctx, &AuditRecord{ 369 369 + Event: AuditEvent_DeleteManifest.Enum(), 370 370 + Domain: proto.String(domain), 371 371 + Project: proto.String(project), 372 372 + }) 373 373 + 374 374 + return audited.Backend.DeleteManifest(ctx, name, opts) 375 375 + } 376 376 + 377 377 + func (audited *auditedBackend) FreezeDomain(ctx context.Context, domain string) (err error) { 378 378 + audited.appendNewAuditRecord(ctx, &AuditRecord{ 379 379 + Event: AuditEvent_FreezeDomain.Enum(), 380 380 + Domain: proto.String(domain), 381 381 + }) 382 382 + 383 383 + return audited.Backend.FreezeDomain(ctx, domain) 384 384 + } 385 385 + 386 386 + func (audited *auditedBackend) UnfreezeDomain(ctx context.Context, domain string) (err error) { 387 387 + audited.appendNewAuditRecord(ctx, &AuditRecord{ 388 388 + Event: AuditEvent_UnfreezeDomain.Enum(), 389 389 + Domain: proto.String(domain), 390 390 + }) 391 391 + 392 392 + return audited.Backend.UnfreezeDomain(ctx, domain) 393 393 + }

+57 -23

src/auth.go

··· 6 6 "encoding/json" 7 7 "errors" 8 8 "fmt" 9 9 - "log" 10 9 "net" 11 10 "net/http" 12 11 "net/url" 13 12 "slices" 14 13 "strings" 15 14 "time" 15 15 + 16 16 + "golang.org/x/net/idna" 16 17 ) 17 18 18 19 type AuthError struct { ··· 32 33 return false 33 34 } 34 35 35 35 - func authorizeInsecure() *Authorization { 36 36 + func authorizeInsecure(r *http.Request) *Authorization { 36 37 if config.Insecure { // for testing only 37 37 - log.Println("auth: INSECURE mode") 38 38 + logc.Println(r.Context(), "auth: INSECURE mode") 38 39 return &Authorization{ 39 40 repoURLs: nil, 40 41 branch: "pages", ··· 43 44 return nil 44 45 } 45 46 47 47 + var idnaProfile = idna.New(idna.MapForLookup(), idna.BidiRule()) 48 48 + 46 49 func GetHost(r *http.Request) (string, error) { 47 47 - // FIXME: handle IDNA 48 50 host, _, err := net.SplitHostPort(r.Host) 49 51 if err != nil { 50 50 - // dirty but the go stdlib doesn't have a "split port if present" function 51 52 host = r.Host 52 53 } 53 53 - if strings.HasPrefix(host, ".") { 54 54 + // this also rejects invalid characters and labels 55 55 + host, err = idnaProfile.ToASCII(host) 56 56 + if err != nil { 57 57 + if config.Feature("relaxed-idna") { 58 58 + // unfortunately, the go IDNA library has some significant issues around its 59 59 + // Unicode TR46 implementation: https://github.com/golang/go/issues/76804 60 60 + // we would like to allow *just* the _ here, but adding `idna.StrictDomainName(false)` 61 61 + // would also accept domains like `*.foo.bar` which should clearly be disallowed. 62 62 + // as a workaround, accept a domain name if it is valid with all `_` characters 63 63 + // replaced with an alphanumeric character (we use `a`); this allows e.g. `foo_bar.xxx` 64 64 + // and `foo__bar.xxx`, as well as `_foo.xxx` and `foo_.xxx`. labels starting with 65 65 + // an underscore are explicitly rejected below. 66 66 + _, err = idnaProfile.ToASCII(strings.ReplaceAll(host, "_", "a")) 67 67 + } 68 68 + if err != nil { 69 69 + return "", AuthError{http.StatusBadRequest, 70 70 + fmt.Sprintf("malformed host name %q", host)} 71 71 + } 72 72 + } 73 73 + if strings.HasPrefix(host, ".") || strings.HasPrefix(host, "_") { 54 74 return "", AuthError{http.StatusBadRequest, 55 55 - fmt.Sprintf("host name %q is reserved", host)} 75 75 + fmt.Sprintf("reserved host name %q", host)} 56 76 } 57 77 host = strings.TrimSuffix(host, ".") 58 78 return host, nil 59 79 } 60 80 81 81 + func IsValidProjectName(name string) bool { 82 82 + return !strings.HasPrefix(name, ".") && !strings.Contains(name, "%") 83 83 + } 84 84 + 61 85 func GetProjectName(r *http.Request) (string, error) { 62 86 // path must be either `/` or `/foo/` (`/foo` is accepted as an alias) 63 87 path := strings.TrimPrefix(strings.TrimSuffix(r.URL.Path, "/"), "/") 64 64 - if path == ".index" || strings.HasPrefix(path, ".index/") { 88 88 + if !IsValidProjectName(path) { 65 89 return "", AuthError{http.StatusBadRequest, 66 90 fmt.Sprintf("directory name %q is reserved", ".index")} 67 91 } else if strings.Contains(path, "/") { ··· 159 183 return nil, err 160 184 } 161 185 186 186 + projectName, err := GetProjectName(r) 187 187 + if err != nil { 188 188 + return nil, err 189 189 + } 190 190 + 162 191 allowlistHostname := fmt.Sprintf("_git-pages-repository.%s", host) 163 192 records, err := net.LookupTXT(allowlistHostname) 164 193 if err != nil { 165 194 return nil, AuthError{http.StatusUnauthorized, 166 195 fmt.Sprintf("failed to look up DNS repository allowlist: %s TXT", allowlistHostname)} 196 196 + } 197 197 + 198 198 + if projectName != ".index" { 199 199 + return nil, AuthError{http.StatusUnauthorized, 200 200 + "DNS repository allowlist only authorizes index site"} 167 201 } 168 202 169 203 var ( ··· 266 300 } 267 301 268 302 if len(dnsRecords) > 0 { 269 269 - log.Printf("auth: %s TXT/CNAME: %q\n", host, dnsRecords) 303 303 + logc.Printf(r.Context(), "auth: %s TXT/CNAME: %q\n", host, dnsRecords) 270 304 } 271 305 272 306 for _, dnsRecord := range dnsRecords { ··· 314 348 func AuthorizeMetadataRetrieval(r *http.Request) (*Authorization, error) { 315 349 causes := []error{AuthError{http.StatusUnauthorized, "unauthorized"}} 316 350 317 317 - auth := authorizeInsecure() 351 351 + auth := authorizeInsecure(r) 318 352 if auth != nil { 319 353 return auth, nil 320 354 } ··· 325 359 } else if err != nil { // bad request 326 360 return nil, err 327 361 } else { 328 328 - log.Println("auth: DNS challenge") 362 362 + logc.Println(r.Context(), "auth: DNS challenge") 329 363 return auth, nil 330 364 } 331 365 ··· 336 370 } else if err != nil { // bad request 337 371 return nil, err 338 372 } else { 339 339 - log.Printf("auth: wildcard %s\n", pattern.GetHost()) 373 373 + logc.Printf(r.Context(), "auth: wildcard %s\n", pattern.GetHost()) 340 374 return auth, nil 341 375 } 342 376 } ··· 348 382 } else if err != nil { // bad request 349 383 return nil, err 350 384 } else { 351 351 - log.Printf("auth: codeberg %s\n", r.Host) 385 385 + logc.Printf(r.Context(), "auth: codeberg %s\n", r.Host) 352 386 return auth, nil 353 387 } 354 388 } ··· 366 400 return nil, err 367 401 } 368 402 369 369 - auth := authorizeInsecure() 403 403 + auth := authorizeInsecure(r) 370 404 if auth != nil { 371 405 return auth, nil 372 406 } ··· 378 412 } else if err != nil { // bad request 379 413 return nil, err 380 414 } else { 381 381 - log.Println("auth: DNS challenge: allow *") 415 415 + logc.Println(r.Context(), "auth: DNS challenge: allow *") 382 416 return auth, nil 383 417 } 384 418 ··· 390 424 } else if err != nil { // bad request 391 425 return nil, err 392 426 } else { 393 393 - log.Printf("auth: DNS allowlist: allow %v\n", auth.repoURLs) 427 427 + logc.Printf(r.Context(), "auth: DNS allowlist: allow %v\n", auth.repoURLs) 394 428 return auth, nil 395 429 } 396 430 } ··· 404 438 } else if err != nil { // bad request 405 439 return nil, err 406 440 } else { 407 407 - log.Printf("auth: wildcard %s: allow %v\n", pattern.GetHost(), auth.repoURLs) 441 441 + logc.Printf(r.Context(), "auth: wildcard %s: allow %v\n", pattern.GetHost(), auth.repoURLs) 408 442 return auth, nil 409 443 } 410 444 } ··· 416 450 } else if err != nil { // bad request 417 451 return nil, err 418 452 } else { 419 419 - log.Printf("auth: codeberg %s: allow %v branch %s\n", 453 453 + logc.Printf(r.Context(), "auth: codeberg %s: allow %v branch %s\n", 420 454 r.Host, auth.repoURLs, auth.branch) 421 455 return auth, nil 422 456 } ··· 427 461 } 428 462 429 463 func checkAllowedURLPrefix(repoURL string) error { 430 430 - if config.Limits.AllowedRepositoryURLPrefixes != nil { 464 464 + if len(config.Limits.AllowedRepositoryURLPrefixes) > 0 { 431 465 allowedPrefix := false 432 466 repoURL = strings.ToLower(repoURL) 433 467 for _, allowedRepoURLPrefix := range config.Limits.AllowedRepositoryURLPrefixes { ··· 633 667 return nil, err 634 668 } 635 669 636 636 - auth := authorizeInsecure() 670 670 + auth := authorizeInsecure(r) 637 671 if auth != nil { 638 672 return auth, nil 639 673 } ··· 645 679 } else if err != nil { // bad request 646 680 return nil, err 647 681 } else { 648 648 - log.Printf("auth: forge token: allow\n") 682 682 + logc.Printf(r.Context(), "auth: forge token: allow\n") 649 683 return auth, nil 650 684 } 651 685 652 652 - if config.Limits.AllowedRepositoryURLPrefixes != nil { 686 686 + if len(config.Limits.AllowedRepositoryURLPrefixes) > 0 { 653 687 causes = append(causes, AuthError{http.StatusUnauthorized, "DNS challenge not allowed"}) 654 688 } else { 655 689 // DNS challenge gives absolute authority. ··· 659 693 } else if err != nil { // bad request 660 694 return nil, err 661 695 } else { 662 662 - log.Println("auth: DNS challenge") 696 696 + logc.Println(r.Context(), "auth: DNS challenge") 663 697 return auth, nil 664 698 } 665 699 }

+89 -15

src/backend.go

··· 5 5 "errors" 6 6 "fmt" 7 7 "io" 8 8 - "slices" 8 8 + "iter" 9 9 "strings" 10 10 "time" 11 11 ) 12 12 13 13 var ErrObjectNotFound = errors.New("not found") 14 14 + var ErrPreconditionFailed = errors.New("precondition failed") 15 15 + var ErrWriteConflict = errors.New("write conflict") 16 16 + var ErrDomainFrozen = errors.New("domain administratively frozen") 14 17 15 18 func splitBlobName(name string) []string { 16 16 - algo, hash, found := strings.Cut(name, "-") 17 17 - if found { 18 18 - return slices.Concat([]string{algo}, splitBlobName(hash)) 19 19 + if algo, hash, found := strings.Cut(name, "-"); found { 20 20 + return []string{algo, hash[0:2], hash[2:4], hash[4:]} 19 21 } else { 20 20 - return []string{name[0:2], name[2:4], name[4:]} 22 22 + panic("malformed blob name") 21 23 } 22 24 } 23 25 26 26 + func joinBlobName(parts []string) string { 27 27 + return fmt.Sprintf("%s-%s", parts[0], strings.Join(parts[1:], "")) 28 28 + } 29 29 + 24 30 type BackendFeature string 25 31 26 32 const ( 27 33 FeatureCheckDomainMarker BackendFeature = "check-domain-marker" 28 34 ) 29 35 36 36 + type BlobMetadata struct { 37 37 + Name string 38 38 + Size int64 39 39 + LastModified time.Time 40 40 + } 41 41 + 30 42 type GetManifestOptions struct { 43 43 + // If true and the manifest is past the cache `MaxAge`, `GetManifest` blocks and returns 44 44 + // a fresh object instead of revalidating in background and returning a stale object. 31 45 BypassCache bool 32 46 } 33 47 48 48 + type ManifestMetadata struct { 49 49 + Name string 50 50 + Size int64 51 51 + LastModified time.Time 52 52 + ETag string 53 53 + } 54 54 + 55 55 + type ModifyManifestOptions struct { 56 56 + // If non-zero, the request will only succeed if the manifest hasn't been changed since 57 57 + // the given time. Whether this is racy or not is can be determined via `HasAtomicCAS()`. 58 58 + IfUnmodifiedSince time.Time 59 59 + // If non-empty, the request will only succeed if the manifest hasn't changed from 60 60 + // the state corresponding to the ETag. Whether this is racy or not is can be determined 61 61 + // via `HasAtomicCAS()`. 62 62 + IfMatch string 63 63 + } 64 64 + 65 65 + type SearchAuditLogOptions struct { 66 66 + // Inclusive lower bound on returned audit records, per their Snowflake ID (which may differ 67 67 + // slightly from the embedded timestamp). If zero, audit records are returned since beginning 68 68 + // of time. 69 69 + Since time.Time 70 70 + // Inclusive upper bound on returned audit records, per their Snowflake ID (which may differ 71 71 + // slightly from the embedded timestamp). If zero, audit records are returned until the end 72 72 + // of time. 73 73 + Until time.Time 74 74 + } 75 75 + 76 76 + type SearchAuditLogResult struct { 77 77 + ID AuditID 78 78 + Err error 79 79 + } 80 80 + 34 81 type Backend interface { 35 82 // Returns true if the feature has been enabled for this store, false otherwise. 36 83 HasFeature(ctx context.Context, feature BackendFeature) bool ··· 40 87 41 88 // Retrieve a blob. Returns `reader, size, mtime, err`. 42 89 GetBlob(ctx context.Context, name string) ( 43 43 - reader io.ReadSeeker, size uint64, mtime time.Time, err error, 90 90 + reader io.ReadSeeker, metadata BlobMetadata, err error, 44 91 ) 45 92 46 93 // Store a blob. If a blob called `name` already exists, this function returns `nil` without ··· 51 98 // Delete a blob. This is an unconditional operation that can break integrity of manifests. 52 99 DeleteBlob(ctx context.Context, name string) error 53 100 101 101 + // Iterate through all blobs. Whether blobs that are newly added during iteration will appear 102 102 + // in the results is unspecified. 103 103 + EnumerateBlobs(ctx context.Context) iter.Seq2[BlobMetadata, error] 104 104 + 54 105 // Retrieve a manifest. 55 106 GetManifest(ctx context.Context, name string, opts GetManifestOptions) ( 56 56 - manifest *Manifest, mtime time.Time, err error, 107 107 + manifest *Manifest, metadata ManifestMetadata, err error, 57 108 ) 58 109 59 110 // Stage a manifest. This operation stores a new version of a manifest, locking any blobs ··· 61 112 // effects. 62 113 StageManifest(ctx context.Context, manifest *Manifest) error 63 114 115 115 + // Whether a compare-and-swap operation on a manifest is truly race-free, or only best-effort 116 116 + // atomic with a small but non-zero window where two requests may race where the one committing 117 117 + // first will have its update lost. (Plain swap operations are always guaranteed to be atomic.) 118 118 + HasAtomicCAS(ctx context.Context) bool 119 119 + 64 120 // Commit a manifest. This is an atomic operation; `GetManifest` calls will return either 65 121 // the old version or the new version of the manifest, never anything else. 66 66 - CommitManifest(ctx context.Context, name string, manifest *Manifest) error 122 122 + CommitManifest(ctx context.Context, name string, manifest *Manifest, opts ModifyManifestOptions) error 67 123 68 124 // Delete a manifest. 69 69 - DeleteManifest(ctx context.Context, name string) error 125 125 + DeleteManifest(ctx context.Context, name string, opts ModifyManifestOptions) error 70 126 71 71 - // List all manifests. 72 72 - ListManifests(ctx context.Context) (manifests []string, err error) 127 127 + // Iterate through all manifests. Whether manifests that are newly added during iteration 128 128 + // will appear in the results is unspecified. 129 129 + EnumerateManifests(ctx context.Context) iter.Seq2[ManifestMetadata, error] 73 130 74 131 // Check whether a domain has any deployments. 75 132 CheckDomain(ctx context.Context, domain string) (found bool, err error) 76 133 77 77 - // Creates a domain. This allows us to start serving content for the domain. 134 134 + // Create a domain. This allows us to start serving content for the domain. 78 135 CreateDomain(ctx context.Context, domain string) error 136 136 + 137 137 + // Freeze a domain. This allows a site to be administratively locked, e.g. if it 138 138 + // is discovered serving abusive content. 139 139 + FreezeDomain(ctx context.Context, domain string) error 140 140 + 141 141 + // Thaw a domain. This removes the previously placed administrative lock (if any). 142 142 + UnfreezeDomain(ctx context.Context, domain string) error 143 143 + 144 144 + // Append a record to the audit log. 145 145 + AppendAuditLog(ctx context.Context, id AuditID, record *AuditRecord) error 146 146 + 147 147 + // Retrieve a single record from the audit log. 148 148 + QueryAuditLog(ctx context.Context, id AuditID) (record *AuditRecord, err error) 149 149 + 150 150 + // Retrieve records from the audit log by time range. 151 151 + SearchAuditLog(ctx context.Context, opts SearchAuditLogOptions) iter.Seq2[AuditID, error] 79 152 } 80 153 81 81 - func CreateBackend(config *StorageConfig) (backend Backend, err error) { 154 154 + func CreateBackend(ctx context.Context, config *StorageConfig) (backend Backend, err error) { 82 155 switch config.Type { 83 156 case "fs": 84 84 - if backend, err = NewFSBackend(&config.FS); err != nil { 157 157 + if backend, err = NewFSBackend(ctx, &config.FS); err != nil { 85 158 err = fmt.Errorf("fs backend: %w", err) 86 159 } 87 160 case "s3": 88 88 - if backend, err = NewS3Backend(context.Background(), &config.S3); err != nil { 161 161 + if backend, err = NewS3Backend(ctx, &config.S3); err != nil { 89 162 err = fmt.Errorf("s3 backend: %w", err) 90 163 } 91 164 default: 92 165 err = fmt.Errorf("unknown backend: %s", config.Type) 93 166 } 167 167 + backend = NewAuditedBackend(backend) 94 168 return 95 169 }

+276 -26

src/backend_fs.go

··· 6 6 "errors" 7 7 "fmt" 8 8 "io" 9 9 - "io/fs" 9 9 + iofs "io/fs" 10 10 + "iter" 10 11 "os" 11 12 "path/filepath" 12 13 "strings" 13 13 - "time" 14 14 ) 15 15 16 16 type FSBackend struct { 17 17 - blobRoot *os.Root 18 18 - siteRoot *os.Root 17 17 + blobRoot *os.Root 18 18 + siteRoot *os.Root 19 19 + auditRoot *os.Root 20 20 + hasAtomicCAS bool 19 21 } 20 22 21 23 var _ Backend = (*FSBackend)(nil) ··· 54 56 return tempPath, nil 55 57 } 56 58 57 57 - func NewFSBackend(config *FSConfig) (*FSBackend, error) { 59 59 + func checkAtomicCAS(root *os.Root) bool { 60 60 + fileName := ".hasAtomicCAS" 61 61 + file, err := root.Create(fileName) 62 62 + if err != nil { 63 63 + panic(err) 64 64 + } 65 65 + root.Remove(fileName) 66 66 + defer file.Close() 67 67 + 68 68 + flockErr := FileLock(file) 69 69 + funlockErr := FileUnlock(file) 70 70 + return (flockErr == nil && funlockErr == nil) 71 71 + } 72 72 + 73 73 + func NewFSBackend(ctx context.Context, config *FSConfig) (*FSBackend, error) { 58 74 blobRoot, err := maybeCreateOpenRoot(config.Root, "blob") 59 75 if err != nil { 60 76 return nil, fmt.Errorf("blob: %w", err) ··· 63 79 if err != nil { 64 80 return nil, fmt.Errorf("site: %w", err) 65 81 } 66 66 - return &FSBackend{blobRoot, siteRoot}, nil 82 82 + auditRoot, err := maybeCreateOpenRoot(config.Root, "audit") 83 83 + if err != nil { 84 84 + return nil, fmt.Errorf("audit: %w", err) 85 85 + } 86 86 + hasAtomicCAS := checkAtomicCAS(siteRoot) 87 87 + if hasAtomicCAS { 88 88 + logc.Println(ctx, "fs: has atomic CAS") 89 89 + } else { 90 90 + logc.Println(ctx, "fs: has best-effort CAS") 91 91 + } 92 92 + return &FSBackend{blobRoot, siteRoot, auditRoot, hasAtomicCAS}, nil 67 93 } 68 94 69 95 func (fs *FSBackend) Backend() Backend { ··· 91 117 func (fs *FSBackend) GetBlob( 92 118 ctx context.Context, name string, 93 119 ) ( 94 94 - reader io.ReadSeeker, size uint64, mtime time.Time, err error, 120 120 + reader io.ReadSeeker, metadata BlobMetadata, err error, 95 121 ) { 96 122 blobPath := filepath.Join(splitBlobName(name)...) 97 123 stat, err := fs.blobRoot.Stat(blobPath) ··· 107 133 err = fmt.Errorf("open: %w", err) 108 134 return 109 135 } 110 110 - return file, uint64(stat.Size()), stat.ModTime(), nil 136 136 + return file, BlobMetadata{name, int64(stat.Size()), stat.ModTime()}, nil 111 137 } 112 138 113 139 func (fs *FSBackend) PutBlob(ctx context.Context, name string, data []byte) error { 114 140 blobPath := filepath.Join(splitBlobName(name)...) 115 141 blobDir := filepath.Dir(blobPath) 142 142 + 143 143 + if _, err := fs.blobRoot.Stat(blobPath); err == nil { 144 144 + // Blob already exists. While on Linux it would be benign to write and replace a blob 145 145 + // that already exists, on Windows this is liable to cause access errors. 146 146 + return nil 147 147 + } 116 148 117 149 tempPath, err := createTempInRoot(fs.blobRoot, name, data) 118 150 if err != nil { ··· 149 181 return fs.blobRoot.Remove(blobPath) 150 182 } 151 183 152 152 - func (b *FSBackend) ListManifests(ctx context.Context) (manifests []string, err error) { 153 153 - err = fs.WalkDir(b.siteRoot.FS(), ".", func(path string, d fs.DirEntry, err error) error { 154 154 - if strings.Count(path, "/") > 1 { 155 155 - return fs.SkipDir 156 156 - } 157 157 - _, project, _ := strings.Cut(path, "/") 158 158 - if project == "" || strings.HasPrefix(project, ".") && project != ".index" { 159 159 - return nil 160 160 - } 161 161 - manifests = append(manifests, path) 162 162 - return nil 163 163 - }) 164 164 - return 184 184 + func (fs *FSBackend) EnumerateBlobs(ctx context.Context) iter.Seq2[BlobMetadata, error] { 185 185 + return func(yield func(BlobMetadata, error) bool) { 186 186 + iofs.WalkDir(fs.blobRoot.FS(), ".", 187 187 + func(path string, entry iofs.DirEntry, err error) error { 188 188 + var metadata BlobMetadata 189 189 + if err != nil { 190 190 + // report error 191 191 + } else if entry.IsDir() { 192 192 + // skip directory 193 193 + return nil 194 194 + } else if info, err := entry.Info(); err != nil { 195 195 + // report error 196 196 + } else { 197 197 + // report blob 198 198 + metadata.Name = joinBlobName(strings.Split(path, "/")) 199 199 + metadata.Size = info.Size() 200 200 + metadata.LastModified = info.ModTime() 201 201 + } 202 202 + if !yield(metadata, err) { 203 203 + return iofs.SkipAll 204 204 + } 205 205 + return nil 206 206 + }) 207 207 + } 165 208 } 166 209 167 210 func (fs *FSBackend) GetManifest( 168 211 ctx context.Context, name string, opts GetManifestOptions, 169 212 ) ( 170 170 - manifest *Manifest, mtime time.Time, err error, 213 213 + manifest *Manifest, metadata ManifestMetadata, err error, 171 214 ) { 172 215 stat, err := fs.siteRoot.Stat(name) 173 216 if errors.Is(err, os.ErrNotExist) { ··· 186 229 if err != nil { 187 230 return 188 231 } 189 189 - return manifest, stat.ModTime(), nil 232 232 + return manifest, ManifestMetadata{ 233 233 + LastModified: stat.ModTime(), 234 234 + ETag: fmt.Sprintf("%x", sha256.Sum256(data)), 235 235 + }, nil 190 236 } 191 237 192 238 func stagedManifestName(manifestData []byte) string { ··· 208 254 return nil 209 255 } 210 256 211 211 - func (fs *FSBackend) CommitManifest(ctx context.Context, name string, manifest *Manifest) error { 257 257 + func domainFrozenMarkerName(domain string) string { 258 258 + return filepath.Join(domain, ".frozen") 259 259 + } 260 260 + 261 261 + func (fs *FSBackend) checkDomainFrozen(ctx context.Context, domain string) error { 262 262 + if _, err := fs.siteRoot.Stat(domainFrozenMarkerName(domain)); err == nil { 263 263 + return ErrDomainFrozen 264 264 + } else if !errors.Is(err, os.ErrNotExist) { 265 265 + return fmt.Errorf("stat: %w", err) 266 266 + } else { 267 267 + return nil 268 268 + } 269 269 + } 270 270 + 271 271 + func (fs *FSBackend) HasAtomicCAS(ctx context.Context) bool { 272 272 + // On a suitable filesystem, POSIX advisory locks can be used to implement atomic CAS. 273 273 + // An implementation consists of two parts: 274 274 + // - Intra-process mutex set (one per manifest), to prevent races between goroutines; 275 275 + // - Inter-process POSIX advisory locks (one per manifest), to prevent races between 276 276 + // different git-pages instances. 277 277 + return fs.hasAtomicCAS 278 278 + } 279 279 + 280 280 + type manifestLockGuard struct { 281 281 + file *os.File 282 282 + } 283 283 + 284 284 + func lockManifest(fs *os.Root, name string) (*manifestLockGuard, error) { 285 285 + file, err := fs.Open(name) 286 286 + if errors.Is(err, os.ErrNotExist) { 287 287 + return &manifestLockGuard{nil}, nil 288 288 + } else if err != nil { 289 289 + return nil, fmt.Errorf("open: %w", err) 290 290 + } 291 291 + if err := FileLock(file); err != nil { 292 292 + file.Close() 293 293 + return nil, fmt.Errorf("flock(LOCK_EX): %w", err) 294 294 + } 295 295 + return &manifestLockGuard{file}, nil 296 296 + } 297 297 + 298 298 + func (guard *manifestLockGuard) Unlock() { 299 299 + if guard.file != nil { 300 300 + FileUnlock(guard.file) 301 301 + guard.file.Close() 302 302 + } 303 303 + } 304 304 + 305 305 + func (fs *FSBackend) checkManifestPrecondition( 306 306 + ctx context.Context, name string, opts ModifyManifestOptions, 307 307 + ) error { 308 308 + if !opts.IfUnmodifiedSince.IsZero() { 309 309 + stat, err := fs.siteRoot.Stat(name) 310 310 + if err != nil { 311 311 + return fmt.Errorf("stat: %w", err) 312 312 + } 313 313 + 314 314 + if stat.ModTime().Compare(opts.IfUnmodifiedSince) > 0 { 315 315 + return fmt.Errorf("%w: If-Unmodified-Since", ErrPreconditionFailed) 316 316 + } 317 317 + } 318 318 + 319 319 + if opts.IfMatch != "" { 320 320 + data, err := fs.siteRoot.ReadFile(name) 321 321 + if err != nil { 322 322 + return fmt.Errorf("read: %w", err) 323 323 + } 324 324 + 325 325 + if fmt.Sprintf("%x", sha256.Sum256(data)) != opts.IfMatch { 326 326 + return fmt.Errorf("%w: If-Match", ErrPreconditionFailed) 327 327 + } 328 328 + } 329 329 + 330 330 + return nil 331 331 + } 332 332 + 333 333 + func (fs *FSBackend) CommitManifest( 334 334 + ctx context.Context, name string, manifest *Manifest, opts ModifyManifestOptions, 335 335 + ) error { 336 336 + if fs.hasAtomicCAS { 337 337 + if guard, err := lockManifest(fs.siteRoot, name); err != nil { 338 338 + return err 339 339 + } else { 340 340 + defer guard.Unlock() 341 341 + } 342 342 + } 343 343 + 344 344 + domain := filepath.Dir(name) 345 345 + if err := fs.checkDomainFrozen(ctx, domain); err != nil { 346 346 + return err 347 347 + } 348 348 + 349 349 + if err := fs.checkManifestPrecondition(ctx, name, opts); err != nil { 350 350 + return err 351 351 + } 352 352 + 212 353 manifestData := EncodeManifest(manifest) 213 354 manifestHashName := stagedManifestName(manifestData) 214 355 ··· 216 357 return fmt.Errorf("manifest not staged") 217 358 } 218 359 219 219 - if err := fs.siteRoot.MkdirAll(filepath.Dir(name), 0o755); err != nil { 360 360 + if err := fs.siteRoot.MkdirAll(domain, 0o755); err != nil { 220 361 return fmt.Errorf("mkdir: %w", err) 221 362 } 222 363 ··· 227 368 return nil 228 369 } 229 370 230 230 - func (fs *FSBackend) DeleteManifest(ctx context.Context, name string) error { 371 371 + func (fs *FSBackend) DeleteManifest( 372 372 + ctx context.Context, name string, opts ModifyManifestOptions, 373 373 + ) error { 374 374 + if fs.hasAtomicCAS { 375 375 + if guard, err := lockManifest(fs.siteRoot, name); err != nil { 376 376 + return err 377 377 + } else { 378 378 + defer guard.Unlock() 379 379 + } 380 380 + } 381 381 + 382 382 + domain := filepath.Dir(name) 383 383 + if err := fs.checkDomainFrozen(ctx, domain); err != nil { 384 384 + return err 385 385 + } 386 386 + 387 387 + if err := fs.checkManifestPrecondition(ctx, name, opts); err != nil { 388 388 + return err 389 389 + } 390 390 + 231 391 err := fs.siteRoot.Remove(name) 232 392 if errors.Is(err, os.ErrNotExist) { 233 393 return nil ··· 236 396 } 237 397 } 238 398 399 399 + func (fs *FSBackend) EnumerateManifests(ctx context.Context) iter.Seq2[ManifestMetadata, error] { 400 400 + return func(yield func(ManifestMetadata, error) bool) { 401 401 + iofs.WalkDir(fs.siteRoot.FS(), ".", 402 402 + func(path string, entry iofs.DirEntry, err error) error { 403 403 + _, project, _ := strings.Cut(path, "/") 404 404 + var metadata ManifestMetadata 405 405 + if err != nil { 406 406 + // report error 407 407 + } else if entry.IsDir() { 408 408 + // skip directory 409 409 + return nil 410 410 + } else if project == "" || strings.HasPrefix(project, ".") && project != ".index" { 411 411 + // skip internal 412 412 + return nil 413 413 + } else if info, err := entry.Info(); err != nil { 414 414 + // report error 415 415 + } else { 416 416 + // report blob 417 417 + metadata.Name = path 418 418 + metadata.Size = info.Size() 419 419 + metadata.LastModified = info.ModTime() 420 420 + // not setting metadata.ETag since it is too costly 421 421 + } 422 422 + if !yield(metadata, err) { 423 423 + return iofs.SkipAll 424 424 + } 425 425 + return nil 426 426 + }) 427 427 + } 428 428 + } 429 429 + 239 430 func (fs *FSBackend) CheckDomain(ctx context.Context, domain string) (bool, error) { 240 431 _, err := fs.siteRoot.Stat(domain) 241 432 if errors.Is(err, os.ErrNotExist) { ··· 250 441 func (fs *FSBackend) CreateDomain(ctx context.Context, domain string) error { 251 442 return nil // no-op 252 443 } 444 444 + 445 445 + func (fs *FSBackend) FreezeDomain(ctx context.Context, domain string) error { 446 446 + return fs.siteRoot.WriteFile(domainFrozenMarkerName(domain), []byte{}, 0o644) 447 447 + } 448 448 + 449 449 + func (fs *FSBackend) UnfreezeDomain(ctx context.Context, domain string) error { 450 450 + err := fs.siteRoot.Remove(domainFrozenMarkerName(domain)) 451 451 + if errors.Is(err, os.ErrNotExist) { 452 452 + return nil 453 453 + } else { 454 454 + return err 455 455 + } 456 456 + } 457 457 + 458 458 + func (fs *FSBackend) AppendAuditLog(ctx context.Context, id AuditID, record *AuditRecord) error { 459 459 + if _, err := fs.auditRoot.Stat(id.String()); err == nil { 460 460 + panic(fmt.Errorf("audit ID collision: %s", id)) 461 461 + } 462 462 + 463 463 + return fs.auditRoot.WriteFile(id.String(), EncodeAuditRecord(record), 0o644) 464 464 + } 465 465 + 466 466 + func (fs *FSBackend) QueryAuditLog(ctx context.Context, id AuditID) (*AuditRecord, error) { 467 467 + if data, err := fs.auditRoot.ReadFile(id.String()); err != nil { 468 468 + return nil, fmt.Errorf("read: %w", err) 469 469 + } else if record, err := DecodeAuditRecord(data); err != nil { 470 470 + return nil, fmt.Errorf("decode: %w", err) 471 471 + } else { 472 472 + return record, nil 473 473 + } 474 474 + } 475 475 + 476 476 + func (fs *FSBackend) SearchAuditLog( 477 477 + ctx context.Context, opts SearchAuditLogOptions, 478 478 + ) iter.Seq2[AuditID, error] { 479 479 + return func(yield func(AuditID, error) bool) { 480 480 + iofs.WalkDir(fs.auditRoot.FS(), ".", 481 481 + func(path string, entry iofs.DirEntry, err error) error { 482 482 + if path == "." { 483 483 + return nil // skip 484 484 + } 485 485 + var id AuditID 486 486 + if err != nil { 487 487 + // report error 488 488 + } else if id, err = ParseAuditID(path); err != nil { 489 489 + // report error 490 490 + } else if !opts.Since.IsZero() && id.CompareTime(opts.Since) < 0 { 491 491 + return nil // skip 492 492 + } else if !opts.Until.IsZero() && id.CompareTime(opts.Until) > 0 { 493 493 + return nil // skip 494 494 + } 495 495 + if !yield(id, err) { 496 496 + return iofs.SkipAll // break 497 497 + } else { 498 498 + return nil // continue 499 499 + } 500 500 + }) 501 501 + } 502 502 + }

+330 -81

src/backend_s3.go

··· 6 6 "crypto/sha256" 7 7 "fmt" 8 8 "io" 9 9 - "log" 9 9 + "iter" 10 10 "net/http" 11 11 "path" 12 12 "strings" ··· 36 36 manifestCacheEvictionsCount prometheus.Counter 37 37 38 38 s3GetObjectDurationSeconds *prometheus.HistogramVec 39 39 - s3GetObjectErrorsCount *prometheus.CounterVec 39 39 + s3GetObjectResponseCount *prometheus.CounterVec 40 40 ) 41 41 42 42 func initS3BackendMetrics() { ··· 96 96 NativeHistogramMaxBucketNumber: 100, 97 97 NativeHistogramMinResetDuration: 10 * time.Minute, 98 98 }, []string{"kind"}) 99 99 - s3GetObjectErrorsCount = promauto.NewCounterVec(prometheus.CounterOpts{ 100 100 - Name: "git_pages_s3_get_object_errors_count", 101 101 - Help: "Count of s3:GetObject errors", 102 102 - }, []string{"object_kind"}) 99 99 + s3GetObjectResponseCount = promauto.NewCounterVec(prometheus.CounterOpts{ 100 100 + Name: "git_pages_s3_get_object_responses_count", 101 101 + Help: "Count of s3:GetObject responses", 102 102 + }, []string{"kind", "code"}) 103 103 } 104 104 105 105 // Blobs can be safely cached indefinitely. They only need to be evicted to preserve memory. ··· 117 117 type CachedManifest struct { 118 118 manifest *Manifest 119 119 weight uint32 120 120 - mtime time.Time 121 121 - etag string 120 120 + metadata ManifestMetadata 122 121 err error 123 122 } 124 123 ··· 144 143 options.Weigher = weigher 145 144 } 146 145 if config.MaxStale != 0 { 147 147 - options.RefreshCalculator = otter.RefreshWriting[K, V](time.Duration(config.MaxAge)) 146 146 + options.RefreshCalculator = otter.RefreshWriting[K, V]( 147 147 + time.Duration(config.MaxAge)) 148 148 } 149 149 if config.MaxAge != 0 || config.MaxStale != 0 { 150 150 - options.ExpiryCalculator = otter.ExpiryWriting[K, V](time.Duration(config.MaxAge + config.MaxStale)) 150 150 + options.ExpiryCalculator = otter.ExpiryWriting[K, V]( 151 151 + time.Duration(config.MaxAge + config.MaxStale)) 151 152 } 152 153 return options 153 154 } ··· 170 171 if err != nil { 171 172 return nil, err 172 173 } else if !exists { 173 173 - log.Printf("s3: create bucket %s\n", bucket) 174 174 + logc.Printf(ctx, "s3: create bucket %s\n", bucket) 174 175 175 176 err = client.MakeBucket(ctx, bucket, 176 177 minio.MakeBucketOptions{Region: config.Region}) ··· 236 237 minio.StatObjectOptions{}) 237 238 if err != nil { 238 239 if errResp := minio.ToErrorResponse(err); errResp.Code == "NoSuchKey" { 239 239 - log.Printf("s3 feature %q: disabled", feature) 240 240 + logc.Printf(ctx, "s3 feature %q: disabled", feature) 240 241 return false, nil 241 242 } else { 242 243 return false, err 243 244 } 244 245 } 245 245 - log.Printf("s3 feature %q: enabled", feature) 246 246 + logc.Printf(ctx, "s3 feature %q: enabled", feature) 246 247 return true, nil 247 248 } 248 249 ··· 250 251 if err != nil { 251 252 err = fmt.Errorf("getting s3 backend feature %q: %w", feature, err) 252 253 ObserveError(err) 253 253 - log.Print(err) 254 254 + logc.Println(ctx, err) 254 255 return false 255 256 } 256 257 return isOn ··· 265 266 func (s3 *S3Backend) GetBlob( 266 267 ctx context.Context, name string, 267 268 ) ( 268 268 - reader io.ReadSeeker, size uint64, mtime time.Time, err error, 269 269 + reader io.ReadSeeker, metadata BlobMetadata, err error, 269 270 ) { 270 271 loader := func(ctx context.Context, name string) (*CachedBlob, error) { 271 271 - log.Printf("s3: get blob %s\n", name) 272 272 + logc.Printf(ctx, "s3: get blob %s\n", name) 272 273 273 274 startTime := time.Now() 274 275 ··· 297 298 return &CachedBlob{data, stat.LastModified}, nil 298 299 } 299 300 301 301 + observer := func(ctx context.Context, name string) (*CachedBlob, error) { 302 302 + cached, err := loader(ctx, name) 303 303 + var code = "OK" 304 304 + if resp, ok := err.(minio.ErrorResponse); ok { 305 305 + code = resp.Code 306 306 + } 307 307 + s3GetObjectResponseCount.With(prometheus.Labels{"kind": "blob", "code": code}).Inc() 308 308 + return cached, err 309 309 + } 310 310 + 300 311 var cached *CachedBlob 301 301 - cached, err = s3.blobCache.Get(ctx, name, otter.LoaderFunc[string, *CachedBlob](loader)) 312 312 + cached, err = s3.blobCache.Get(ctx, name, otter.LoaderFunc[string, *CachedBlob](observer)) 302 313 if err != nil { 303 314 if errResp := minio.ToErrorResponse(err); errResp.Code == "NoSuchKey" { 304 304 - s3GetObjectErrorsCount.With(prometheus.Labels{"object_kind": "blob"}).Inc() 305 315 err = fmt.Errorf("%w: %s", ErrObjectNotFound, errResp.Key) 306 316 } 307 317 } else { 308 318 reader = bytes.NewReader(cached.blob) 309 309 - size = uint64(len(cached.blob)) 310 310 - mtime = cached.mtime 319 319 + metadata.Name = name 320 320 + metadata.Size = int64(len(cached.blob)) 321 321 + metadata.LastModified = cached.mtime 311 322 } 312 323 return 313 324 } 314 325 315 326 func (s3 *S3Backend) PutBlob(ctx context.Context, name string, data []byte) error { 316 316 - log.Printf("s3: put blob %s (%s)\n", name, datasize.ByteSize(len(data)).HumanReadable()) 327 327 + logc.Printf(ctx, "s3: put blob %s (%s)\n", name, datasize.ByteSize(len(data)).HumanReadable()) 317 328 318 329 _, err := s3.client.StatObject(ctx, s3.bucket, blobObjectName(name), 319 330 minio.GetObjectOptions{}) ··· 325 336 return err 326 337 } else { 327 338 ObserveData(ctx, "blob.status", "created") 328 328 - log.Printf("s3: put blob %s (created)\n", name) 339 339 + logc.Printf(ctx, "s3: put blob %s (created)\n", name) 329 340 return nil 330 341 } 331 342 } else { ··· 333 344 } 334 345 } else { 335 346 ObserveData(ctx, "blob.status", "exists") 336 336 - log.Printf("s3: put blob %s (exists)\n", name) 347 347 + logc.Printf(ctx, "s3: put blob %s (exists)\n", name) 337 348 blobsDedupedCount.Inc() 338 349 blobsDedupedBytes.Add(float64(len(data))) 339 350 return nil ··· 341 352 } 342 353 343 354 func (s3 *S3Backend) DeleteBlob(ctx context.Context, name string) error { 344 344 - log.Printf("s3: delete blob %s\n", name) 355 355 + logc.Printf(ctx, "s3: delete blob %s\n", name) 345 356 346 357 return s3.client.RemoveObject(ctx, s3.bucket, blobObjectName(name), 347 358 minio.RemoveObjectOptions{}) 348 359 } 349 360 361 361 + func (s3 *S3Backend) EnumerateBlobs(ctx context.Context) iter.Seq2[BlobMetadata, error] { 362 362 + return func(yield func(BlobMetadata, error) bool) { 363 363 + logc.Print(ctx, "s3: enumerate blobs") 364 364 + 365 365 + ctx, cancel := context.WithCancel(ctx) 366 366 + defer cancel() 367 367 + 368 368 + prefix := "blob/" 369 369 + for object := range s3.client.ListObjectsIter(ctx, s3.bucket, minio.ListObjectsOptions{ 370 370 + Prefix: prefix, 371 371 + Recursive: true, 372 372 + }) { 373 373 + var metadata BlobMetadata 374 374 + var err error 375 375 + if err = object.Err; err == nil { 376 376 + key := strings.TrimPrefix(object.Key, prefix) 377 377 + if strings.HasSuffix(key, "/") { 378 378 + continue // directory; skip 379 379 + } else { 380 380 + metadata.Name = joinBlobName(strings.Split(key, "/")) 381 381 + metadata.Size = object.Size 382 382 + metadata.LastModified = object.LastModified 383 383 + } 384 384 + } 385 385 + if !yield(metadata, err) { 386 386 + break 387 387 + } 388 388 + } 389 389 + } 390 390 + } 391 391 + 350 392 func manifestObjectName(name string) string { 351 393 return fmt.Sprintf("site/%s", name) 352 394 } ··· 355 397 return fmt.Sprintf("dirty/%x", sha256.Sum256(manifestData)) 356 398 } 357 399 358 358 - func (s3 *S3Backend) ListManifests(ctx context.Context) (manifests []string, err error) { 359 359 - log.Print("s3: list manifests") 360 360 - 361 361 - ctx, cancel := context.WithCancel(ctx) 362 362 - defer cancel() 363 363 - 364 364 - prefix := manifestObjectName("") 365 365 - for object := range s3.client.ListObjectsIter(ctx, s3.bucket, minio.ListObjectsOptions{ 366 366 - Prefix: prefix, 367 367 - Recursive: true, 368 368 - }) { 369 369 - if object.Err != nil { 370 370 - return nil, object.Err 371 371 - } 372 372 - key := strings.TrimRight(strings.TrimPrefix(object.Key, prefix), "/") 373 373 - if strings.Count(key, "/") > 1 { 374 374 - continue 375 375 - } 376 376 - _, project, _ := strings.Cut(key, "/") 377 377 - if project == "" || strings.HasPrefix(project, ".") && project != ".index" { 378 378 - continue 379 379 - } 380 380 - manifests = append(manifests, key) 381 381 - } 382 382 - 383 383 - return 384 384 - } 385 385 - 386 400 type s3ManifestLoader struct { 387 401 s3 *S3Backend 388 402 } 389 403 390 390 - func (l s3ManifestLoader) Load(ctx context.Context, key string) (*CachedManifest, error) { 404 404 + func (l s3ManifestLoader) Load( 405 405 + ctx context.Context, key string, 406 406 + ) ( 407 407 + *CachedManifest, error, 408 408 + ) { 391 409 return l.load(ctx, key, nil) 392 410 } 393 411 394 394 - func (l s3ManifestLoader) Reload(ctx context.Context, key string, oldValue *CachedManifest) (*CachedManifest, error) { 412 412 + func (l s3ManifestLoader) Reload( 413 413 + ctx context.Context, key string, oldValue *CachedManifest, 414 414 + ) ( 415 415 + *CachedManifest, error, 416 416 + ) { 395 417 return l.load(ctx, key, oldValue) 396 418 } 397 419 398 398 - func (l s3ManifestLoader) load(ctx context.Context, name string, oldManifest *CachedManifest) (*CachedManifest, error) { 420 420 + func (l s3ManifestLoader) load( 421 421 + ctx context.Context, name string, oldManifest *CachedManifest, 422 422 + ) ( 423 423 + *CachedManifest, error, 424 424 + ) { 425 425 + logc.Printf(ctx, "s3: get manifest %s\n", name) 426 426 + 399 427 loader := func() (*CachedManifest, error) { 400 400 - log.Printf("s3: get manifest %s\n", name) 401 401 - 402 402 - startTime := time.Now() 403 403 - 404 428 opts := minio.GetObjectOptions{} 405 405 - if oldManifest != nil && oldManifest.etag != "" { 406 406 - opts.SetMatchETagExcept(oldManifest.etag) 429 429 + if oldManifest != nil && oldManifest.metadata.ETag != "" { 430 430 + opts.SetMatchETagExcept(oldManifest.metadata.ETag) 407 431 } 408 432 object, err := l.s3.client.GetObject(ctx, l.s3.bucket, manifestObjectName(name), opts) 409 433 // Note that many errors (e.g. NoSuchKey) will be reported only after this point. ··· 427 451 return nil, err 428 452 } 429 453 430 430 - s3GetObjectDurationSeconds. 431 431 - With(prometheus.Labels{"kind": "manifest"}). 432 432 - Observe(time.Since(startTime).Seconds()) 454 454 + metadata := ManifestMetadata{ 455 455 + LastModified: stat.LastModified, 456 456 + ETag: stat.ETag, 457 457 + } 458 458 + return &CachedManifest{manifest, uint32(len(data)), metadata, nil}, nil 459 459 + } 433 460 434 434 - return &CachedManifest{manifest, uint32(len(data)), stat.LastModified, stat.ETag, nil}, nil 461 461 + observer := func() (*CachedManifest, error) { 462 462 + cached, err := loader() 463 463 + var code = "OK" 464 464 + if resp, ok := err.(minio.ErrorResponse); ok { 465 465 + code = resp.Code 466 466 + } 467 467 + s3GetObjectResponseCount.With(prometheus.Labels{"kind": "manifest", "code": code}).Inc() 468 468 + return cached, err 435 469 } 436 470 437 437 - var cached *CachedManifest 438 438 - cached, err := loader() 471 471 + startTime := time.Now() 472 472 + cached, err := observer() 473 473 + s3GetObjectDurationSeconds. 474 474 + With(prometheus.Labels{"kind": "manifest"}). 475 475 + Observe(time.Since(startTime).Seconds()) 476 476 + 439 477 if err != nil { 440 440 - if errResp := minio.ToErrorResponse(err); errResp.Code == "NoSuchKey" { 441 441 - s3GetObjectErrorsCount.With(prometheus.Labels{"object_kind": "manifest"}).Inc() 478 478 + errResp := minio.ToErrorResponse(err) 479 479 + if errResp.Code == "NoSuchKey" { 442 480 err = fmt.Errorf("%w: %s", ErrObjectNotFound, errResp.Key) 443 443 - return &CachedManifest{nil, 1, time.Time{}, "", err}, nil 481 481 + return &CachedManifest{nil, 1, ManifestMetadata{}, err}, nil 444 482 } else if errResp.StatusCode == http.StatusNotModified && oldManifest != nil { 445 483 return oldManifest, nil 446 484 } else { ··· 454 492 func (s3 *S3Backend) GetManifest( 455 493 ctx context.Context, name string, opts GetManifestOptions, 456 494 ) ( 457 457 - manifest *Manifest, mtime time.Time, err error, 495 495 + manifest *Manifest, metadata ManifestMetadata, err error, 458 496 ) { 459 497 if opts.BypassCache { 460 498 entry, found := s3.siteCache.Cache.GetEntry(name) ··· 469 507 return 470 508 } else { 471 509 // This could be `manifest, mtime, nil` or `nil, time.Time{}, ErrObjectNotFound`. 472 472 - manifest, mtime, err = cached.manifest, cached.mtime, cached.err 510 510 + manifest, metadata, err = cached.manifest, cached.metadata, cached.err 473 511 return 474 512 } 475 513 } 476 514 477 515 func (s3 *S3Backend) StageManifest(ctx context.Context, manifest *Manifest) error { 478 516 data := EncodeManifest(manifest) 479 479 - log.Printf("s3: stage manifest %x\n", sha256.Sum256(data)) 517 517 + logc.Printf(ctx, "s3: stage manifest %x\n", sha256.Sum256(data)) 480 518 481 519 _, err := s3.client.PutObject(ctx, s3.bucket, stagedManifestObjectName(data), 482 520 bytes.NewReader(data), int64(len(data)), minio.PutObjectOptions{}) 483 521 return err 484 522 } 485 523 486 486 - func (s3 *S3Backend) CommitManifest(ctx context.Context, name string, manifest *Manifest) error { 524 524 + func domainFrozenObjectName(domain string) string { 525 525 + return manifestObjectName(fmt.Sprintf("%s/.frozen", domain)) 526 526 + } 527 527 + 528 528 + func (s3 *S3Backend) checkDomainFrozen(ctx context.Context, domain string) error { 529 529 + _, err := s3.client.StatObject(ctx, s3.bucket, domainFrozenObjectName(domain), 530 530 + minio.GetObjectOptions{}) 531 531 + if err == nil { 532 532 + return ErrDomainFrozen 533 533 + } else if errResp := minio.ToErrorResponse(err); errResp.Code == "NoSuchKey" { 534 534 + return nil 535 535 + } else { 536 536 + return err 537 537 + } 538 538 + } 539 539 + 540 540 + func (s3 *S3Backend) HasAtomicCAS(ctx context.Context) bool { 541 541 + // Support for `If-Unmodified-Since:` or `If-Match:` for PutObject requests is very spotty: 542 542 + // - AWS supports only `If-Match:`: 543 543 + // https://docs.aws.amazon.com/AmazonS3/latest/API/API_PutObject.html 544 544 + // - Minio supports `If-Match:`: 545 545 + // https://blog.min.io/leading-the-way-minios-conditional-write-feature-for-modern-data-workloads/ 546 546 + // - Tigris supports `If-Unmodified-Since:` and `If-Match:`, but only with `X-Tigris-Consistent: true`; 547 547 + // https://www.tigrisdata.com/docs/objects/conditionals/ 548 548 + // Note that the `X-Tigris-Consistent: true` header must be present on *every* transaction 549 549 + // touching the object, not just on the CAS transactions. 550 550 + // - Wasabi does not support either one and docs seem to suggest that the headers are ignored; 551 551 + // - Garage does not support either one and source code suggests the headers are ignored. 552 552 + // It seems that the only safe option is to not claim support for atomic CAS, and only do 553 553 + // best-effort CAS implementation using HeadObject and PutObject/DeleteObject. 554 554 + return false 555 555 + } 556 556 + 557 557 + func (s3 *S3Backend) checkManifestPrecondition( 558 558 + ctx context.Context, name string, opts ModifyManifestOptions, 559 559 + ) error { 560 560 + if opts.IfUnmodifiedSince.IsZero() && opts.IfMatch == "" { 561 561 + return nil 562 562 + } 563 563 + 564 564 + stat, err := s3.client.StatObject(ctx, s3.bucket, manifestObjectName(name), 565 565 + minio.GetObjectOptions{}) 566 566 + if err != nil { 567 567 + return err 568 568 + } 569 569 + 570 570 + if !opts.IfUnmodifiedSince.IsZero() && stat.LastModified.Compare(opts.IfUnmodifiedSince) > 0 { 571 571 + return fmt.Errorf("%w: If-Unmodified-Since", ErrPreconditionFailed) 572 572 + } 573 573 + if opts.IfMatch != "" && stat.ETag != opts.IfMatch { 574 574 + return fmt.Errorf("%w: If-Match", ErrPreconditionFailed) 575 575 + } 576 576 + 577 577 + return nil 578 578 + } 579 579 + 580 580 + func (s3 *S3Backend) CommitManifest( 581 581 + ctx context.Context, name string, manifest *Manifest, opts ModifyManifestOptions, 582 582 + ) error { 487 583 data := EncodeManifest(manifest) 488 488 - log.Printf("s3: commit manifest %x -> %s", sha256.Sum256(data), name) 584 584 + logc.Printf(ctx, "s3: commit manifest %x -> %s", sha256.Sum256(data), name) 585 585 + 586 586 + _, domain, _ := strings.Cut(name, "/") 587 587 + if err := s3.checkDomainFrozen(ctx, domain); err != nil { 588 588 + return err 589 589 + } 590 590 + 591 591 + if err := s3.checkManifestPrecondition(ctx, name, opts); err != nil { 592 592 + return err 593 593 + } 489 594 490 595 // Remove staged object unconditionally (whether commit succeeded or failed), since 491 596 // the upper layer has to retry the complete operation anyway. 597 597 + putOptions := minio.PutObjectOptions{} 598 598 + putOptions.Header().Add("X-Tigris-Consistent", "true") 599 599 + if opts.IfMatch != "" { 600 600 + // Not guaranteed to do anything (see `HasAtomicCAS`), but let's try anyway; 601 601 + // this is a "belt and suspenders" approach, together with `checkManifestPrecondition`. 602 602 + // It does reliably work on MinIO at least. 603 603 + putOptions.SetMatchETag(opts.IfMatch) 604 604 + } 492 605 _, putErr := s3.client.PutObject(ctx, s3.bucket, manifestObjectName(name), 493 493 - bytes.NewReader(data), int64(len(data)), minio.PutObjectOptions{}) 606 606 + bytes.NewReader(data), int64(len(data)), putOptions) 494 607 removeErr := s3.client.RemoveObject(ctx, s3.bucket, stagedManifestObjectName(data), 495 608 minio.RemoveObjectOptions{}) 496 609 s3.siteCache.Cache.Invalidate(name) 497 610 if putErr != nil { 498 498 - return putErr 611 611 + if errResp := minio.ToErrorResponse(putErr); errResp.Code == "PreconditionFailed" { 612 612 + return ErrPreconditionFailed 613 613 + } else { 614 614 + return putErr 615 615 + } 499 616 } else if removeErr != nil { 500 617 return removeErr 501 618 } else { ··· 503 620 } 504 621 } 505 622 506 506 - func (s3 *S3Backend) DeleteManifest(ctx context.Context, name string) error { 507 507 - log.Printf("s3: delete manifest %s\n", name) 623 623 + func (s3 *S3Backend) DeleteManifest( 624 624 + ctx context.Context, name string, opts ModifyManifestOptions, 625 625 + ) error { 626 626 + logc.Printf(ctx, "s3: delete manifest %s\n", name) 627 627 + 628 628 + _, domain, _ := strings.Cut(name, "/") 629 629 + if err := s3.checkDomainFrozen(ctx, domain); err != nil { 630 630 + return err 631 631 + } 632 632 + 633 633 + if err := s3.checkManifestPrecondition(ctx, name, opts); err != nil { 634 634 + return err 635 635 + } 508 636 509 637 err := s3.client.RemoveObject(ctx, s3.bucket, manifestObjectName(name), 510 638 minio.RemoveObjectOptions{}) ··· 512 640 return err 513 641 } 514 642 643 643 + func (s3 *S3Backend) EnumerateManifests(ctx context.Context) iter.Seq2[ManifestMetadata, error] { 644 644 + return func(yield func(ManifestMetadata, error) bool) { 645 645 + logc.Print(ctx, "s3: enumerate manifests") 646 646 + 647 647 + ctx, cancel := context.WithCancel(ctx) 648 648 + defer cancel() 649 649 + 650 650 + prefix := "site/" 651 651 + for object := range s3.client.ListObjectsIter(ctx, s3.bucket, minio.ListObjectsOptions{ 652 652 + Prefix: prefix, 653 653 + Recursive: true, 654 654 + }) { 655 655 + var metadata ManifestMetadata 656 656 + var err error 657 657 + if err = object.Err; err == nil { 658 658 + key := strings.TrimPrefix(object.Key, prefix) 659 659 + _, project, _ := strings.Cut(key, "/") 660 660 + if strings.HasSuffix(key, "/") { 661 661 + continue // directory; skip 662 662 + } else if project == "" || strings.HasPrefix(project, ".") && project != ".index" { 663 663 + continue // internal; skip 664 664 + } else { 665 665 + metadata.Name = key 666 666 + metadata.Size = object.Size 667 667 + metadata.LastModified = object.LastModified 668 668 + metadata.ETag = object.ETag 669 669 + } 670 670 + } 671 671 + if !yield(metadata, err) { 672 672 + break 673 673 + } 674 674 + } 675 675 + } 676 676 + } 677 677 + 515 678 func domainCheckObjectName(domain string) string { 516 679 return manifestObjectName(fmt.Sprintf("%s/.exists", domain)) 517 680 } 518 681 519 682 func (s3 *S3Backend) CheckDomain(ctx context.Context, domain string) (exists bool, err error) { 520 520 - log.Printf("s3: check domain %s\n", domain) 683 683 + logc.Printf(ctx, "s3: check domain %s\n", domain) 521 684 522 685 _, err = s3.client.StatObject(ctx, s3.bucket, domainCheckObjectName(domain), 523 686 minio.StatObjectOptions{}) ··· 548 711 } 549 712 550 713 func (s3 *S3Backend) CreateDomain(ctx context.Context, domain string) error { 551 551 - log.Printf("s3: create domain %s\n", domain) 714 714 + logc.Printf(ctx, "s3: create domain %s\n", domain) 552 715 553 716 _, err := s3.client.PutObject(ctx, s3.bucket, domainCheckObjectName(domain), 554 717 &bytes.Reader{}, 0, minio.PutObjectOptions{}) 555 718 return err 556 719 } 720 720 + 721 721 + func (s3 *S3Backend) FreezeDomain(ctx context.Context, domain string) error { 722 722 + logc.Printf(ctx, "s3: freeze domain %s\n", domain) 723 723 + 724 724 + _, err := s3.client.PutObject(ctx, s3.bucket, domainFrozenObjectName(domain), 725 725 + &bytes.Reader{}, 0, minio.PutObjectOptions{}) 726 726 + return err 727 727 + 728 728 + } 729 729 + 730 730 + func (s3 *S3Backend) UnfreezeDomain(ctx context.Context, domain string) error { 731 731 + logc.Printf(ctx, "s3: unfreeze domain %s\n", domain) 732 732 + 733 733 + err := s3.client.RemoveObject(ctx, s3.bucket, domainFrozenObjectName(domain), 734 734 + minio.RemoveObjectOptions{}) 735 735 + if errResp := minio.ToErrorResponse(err); errResp.Code == "NoSuchKey" { 736 736 + return nil 737 737 + } else { 738 738 + return err 739 739 + } 740 740 + } 741 741 + 742 742 + func auditObjectName(id AuditID) string { 743 743 + return fmt.Sprintf("audit/%s", id) 744 744 + } 745 745 + 746 746 + func (s3 *S3Backend) AppendAuditLog(ctx context.Context, id AuditID, record *AuditRecord) error { 747 747 + logc.Printf(ctx, "s3: append audit %s\n", id) 748 748 + 749 749 + name := auditObjectName(id) 750 750 + data := EncodeAuditRecord(record) 751 751 + 752 752 + options := minio.PutObjectOptions{} 753 753 + options.SetMatchETagExcept("*") // may or may not be supported 754 754 + _, err := s3.client.PutObject(ctx, s3.bucket, name, 755 755 + bytes.NewReader(data), int64(len(data)), options) 756 756 + if errResp := minio.ToErrorResponse(err); errResp.StatusCode == 412 { 757 757 + panic(fmt.Errorf("audit ID collision: %s", name)) 758 758 + } 759 759 + return err 760 760 + } 761 761 + 762 762 + func (s3 *S3Backend) QueryAuditLog(ctx context.Context, id AuditID) (*AuditRecord, error) { 763 763 + logc.Printf(ctx, "s3: read audit %s\n", id) 764 764 + 765 765 + object, err := s3.client.GetObject(ctx, s3.bucket, auditObjectName(id), 766 766 + minio.GetObjectOptions{}) 767 767 + if err != nil { 768 768 + return nil, err 769 769 + } 770 770 + defer object.Close() 771 771 + 772 772 + data, err := io.ReadAll(object) 773 773 + if err != nil { 774 774 + return nil, err 775 775 + } 776 776 + 777 777 + return DecodeAuditRecord(data) 778 778 + } 779 779 + 780 780 + func (s3 *S3Backend) SearchAuditLog( 781 781 + ctx context.Context, opts SearchAuditLogOptions, 782 782 + ) iter.Seq2[AuditID, error] { 783 783 + return func(yield func(AuditID, error) bool) { 784 784 + logc.Printf(ctx, "s3: search audit\n") 785 785 + 786 786 + ctx, cancel := context.WithCancel(ctx) 787 787 + defer cancel() 788 788 + 789 789 + prefix := "audit/" 790 790 + for object := range s3.client.ListObjectsIter(ctx, s3.bucket, minio.ListObjectsOptions{ 791 791 + Prefix: prefix, 792 792 + }) { 793 793 + var id AuditID 794 794 + var err error 795 795 + if object.Err != nil { 796 796 + err = object.Err 797 797 + } else { 798 798 + id, err = ParseAuditID(strings.TrimPrefix(object.Key, prefix)) 799 799 + } 800 800 + if !yield(id, err) { 801 801 + break 802 802 + } 803 803 + } 804 804 + } 805 805 + }

+34 -31

src/caddy.go

··· 1 1 package git_pages 2 2 3 3 import ( 4 4 + "context" 4 5 "crypto/tls" 5 6 "fmt" 6 6 - "log" 7 7 "net" 8 8 "net/http" 9 9 - "net/url" 10 9 "strings" 11 10 ) 12 11 ··· 22 21 // this isn't really what git-pages is designed for, and object store accesses can cost money. 23 22 // [^1]: https://letsencrypt.org/2025/07/01/issuing-our-first-ip-address-certificate 24 23 if ip := net.ParseIP(domain); ip != nil { 25 25 - log.Println("caddy:", domain, 404, "(bare IP)") 24 24 + logc.Println(r.Context(), "caddy:", domain, 404, "(bare IP)") 26 25 w.WriteHeader(http.StatusNotFound) 27 26 return 28 27 } ··· 35 34 // Pages v2, which would under some circumstances return certificates with subjectAltName 36 35 // not valid for the SNI. Go's TLS stack makes `tls.Dial` return an error for these, 37 36 // thankfully making it unnecessary to examine X.509 certificates manually here.) 38 38 - for _, wildcardConfig := range config.Wildcard { 39 39 - if wildcardConfig.FallbackProxyTo == "" { 40 40 - continue 41 41 - } 42 42 - fallbackURL, err := url.Parse(wildcardConfig.FallbackProxyTo) 43 43 - if err != nil { 44 44 - continue 45 45 - } 46 46 - if fallbackURL.Scheme != "https" { 47 47 - continue 48 48 - } 49 49 - connectHost := fallbackURL.Host 50 50 - if fallbackURL.Port() != "" { 51 51 - connectHost += ":" + fallbackURL.Port() 52 52 - } else { 53 53 - connectHost += ":443" 54 54 - } 55 55 - log.Printf("caddy: check TLS %s", fallbackURL) 56 56 - connection, err := tls.Dial("tcp", connectHost, &tls.Config{ServerName: domain}) 57 57 - if err != nil { 58 58 - continue 59 59 - } 60 60 - connection.Close() 61 61 - found = true 62 62 - break 37 37 + found, err = tryDialWithSNI(r.Context(), domain) 38 38 + if err != nil { 39 39 + logc.Printf(r.Context(), "caddy err: check SNI: %s\n", err) 63 40 } 64 41 } 65 42 66 43 if found { 67 67 - log.Println("caddy:", domain, 200) 44 44 + logc.Println(r.Context(), "caddy:", domain, 200) 68 45 w.WriteHeader(http.StatusOK) 69 46 } else if err == nil { 70 70 - log.Println("caddy:", domain, 404) 47 47 + logc.Println(r.Context(), "caddy:", domain, 404) 71 48 w.WriteHeader(http.StatusNotFound) 72 49 } else { 73 73 - log.Println("caddy:", domain, 500) 50 50 + logc.Println(r.Context(), "caddy:", domain, 500) 74 51 w.WriteHeader(http.StatusInternalServerError) 75 52 fmt.Fprintln(w, err) 76 53 } 77 54 } 55 55 + 56 56 + func tryDialWithSNI(ctx context.Context, domain string) (bool, error) { 57 57 + if config.Fallback.ProxyTo == nil { 58 58 + return false, nil 59 59 + } 60 60 + 61 61 + fallbackURL := config.Fallback.ProxyTo 62 62 + if fallbackURL.Scheme != "https" { 63 63 + return false, nil 64 64 + } 65 65 + 66 66 + connectHost := fallbackURL.Host 67 67 + if fallbackURL.Port() != "" { 68 68 + connectHost += ":" + fallbackURL.Port() 69 69 + } else { 70 70 + connectHost += ":443" 71 71 + } 72 72 + 73 73 + logc.Printf(ctx, "caddy: check TLS %s", fallbackURL) 74 74 + connection, err := tls.Dial("tcp", connectHost, &tls.Config{ServerName: domain}) 75 75 + if err != nil { 76 76 + return false, err 77 77 + } 78 78 + connection.Close() 79 79 + return true, nil 80 80 + }

+28 -22

src/collect.go

··· 5 5 "context" 6 6 "fmt" 7 7 "io" 8 8 - "time" 9 8 ) 10 9 11 10 type Flusher interface { ··· 14 13 15 14 // Inverse of `ExtractTar`. 16 15 func CollectTar( 17 17 - context context.Context, writer io.Writer, manifest *Manifest, manifestMtime time.Time, 16 16 + context context.Context, writer io.Writer, manifest *Manifest, metadata ManifestMetadata, 18 17 ) ( 19 18 err error, 20 19 ) { ··· 22 21 23 22 appendFile := func(header *tar.Header, data []byte, transform Transform) (err error) { 24 23 switch transform { 25 25 - case Transform_None: 26 26 - case Transform_Zstandard: 24 24 + case Transform_Identity: 25 25 + case Transform_Zstd: 27 26 data, err = zstdDecoder.DecodeAll(data, []byte{}) 28 27 if err != nil { 29 29 - return err 28 28 + return fmt.Errorf("zstd: %s: %w", header.Name, err) 30 29 } 31 30 default: 32 32 - return fmt.Errorf("unexpected transform") 31 31 + return fmt.Errorf("%s: unexpected transform", header.Name) 33 32 } 34 33 header.Size = int64(len(data)) 35 34 36 35 err = archive.WriteHeader(header) 37 36 if err != nil { 38 38 - return 37 37 + return fmt.Errorf("tar: %w", err) 39 38 } 40 39 _, err = archive.Write(data) 40 40 + if err != nil { 41 41 + return fmt.Errorf("tar: %w", err) 42 42 + } 41 43 return 42 44 } 43 45 ··· 52 54 case Type_Directory: 53 55 header.Typeflag = tar.TypeDir 54 56 header.Mode = 0755 55 55 - header.ModTime = manifestMtime 56 56 - err = appendFile(&header, nil, Transform_None) 57 57 + header.ModTime = metadata.LastModified 58 58 + err = appendFile(&header, nil, Transform_Identity) 57 59 58 60 case Type_InlineFile: 59 61 header.Typeflag = tar.TypeReg 60 62 header.Mode = 0644 61 61 - header.ModTime = manifestMtime 63 63 + header.ModTime = metadata.LastModified 62 64 err = appendFile(&header, entry.GetData(), entry.GetTransform()) 63 65 64 66 case Type_ExternalFile: 65 67 var blobReader io.Reader 66 66 - var blobMtime time.Time 68 68 + var blobMetadata BlobMetadata 67 69 var blobData []byte 68 68 - blobReader, _, blobMtime, err = backend.GetBlob(context, string(entry.Data)) 70 70 + blobReader, blobMetadata, err = backend.GetBlob(context, string(entry.Data)) 69 71 if err != nil { 70 72 return 71 73 } 72 72 - blobData, _ = io.ReadAll(blobReader) 74 74 + blobData, err = io.ReadAll(blobReader) 75 75 + if err != nil { 76 76 + return 77 77 + } 73 78 header.Typeflag = tar.TypeReg 74 79 header.Mode = 0644 75 75 - header.ModTime = blobMtime 80 80 + header.ModTime = blobMetadata.LastModified 76 81 err = appendFile(&header, blobData, entry.GetTransform()) 77 82 78 83 case Type_Symlink: 79 84 header.Typeflag = tar.TypeSymlink 80 85 header.Mode = 0644 81 81 - header.ModTime = manifestMtime 82 82 - err = appendFile(&header, entry.GetData(), Transform_None) 86 86 + header.ModTime = metadata.LastModified 87 87 + err = appendFile(&header, entry.GetData(), Transform_Identity) 83 88 84 89 default: 85 85 - return fmt.Errorf("unexpected entry type") 90 90 + panic(fmt.Errorf("CollectTar encountered invalid entry: %v, %v", 91 91 + entry.GetType(), entry.GetTransform())) 86 92 } 87 93 if err != nil { 88 94 return err ··· 94 100 Name: RedirectsFileName, 95 101 Typeflag: tar.TypeReg, 96 102 Mode: 0644, 97 97 - ModTime: manifestMtime, 98 98 - }, []byte(redirects), Transform_None) 103 103 + ModTime: metadata.LastModified, 104 104 + }, []byte(redirects), Transform_Identity) 99 105 if err != nil { 100 106 return err 101 107 } ··· 106 112 Name: HeadersFileName, 107 113 Typeflag: tar.TypeReg, 108 114 Mode: 0644, 109 109 - ModTime: manifestMtime, 110 110 - }, []byte(headers), Transform_None) 115 115 + ModTime: metadata.LastModified, 116 116 + }, []byte(headers), Transform_Identity) 111 117 if err != nil { 112 118 return err 113 119 } ··· 115 121 116 122 err = archive.Flush() 117 123 if err != nil { 118 118 - return err 124 124 + return fmt.Errorf("tar: %w", err) 119 125 } 120 126 121 127 flusher, ok := writer.(Flusher)

+73 -19

src/config.go

··· 4 4 "bytes" 5 5 "encoding/json" 6 6 "fmt" 7 7 + "net/url" 7 8 "os" 8 9 "reflect" 9 10 "slices" ··· 16 17 "github.com/pelletier/go-toml/v2" 17 18 ) 18 19 19 19 - // For some reason, the standard `time.Duration` type doesn't implement the standard 20 20 + // For an unknown reason, the standard `time.Duration` type doesn't implement the standard 20 21 // `encoding.{TextMarshaler,TextUnmarshaler}` interfaces. 21 22 type Duration time.Duration 22 23 ··· 26 27 27 28 func (t *Duration) UnmarshalText(data []byte) (err error) { 28 29 u, err := time.ParseDuration(string(data)) 29 29 - *t = Duration(u) 30 30 + if err == nil { 31 31 + *t = Duration(u) 32 32 + } 30 33 return 31 34 } 32 35 ··· 34 37 return []byte(t.String()), nil 35 38 } 36 39 40 40 + // For a known but upsetting reason, the standard `url.URL` type doesn't implement the standard 41 41 + // `encoding.{TextMarshaler,TextUnmarshaler}` interfaces. 42 42 + type URL struct { 43 43 + url.URL 44 44 + } 45 45 + 46 46 + func (t *URL) String() string { 47 47 + return fmt.Sprint(&t.URL) 48 48 + } 49 49 + 50 50 + func (t *URL) UnmarshalText(data []byte) (err error) { 51 51 + u, err := url.Parse(string(data)) 52 52 + if err == nil { 53 53 + *t = URL{*u} 54 54 + } 55 55 + return 56 56 + } 57 57 + 58 58 + func (t *URL) MarshalText() ([]byte, error) { 59 59 + return []byte(t.String()), nil 60 60 + } 61 61 + 37 62 type Config struct { 38 63 Insecure bool `toml:"-" env:"insecure"` 39 64 Features []string `toml:"features"` 40 65 LogFormat string `toml:"log-format" default:"text"` 41 66 Server ServerConfig `toml:"server"` 42 67 Wildcard []WildcardConfig `toml:"wildcard"` 68 68 + Fallback FallbackConfig `toml:"fallback"` 43 69 Storage StorageConfig `toml:"storage"` 44 70 Limits LimitsConfig `toml:"limits"` 71 71 + Audit AuditConfig `toml:"audit"` 45 72 Observability ObservabilityConfig `toml:"observability"` 46 73 } 47 74 48 75 type ServerConfig struct { 49 49 - Pages string `toml:"pages" default:"tcp/:3000"` 50 50 - Caddy string `toml:"caddy" default:"tcp/:3001"` 51 51 - Metrics string `toml:"metrics" default:"tcp/:3002"` 76 76 + Pages string `toml:"pages" default:"tcp/localhost:3000"` 77 77 + Caddy string `toml:"caddy" default:"tcp/localhost:3001"` 78 78 + Metrics string `toml:"metrics" default:"tcp/localhost:3002"` 52 79 } 53 80 54 81 type WildcardConfig struct { 55 55 - Domain string `toml:"domain"` 56 56 - CloneURL string `toml:"clone-url"` 57 57 - IndexRepos []string `toml:"index-repos" default:"[]"` 58 58 - IndexRepoBranch string `toml:"index-repo-branch" default:"pages"` 59 59 - Authorization string `toml:"authorization"` 60 60 - FallbackProxyTo string `toml:"fallback-proxy-to"` 61 61 - FallbackInsecure bool `toml:"fallback-insecure"` 82 82 + Domain string `toml:"domain"` 83 83 + CloneURL string `toml:"clone-url"` // URL template, not an exact URL 84 84 + IndexRepos []string `toml:"index-repos" default:"[]"` 85 85 + IndexRepoBranch string `toml:"index-repo-branch" default:"pages"` 86 86 + Authorization string `toml:"authorization"` 87 87 + } 88 88 + 89 89 + type FallbackConfig struct { 90 90 + ProxyTo *URL `toml:"proxy-to"` 91 91 + Insecure bool `toml:"insecure"` 62 92 } 63 93 64 94 type CacheConfig struct { ··· 109 139 // List of domains unconditionally forbidden for uploads. 110 140 ForbiddenDomains []string `toml:"forbidden-domains" default:"[]"` 111 141 // List of allowed repository URL prefixes. Setting this option prohibits uploading archives. 112 112 - AllowedRepositoryURLPrefixes []string `toml:"allowed-repository-url-prefixes"` 142 142 + AllowedRepositoryURLPrefixes []string `toml:"allowed-repository-url-prefixes" default:"[]"` 113 143 // List of allowed custom headers. Header name must be in the MIME canonical form, 114 144 // e.g. `Foo-Bar`. Setting this option permits including this custom header in `_headers`, 115 145 // unless it is fundamentally unsafe. 116 146 AllowedCustomHeaders []string `toml:"allowed-custom-headers" default:"[\"X-Clacks-Overhead\"]"` 117 147 } 118 148 149 149 + type AuditConfig struct { 150 150 + // Globally unique machine identifier (0 to 63 inclusive). 151 151 + NodeID int `toml:"node-id"` 152 152 + // Whether audit reports should be stored whenever an audit event occurs. 153 153 + Collect bool `toml:"collect"` 154 154 + // If not empty, includes the principal's IP address in audit reports, with the value specifying 155 155 + // the source of the IP address. If the value is "X-Forwarded-For", the last item of the 156 156 + // corresponding header field (assumed to be comma-separated) is used. If the value is 157 157 + // "RemoteAddr", the connecting host's address is used. Any other value is disallowed. 158 158 + IncludeIPs string `toml:"include-ip"` 159 159 + // Endpoint to notify with a `GET /<notify-url>?<id>` whenever an audit event occurs. 160 160 + NotifyURL *URL `toml:"notify-url"` 161 161 + } 162 162 + 119 163 type ObservabilityConfig struct { 120 164 // Minimum duration for an HTTP request transaction to be unconditionally sampled. 121 165 SlowResponseThreshold Duration `toml:"slow-response-threshold" default:"500ms"` 122 166 } 123 167 124 124 - func (config *Config) DebugJSON() string { 125 125 - result, err := json.MarshalIndent(config, "", " ") 168 168 + func (config *Config) TOML() string { 169 169 + result, err := toml.Marshal(config) 126 170 if err != nil { 127 171 panic(err) 128 172 } ··· 192 236 if valueCast, err = strconv.ParseBool(repr); err == nil { 193 237 reflValue.SetBool(valueCast) 194 238 } 239 239 + case int: 240 240 + var parsed int64 241 241 + if parsed, err = strconv.ParseInt(repr, 10, strconv.IntSize); err == nil { 242 242 + reflValue.SetInt(parsed) 243 243 + } 195 244 case uint: 196 245 var parsed uint64 197 246 if parsed, err = strconv.ParseUint(repr, 10, strconv.IntSize); err == nil { ··· 203 252 } 204 253 case datasize.ByteSize: 205 254 if valueCast, err = datasize.ParseString(repr); err == nil { 206 206 - reflValue.Set(reflect.ValueOf(valueCast)) 207 207 - } 208 208 - case time.Duration: 209 209 - if valueCast, err = time.ParseDuration(repr); err == nil { 210 255 reflValue.Set(reflect.ValueOf(valueCast)) 211 256 } 212 257 case Duration: 213 258 var parsed time.Duration 214 259 if parsed, err = time.ParseDuration(repr); err == nil { 215 260 reflValue.Set(reflect.ValueOf(Duration(parsed))) 261 261 + } 262 262 + case *URL: 263 263 + if repr == "" { 264 264 + reflValue.Set(reflect.ValueOf(nil)) 265 265 + } else { 266 266 + var parsed *url.URL 267 267 + if parsed, err = url.Parse(repr); err == nil { 268 268 + reflValue.Set(reflect.ValueOf(&URL{*parsed})) 269 269 + } 216 270 } 217 271 case []WildcardConfig: 218 272 var parsed []*WildcardConfig

+141 -59

src/extract.go

··· 5 5 "archive/zip" 6 6 "bytes" 7 7 "compress/gzip" 8 8 + "context" 8 9 "errors" 9 10 "fmt" 10 11 "io" 12 12 + "math" 11 13 "os" 12 14 "strings" 13 15 14 16 "github.com/c2h5oh/datasize" 17 17 + "github.com/go-git/go-git/v6/plumbing" 15 18 "github.com/klauspost/compress/zstd" 16 16 - "google.golang.org/protobuf/proto" 17 19 ) 18 20 19 21 var ErrArchiveTooLarge = errors.New("archive too large") 20 22 21 21 - func ExtractTar(reader io.Reader) (*Manifest, error) { 22 22 - // If the tar stream is itself compressed, both the outer and the inner bounds checks 23 23 - // are load-bearing. 24 24 - boundedReader := ReadAtMost(reader, int64(config.Limits.MaxSiteSize.Bytes()), 23 23 + func boundArchiveStream(reader io.Reader) io.Reader { 24 24 + return ReadAtMost(reader, int64(config.Limits.MaxSiteSize.Bytes()), 25 25 fmt.Errorf("%w: %s limit exceeded", ErrArchiveTooLarge, config.Limits.MaxSiteSize.HR())) 26 26 + } 26 27 27 27 - archive := tar.NewReader(boundedReader) 28 28 + func ExtractGzip( 29 29 + ctx context.Context, reader io.Reader, 30 30 + next func(context.Context, io.Reader) (*Manifest, error), 31 31 + ) (*Manifest, error) { 32 32 + stream, err := gzip.NewReader(reader) 33 33 + if err != nil { 34 34 + return nil, err 35 35 + } 36 36 + defer stream.Close() 28 37 29 29 - manifest := Manifest{ 30 30 - Contents: map[string]*Entry{ 31 31 - "": {Type: Type_Directory.Enum()}, 32 32 - }, 38 38 + return next(ctx, boundArchiveStream(stream)) 39 39 + } 40 40 + 41 41 + func ExtractZstd( 42 42 + ctx context.Context, reader io.Reader, 43 43 + next func(context.Context, io.Reader) (*Manifest, error), 44 44 + ) (*Manifest, error) { 45 45 + stream, err := zstd.NewReader(reader) 46 46 + if err != nil { 47 47 + return nil, err 33 48 } 49 49 + defer stream.Close() 50 50 + 51 51 + return next(ctx, boundArchiveStream(stream)) 52 52 + } 53 53 + 54 54 + const BlobReferencePrefix = "/git/blobs/" 55 55 + 56 56 + type UnresolvedRefError struct { 57 57 + missing []string 58 58 + } 59 59 + 60 60 + func (err UnresolvedRefError) Error() string { 61 61 + return fmt.Sprintf("%d unresolved blob references", len(err.missing)) 62 62 + } 63 63 + 64 64 + // Returns a map of git hash to entry. If `manifest` is nil, returns an empty map. 65 65 + func indexManifestByGitHash(manifest *Manifest) map[string]*Entry { 66 66 + index := map[string]*Entry{} 67 67 + for _, entry := range manifest.GetContents() { 68 68 + if hash := entry.GetGitHash(); hash != "" { 69 69 + if _, ok := plumbing.FromHex(hash); ok { 70 70 + index[hash] = entry 71 71 + } else { 72 72 + panic(fmt.Errorf("index: malformed hash: %s", hash)) 73 73 + } 74 74 + } 75 75 + } 76 76 + return index 77 77 + } 78 78 + 79 79 + func addSymlinkOrBlobReference( 80 80 + manifest *Manifest, fileName string, target string, 81 81 + index map[string]*Entry, missing *[]string, 82 82 + ) *Entry { 83 83 + if hash, found := strings.CutPrefix(target, BlobReferencePrefix); found { 84 84 + if entry, found := index[hash]; found { 85 85 + manifest.Contents[fileName] = entry 86 86 + return entry 87 87 + } else { 88 88 + *missing = append(*missing, hash) 89 89 + return nil 90 90 + } 91 91 + } else { 92 92 + return AddSymlink(manifest, fileName, target) 93 93 + } 94 94 + } 95 95 + 96 96 + func ExtractTar(ctx context.Context, reader io.Reader, oldManifest *Manifest) (*Manifest, error) { 97 97 + archive := tar.NewReader(reader) 98 98 + 99 99 + var dataBytesRecycled int64 100 100 + var dataBytesTransferred int64 101 101 + 102 102 + index := indexManifestByGitHash(oldManifest) 103 103 + missing := []string{} 104 104 + manifest := NewManifest() 34 105 for { 35 106 header, err := archive.Next() 36 107 if err == io.EOF { ··· 50 121 } 51 122 } 52 123 53 53 - manifestEntry := Entry{} 54 124 switch header.Typeflag { 55 125 case tar.TypeReg: 56 126 fileData, err := io.ReadAll(archive) 57 127 if err != nil { 58 128 return nil, fmt.Errorf("tar: %s: %w", fileName, err) 59 129 } 60 60 - 61 61 - manifestEntry.Type = Type_InlineFile.Enum() 62 62 - manifestEntry.Size = proto.Int64(header.Size) 63 63 - manifestEntry.Data = fileData 64 64 - 130 130 + AddFile(manifest, fileName, fileData) 131 131 + dataBytesTransferred += int64(len(fileData)) 65 132 case tar.TypeSymlink: 66 66 - manifestEntry.Type = Type_Symlink.Enum() 67 67 - manifestEntry.Size = proto.Int64(header.Size) 68 68 - manifestEntry.Data = []byte(header.Linkname) 69 69 - 133 133 + entry := addSymlinkOrBlobReference( 134 134 + manifest, fileName, header.Linkname, index, &missing) 135 135 + dataBytesRecycled += entry.GetOriginalSize() 70 136 case tar.TypeDir: 71 71 - manifestEntry.Type = Type_Directory.Enum() 72 72 - fileName = strings.TrimSuffix(fileName, "/") 73 73 - 137 137 + AddDirectory(manifest, fileName) 74 138 default: 75 75 - AddProblem(&manifest, fileName, "unsupported type '%c'", header.Typeflag) 139 139 + AddProblem(manifest, fileName, "tar: unsupported type '%c'", header.Typeflag) 76 140 continue 77 141 } 78 78 - manifest.Contents[fileName] = &manifestEntry 79 142 } 80 80 - return &manifest, nil 81 81 - } 82 143 83 83 - func ExtractTarGzip(reader io.Reader) (*Manifest, error) { 84 84 - stream, err := gzip.NewReader(reader) 85 85 - if err != nil { 86 86 - return nil, err 144 144 + if len(missing) > 0 { 145 145 + return nil, UnresolvedRefError{missing} 87 146 } 88 88 - defer stream.Close() 89 147 90 90 - // stream length is limited in `ExtractTar` 91 91 - return ExtractTar(stream) 92 92 - } 148 148 + // Ensure parent directories exist for all entries. 149 149 + EnsureLeadingDirectories(manifest) 93 150 94 94 - func ExtractTarZstd(reader io.Reader) (*Manifest, error) { 95 95 - stream, err := zstd.NewReader(reader) 96 96 - if err != nil { 97 97 - return nil, err 98 98 - } 99 99 - defer stream.Close() 151 151 + logc.Printf(ctx, 152 152 + "reuse: %s recycled, %s transferred\n", 153 153 + datasize.ByteSize(dataBytesRecycled).HR(), 154 154 + datasize.ByteSize(dataBytesTransferred).HR(), 155 155 + ) 100 156 101 101 - // stream length is limited in `ExtractTar` 102 102 - return ExtractTar(stream) 157 157 + return manifest, nil 103 158 } 104 159 105 105 - func ExtractZip(reader io.Reader) (*Manifest, error) { 160 160 + // Used for zstd decompression inside zip files, it is recommended to share this. 161 161 + var zstdDecomp = zstd.ZipDecompressor() 162 162 + 163 163 + func ExtractZip(ctx context.Context, reader io.Reader, oldManifest *Manifest) (*Manifest, error) { 106 164 data, err := io.ReadAll(reader) 107 165 if err != nil { 108 166 return nil, err ··· 112 170 if err != nil { 113 171 return nil, err 114 172 } 173 173 + 174 174 + // Support zstd compression inside zip files. 175 175 + archive.RegisterDecompressor(zstd.ZipMethodWinZip, zstdDecomp) 176 176 + archive.RegisterDecompressor(zstd.ZipMethodPKWare, zstdDecomp) 115 177 116 178 // Detect and defuse zipbombs. 117 179 var totalSize uint64 118 180 for _, file := range archive.File { 181 181 + if totalSize+file.UncompressedSize64 < totalSize { 182 182 + // Would overflow 183 183 + totalSize = math.MaxUint64 184 184 + break 185 185 + } 119 186 totalSize += file.UncompressedSize64 120 187 } 121 188 if totalSize > config.Limits.MaxSiteSize.Bytes() { ··· 126 193 ) 127 194 } 128 195 129 129 - manifest := Manifest{ 130 130 - Contents: map[string]*Entry{ 131 131 - "": {Type: Type_Directory.Enum()}, 132 132 - }, 133 133 - } 196 196 + var dataBytesRecycled int64 197 197 + var dataBytesTransferred int64 198 198 + 199 199 + index := indexManifestByGitHash(oldManifest) 200 200 + missing := []string{} 201 201 + manifest := NewManifest() 134 202 for _, file := range archive.File { 135 135 - manifestEntry := Entry{} 136 136 - if !strings.HasSuffix(file.Name, "/") { 203 203 + if strings.HasSuffix(file.Name, "/") { 204 204 + AddDirectory(manifest, file.Name) 205 205 + } else { 137 206 fileReader, err := file.Open() 138 207 if err != nil { 139 208 return nil, err ··· 146 215 } 147 216 148 217 if file.Mode()&os.ModeSymlink != 0 { 149 149 - manifestEntry.Type = Type_Symlink.Enum() 218 218 + entry := addSymlinkOrBlobReference( 219 219 + manifest, file.Name, string(fileData), index, &missing) 220 220 + dataBytesRecycled += entry.GetOriginalSize() 150 221 } else { 151 151 - manifestEntry.Type = Type_InlineFile.Enum() 222 222 + AddFile(manifest, file.Name, fileData) 223 223 + dataBytesTransferred += int64(len(fileData)) 152 224 } 153 153 - manifestEntry.Size = proto.Int64(int64(file.UncompressedSize64)) 154 154 - manifestEntry.Data = fileData 155 155 - } else { 156 156 - manifestEntry.Type = Type_Directory.Enum() 157 225 } 158 158 - manifest.Contents[strings.TrimSuffix(file.Name, "/")] = &manifestEntry 159 226 } 160 160 - return &manifest, nil 227 227 + 228 228 + if len(missing) > 0 { 229 229 + return nil, UnresolvedRefError{missing} 230 230 + } 231 231 + 232 232 + // Ensure parent directories exist for all entries. 233 233 + EnsureLeadingDirectories(manifest) 234 234 + 235 235 + logc.Printf(ctx, 236 236 + "reuse: %s recycled, %s transferred\n", 237 237 + datasize.ByteSize(dataBytesRecycled).HR(), 238 238 + datasize.ByteSize(dataBytesTransferred).HR(), 239 239 + ) 240 240 + 241 241 + return manifest, nil 161 242 } 243 243 +

+189 -49

src/fetch.go

··· 2 2 3 3 import ( 4 4 "context" 5 5 + "errors" 5 6 "fmt" 6 7 "io" 8 8 + "maps" 9 9 + "net/url" 7 10 "os" 11 11 + "slices" 8 12 13 13 + "github.com/c2h5oh/datasize" 9 14 "github.com/go-git/go-billy/v6/osfs" 10 15 "github.com/go-git/go-git/v6" 11 16 "github.com/go-git/go-git/v6/plumbing" 12 17 "github.com/go-git/go-git/v6/plumbing/cache" 13 18 "github.com/go-git/go-git/v6/plumbing/filemode" 14 19 "github.com/go-git/go-git/v6/plumbing/object" 20 20 + "github.com/go-git/go-git/v6/plumbing/protocol/packp" 21 21 + "github.com/go-git/go-git/v6/plumbing/transport" 15 22 "github.com/go-git/go-git/v6/storage/filesystem" 16 23 "google.golang.org/protobuf/proto" 17 24 ) 18 25 19 19 - func FetchRepository(ctx context.Context, repoURL string, branch string) (*Manifest, error) { 26 26 + var ErrRepositoryTooLarge = errors.New("repository too large") 27 27 + 28 28 + func FetchRepository( 29 29 + ctx context.Context, repoURL string, branch string, oldManifest *Manifest, 30 30 + ) ( 31 31 + *Manifest, error, 32 32 + ) { 20 33 span, ctx := ObserveFunction(ctx, "FetchRepository", 21 34 "git.repository", repoURL, "git.branch", branch) 22 35 defer span.Finish() 23 36 24 24 - baseDir, err := os.MkdirTemp("", "fetchRepo") 37 37 + parsedRepoURL, err := url.Parse(repoURL) 25 38 if err != nil { 26 26 - return nil, fmt.Errorf("mkdtemp: %w", err) 39 39 + return nil, fmt.Errorf("URL parse: %w", err) 27 40 } 28 28 - defer os.RemoveAll(baseDir) 29 41 30 30 - fs := osfs.New(baseDir, osfs.WithBoundOS()) 31 31 - cache := cache.NewObjectLRUDefault() 32 32 - storer := filesystem.NewStorageWithOptions(fs, cache, filesystem.Options{ 33 33 - ExclusiveAccess: true, 34 34 - LargeObjectThreshold: int64(config.Limits.GitLargeObjectThreshold.Bytes()), 35 35 - }) 36 36 - repo, err := git.CloneContext(ctx, storer, nil, &git.CloneOptions{ 37 37 - Bare: true, 38 38 - URL: repoURL, 39 39 - ReferenceName: plumbing.ReferenceName(branch), 40 40 - SingleBranch: true, 41 41 - Depth: 1, 42 42 - Tags: git.NoTags, 43 43 - }) 42 42 + var repo *git.Repository 43 43 + var storer *filesystem.Storage 44 44 + for _, filter := range []packp.Filter{packp.FilterBlobNone(), packp.Filter("")} { 45 45 + var tempDir string 46 46 + if tempDir, err = os.MkdirTemp("", "fetchRepo"); err != nil { 47 47 + return nil, fmt.Errorf("mkdtemp: %w", err) 48 48 + } 49 49 + defer os.RemoveAll(tempDir) 50 50 + 51 51 + storer = filesystem.NewStorageWithOptions( 52 52 + osfs.New(tempDir, osfs.WithBoundOS()), 53 53 + cache.NewObjectLRUDefault(), 54 54 + filesystem.Options{ 55 55 + ExclusiveAccess: true, 56 56 + LargeObjectThreshold: int64(config.Limits.GitLargeObjectThreshold.Bytes()), 57 57 + }, 58 58 + ) 59 59 + repo, err = git.CloneContext(ctx, storer, nil, &git.CloneOptions{ 60 60 + Bare: true, 61 61 + URL: repoURL, 62 62 + ReferenceName: plumbing.NewBranchReferenceName(branch), 63 63 + SingleBranch: true, 64 64 + Depth: 1, 65 65 + Tags: git.NoTags, 66 66 + Filter: filter, 67 67 + }) 68 68 + if err != nil { 69 69 + logc.Printf(ctx, "clone err: %s %s filter=%q\n", repoURL, branch, filter) 70 70 + continue 71 71 + } else { 72 72 + logc.Printf(ctx, "clone ok: %s %s filter=%q\n", repoURL, branch, filter) 73 73 + break 74 74 + } 75 75 + } 44 76 if err != nil { 45 77 return nil, fmt.Errorf("git clone: %w", err) 46 78 } ··· 63 95 walker := object.NewTreeWalker(tree, true, make(map[plumbing.Hash]bool)) 64 96 defer walker.Close() 65 97 66 66 - manifest := Manifest{ 67 67 - RepoUrl: proto.String(repoURL), 68 68 - Branch: proto.String(branch), 69 69 - Commit: proto.String(ref.Hash().String()), 70 70 - Contents: map[string]*Entry{ 71 71 - "": {Type: Type_Directory.Enum()}, 72 72 - }, 73 73 - } 98 98 + // Create a manifest for the tree object corresponding to `branch`, but do not populate it 99 99 + // with data yet; instead, record all the blobs we'll need. 100 100 + manifest := NewManifest() 101 101 + manifest.RepoUrl = proto.String(repoURL) 102 102 + manifest.Branch = proto.String(branch) 103 103 + manifest.Commit = proto.String(ref.Hash().String()) 104 104 + blobsNeeded := map[plumbing.Hash]*Entry{} 74 105 for { 75 106 name, entry, err := walker.Next() 76 107 if err == io.EOF { ··· 78 109 } else if err != nil { 79 110 return nil, fmt.Errorf("git walker: %w", err) 80 111 } else { 81 81 - manifestEntry := Entry{} 82 82 - if entry.Mode.IsFile() { 83 83 - blob, err := repo.BlobObject(entry.Hash) 84 84 - if err != nil { 85 85 - return nil, fmt.Errorf("git blob %s: %w", name, err) 86 86 - } 87 87 - 88 88 - reader, err := blob.Reader() 89 89 - if err != nil { 90 90 - return nil, fmt.Errorf("git blob open: %w", err) 91 91 - } 92 92 - defer reader.Close() 93 93 - 94 94 - data, err := io.ReadAll(reader) 95 95 - if err != nil { 96 96 - return nil, fmt.Errorf("git blob read: %w", err) 97 97 - } 98 98 - 112 112 + manifestEntry := &Entry{} 113 113 + if existingManifestEntry, found := blobsNeeded[entry.Hash]; found { 114 114 + // If the same blob is present twice, we only need to fetch it once (and both 115 115 + // instances will alias the same `Entry` structure in the manifest). 116 116 + manifestEntry = existingManifestEntry 117 117 + } else if entry.Mode.IsFile() { 118 118 + blobsNeeded[entry.Hash] = manifestEntry 99 119 if entry.Mode == filemode.Symlink { 100 120 manifestEntry.Type = Type_Symlink.Enum() 101 121 } else { 102 122 manifestEntry.Type = Type_InlineFile.Enum() 103 123 } 104 104 - manifestEntry.Size = proto.Int64(blob.Size) 105 105 - manifestEntry.Data = data 124 124 + manifestEntry.GitHash = proto.String(entry.Hash.String()) 106 125 } else if entry.Mode == filemode.Dir { 107 126 manifestEntry.Type = Type_Directory.Enum() 108 127 } else { 109 109 - AddProblem(&manifest, name, "unsupported mode %#o", entry.Mode) 128 128 + AddProblem(manifest, name, "unsupported mode %#o", entry.Mode) 110 129 continue 111 130 } 112 112 - manifest.Contents[name] = &manifestEntry 131 131 + manifest.Contents[name] = manifestEntry 113 132 } 114 133 } 115 115 - return &manifest, nil 134 134 + 135 135 + // Collect checkout statistics. 136 136 + var dataBytesRecycled int64 137 137 + var dataBytesTransferred int64 138 138 + 139 139 + // First, see if we can extract the blobs from the old manifest. This is the preferred option 140 140 + // because it avoids both network transfers and recompression. Note that we do not request 141 141 + // blobs from the backend under any circumstances to avoid creating a blob existence oracle. 142 142 + for _, oldManifestEntry := range oldManifest.GetContents() { 143 143 + if hash, ok := plumbing.FromHex(oldManifestEntry.GetGitHash()); ok { 144 144 + if manifestEntry, found := blobsNeeded[hash]; found { 145 145 + manifestEntry.Reset() 146 146 + proto.Merge(manifestEntry, oldManifestEntry) 147 147 + dataBytesRecycled += oldManifestEntry.GetOriginalSize() 148 148 + delete(blobsNeeded, hash) 149 149 + } 150 150 + } 151 151 + } 152 152 + 153 153 + // Second, fill the manifest entries with data from the git checkout we just made. 154 154 + // This will only succeed if a `blob:none` filter isn't supported and we got a full 155 155 + // clone despite asking for a partial clone. 156 156 + for hash, manifestEntry := range blobsNeeded { 157 157 + if err := readGitBlob(repo, hash, manifestEntry, &dataBytesTransferred); err == nil { 158 158 + delete(blobsNeeded, hash) 159 159 + } else if errors.Is(err, ErrRepositoryTooLarge) { 160 160 + return nil, err 161 161 + } 162 162 + } 163 163 + 164 164 + // Third, if we still don't have data for some manifest entries, re-establish a git transport 165 165 + // and request the missing blobs (only) from the server. 166 166 + if len(blobsNeeded) > 0 { 167 167 + client, err := transport.Get(parsedRepoURL.Scheme) 168 168 + if err != nil { 169 169 + return nil, fmt.Errorf("git transport: %w", err) 170 170 + } 171 171 + 172 172 + endpoint, err := transport.NewEndpoint(repoURL) 173 173 + if err != nil { 174 174 + return nil, fmt.Errorf("git endpoint: %w", err) 175 175 + } 176 176 + 177 177 + session, err := client.NewSession(storer, endpoint, nil) 178 178 + if err != nil { 179 179 + return nil, fmt.Errorf("git session: %w", err) 180 180 + } 181 181 + 182 182 + connection, err := session.Handshake(ctx, transport.UploadPackService) 183 183 + if err != nil { 184 184 + return nil, fmt.Errorf("git connection: %w", err) 185 185 + } 186 186 + defer connection.Close() 187 187 + 188 188 + if err := connection.Fetch(ctx, &transport.FetchRequest{ 189 189 + Wants: slices.Collect(maps.Keys(blobsNeeded)), 190 190 + Depth: 1, 191 191 + // Git CLI behaves like this, even if the wants above are references to blobs. 192 192 + Filter: "blob:none", 193 193 + }); err != nil && !errors.Is(err, transport.ErrNoChange) { 194 194 + return nil, fmt.Errorf("git blob fetch request: %w", err) 195 195 + } 196 196 + 197 197 + // All remaining blobs should now be available. 198 198 + for hash, manifestEntry := range blobsNeeded { 199 199 + if err := readGitBlob(repo, hash, manifestEntry, &dataBytesTransferred); err != nil { 200 200 + return nil, err 201 201 + } 202 202 + delete(blobsNeeded, hash) 203 203 + } 204 204 + } 205 205 + 206 206 + logc.Printf(ctx, 207 207 + "reuse: %s recycled, %s transferred\n", 208 208 + datasize.ByteSize(dataBytesRecycled).HR(), 209 209 + datasize.ByteSize(dataBytesTransferred).HR(), 210 210 + ) 211 211 + 212 212 + return manifest, nil 213 213 + } 214 214 + 215 215 + func readGitBlob( 216 216 + repo *git.Repository, hash plumbing.Hash, entry *Entry, bytesTransferred *int64, 217 217 + ) error { 218 218 + blob, err := repo.BlobObject(hash) 219 219 + if err != nil { 220 220 + return fmt.Errorf("git blob %s: %w", hash, err) 221 221 + } 222 222 + 223 223 + reader, err := blob.Reader() 224 224 + if err != nil { 225 225 + return fmt.Errorf("git blob open: %w", err) 226 226 + } 227 227 + defer reader.Close() 228 228 + 229 229 + data, err := io.ReadAll(reader) 230 230 + if err != nil { 231 231 + return fmt.Errorf("git blob read: %w", err) 232 232 + } 233 233 + 234 234 + switch entry.GetType() { 235 235 + case Type_InlineFile, Type_Symlink: 236 236 + // okay 237 237 + default: 238 238 + panic(fmt.Errorf("readGitBlob encountered invalid entry: %v, %v", 239 239 + entry.GetType(), entry.GetTransform())) 240 240 + } 241 241 + 242 242 + entry.Data = data 243 243 + entry.Transform = Transform_Identity.Enum() 244 244 + entry.OriginalSize = proto.Int64(blob.Size) 245 245 + entry.CompressedSize = proto.Int64(blob.Size) 246 246 + 247 247 + *bytesTransferred += blob.Size 248 248 + if uint64(*bytesTransferred) > config.Limits.MaxSiteSize.Bytes() { 249 249 + return fmt.Errorf("%w: fetch exceeds %s limit", 250 250 + ErrRepositoryTooLarge, 251 251 + config.Limits.MaxSiteSize.HR(), 252 252 + ) 253 253 + } 254 254 + 255 255 + return nil 116 256 }

+16

src/flock_other.go

··· 1 1 + //go:build !unix 2 2 + 3 3 + package git_pages 4 4 + 5 5 + import ( 6 6 + "fmt" 7 7 + "os" 8 8 + ) 9 9 + 10 10 + func FileLock(file *os.File) error { 11 11 + return fmt.Errorf("unimplemented") 12 12 + } 13 13 + 14 14 + func FileUnlock(file *os.File) error { 15 15 + return fmt.Errorf("unimplemented") 16 16 + }

+16

src/flock_posix.go

··· 1 1 + //go:build unix 2 2 + 3 3 + package git_pages 4 4 + 5 5 + import ( 6 6 + "os" 7 7 + "syscall" 8 8 + ) 9 9 + 10 10 + func FileLock(file *os.File) error { 11 11 + return syscall.Flock(int(file.Fd()), syscall.LOCK_EX) 12 12 + } 13 13 + 14 14 + func FileUnlock(file *os.File) error { 15 15 + return syscall.Flock(int(file.Fd()), syscall.LOCK_UN) 16 16 + }

+88

src/garbage.go

··· 1 1 + package git_pages 2 2 + 3 3 + import ( 4 4 + "context" 5 5 + "fmt" 6 6 + 7 7 + "github.com/c2h5oh/datasize" 8 8 + "github.com/dghubble/trie" 9 9 + ) 10 10 + 11 11 + func trieReduce(data trie.Trier) (items, total int64) { 12 12 + data.Walk(func(key string, value any) error { 13 13 + items += 1 14 14 + total += *value.(*int64) 15 15 + return nil 16 16 + }) 17 17 + return 18 18 + } 19 19 + 20 20 + func TraceGarbage(ctx context.Context) error { 21 21 + allBlobs := trie.NewRuneTrie() 22 22 + liveBlobs := trie.NewRuneTrie() 23 23 + 24 24 + traceManifest := func(manifestName string, manifest *Manifest) error { 25 25 + for _, entry := range manifest.GetContents() { 26 26 + if entry.GetType() == Type_ExternalFile { 27 27 + blobName := string(entry.Data) 28 28 + if size := allBlobs.Get(blobName); size == nil { 29 29 + return fmt.Errorf("%s: dangling reference %s", manifestName, blobName) 30 30 + } else { 31 31 + liveBlobs.Put(blobName, size) 32 32 + } 33 33 + } 34 34 + } 35 35 + return nil 36 36 + } 37 37 + 38 38 + // Enumerate all blobs. 39 39 + for metadata, err := range backend.EnumerateBlobs(ctx) { 40 40 + if err != nil { 41 41 + return fmt.Errorf("trace blobs err: %w", err) 42 42 + } 43 43 + allBlobs.Put(metadata.Name, &metadata.Size) 44 44 + } 45 45 + 46 46 + // Enumerate blobs live via site manifests. 47 47 + for metadata, err := range backend.EnumerateManifests(ctx) { 48 48 + if err != nil { 49 49 + return fmt.Errorf("trace sites err: %w", err) 50 50 + } 51 51 + manifest, _, err := backend.GetManifest(ctx, metadata.Name, GetManifestOptions{}) 52 52 + if err != nil { 53 53 + return fmt.Errorf("trace sites err: %w", err) 54 54 + } 55 55 + err = traceManifest(metadata.Name, manifest) 56 56 + if err != nil { 57 57 + return fmt.Errorf("trace sites err: %w", err) 58 58 + } 59 59 + } 60 60 + 61 61 + // Enumerate blobs live via audit records. 62 62 + for auditID, err := range backend.SearchAuditLog(ctx, SearchAuditLogOptions{}) { 63 63 + if err != nil { 64 64 + return fmt.Errorf("trace audit err: %w", err) 65 65 + } 66 66 + auditRecord, err := backend.QueryAuditLog(ctx, auditID) 67 67 + if err != nil { 68 68 + return fmt.Errorf("trace audit err: %w", err) 69 69 + } 70 70 + if auditRecord.Manifest != nil { 71 71 + err = traceManifest(auditID.String(), auditRecord.Manifest) 72 72 + if err != nil { 73 73 + return fmt.Errorf("trace audit err: %w", err) 74 74 + } 75 75 + } 76 76 + } 77 77 + 78 78 + allBlobsCount, allBlobsSize := trieReduce(allBlobs) 79 79 + liveBlobsCount, liveBlobsSize := trieReduce(liveBlobs) 80 80 + logc.Printf(ctx, "trace all: %d blobs, %s", 81 81 + allBlobsCount, datasize.ByteSize(allBlobsSize).HR()) 82 82 + logc.Printf(ctx, "trace live: %d blobs, %s", 83 83 + liveBlobsCount, datasize.ByteSize(liveBlobsSize).HR()) 84 84 + logc.Printf(ctx, "trace dead: %d blobs, %s", 85 85 + allBlobsCount-liveBlobsCount, datasize.ByteSize(allBlobsSize-liveBlobsSize).HR()) 86 86 + 87 87 + return nil 88 88 + }

+114 -26

src/http.go

··· 2 2 3 3 import ( 4 4 "cmp" 5 5 + "fmt" 6 6 + "net" 7 7 + "net/http" 5 8 "regexp" 6 9 "slices" 7 10 "strconv" 8 11 "strings" 9 12 ) 10 13 11 11 - var httpAcceptEncodingRegexp = regexp.MustCompile(`` + 14 14 + var httpAcceptRegexp = regexp.MustCompile(`` + 12 15 // token optionally prefixed by whitespace 13 13 - `^[ \t]*([a-zA-Z0-9$!#$%&'*+.^_\x60|~-]+)` + 16 16 + `^[ \t]*([a-zA-Z0-9$!#$%&'*+./^_\x60|~-]+)` + 14 17 // quality value prefixed by a semicolon optionally surrounded by whitespace 15 18 `(?:[ \t]*;[ \t]*q=(0(?:\.[0-9]{1,3})?|1(?:\.0{1,3})?))?` + 16 19 // optional whitespace followed by comma or end of line 17 20 `[ \t]*(?:,|$)`, 18 21 ) 19 22 20 20 - type httpEncoding struct { 23 23 + type httpAcceptOffer struct { 21 24 code string 22 25 qval float64 23 26 } 24 27 25 25 - type httpEncodings struct { 26 26 - encodings []httpEncoding 27 27 - } 28 28 - 29 29 - func parseHTTPEncodings(headerValue string) (result httpEncodings) { 28 28 + func parseGenericAcceptHeader(headerValue string) (result []httpAcceptOffer) { 30 29 for headerValue != "" { 31 31 - matches := httpAcceptEncodingRegexp.FindStringSubmatch(headerValue) 30 30 + matches := httpAcceptRegexp.FindStringSubmatch(headerValue) 32 31 if matches == nil { 33 33 - return httpEncodings{} 32 32 + return 34 33 } 35 35 - enc := httpEncoding{strings.ToLower(matches[1]), 1.0} 34 34 + offer := httpAcceptOffer{strings.ToLower(matches[1]), 1.0} 36 35 if matches[2] != "" { 37 37 - enc.qval, _ = strconv.ParseFloat(matches[2], 64) 36 36 + offer.qval, _ = strconv.ParseFloat(matches[2], 64) 38 37 } 39 39 - result.encodings = append(result.encodings, enc) 38 38 + result = append(result, offer) 40 39 headerValue = headerValue[len(matches[0]):] 41 40 } 41 41 + return 42 42 + } 43 43 + 44 44 + func preferredAcceptOffer(offers []httpAcceptOffer) string { 45 45 + slices.SortStableFunc(offers, func(a, b httpAcceptOffer) int { 46 46 + return -cmp.Compare(a.qval, b.qval) 47 47 + }) 48 48 + for _, offer := range offers { 49 49 + if offer.qval != 0 { 50 50 + return offer.code 51 51 + } 52 52 + } 53 53 + return "" 54 54 + } 55 55 + 56 56 + type HTTPContentTypes struct { 57 57 + contentTypes []httpAcceptOffer 58 58 + } 59 59 + 60 60 + func ParseAcceptHeader(headerValue string) (result HTTPContentTypes) { 61 61 + if headerValue == "" { 62 62 + headerValue = "*/*" 63 63 + } 64 64 + result = HTTPContentTypes{parseGenericAcceptHeader(headerValue)} 65 65 + return 66 66 + } 67 67 + 68 68 + func (e *HTTPContentTypes) Negotiate(offers ...string) string { 69 69 + prefs := make(map[string]float64, len(offers)) 70 70 + for _, code := range offers { 71 71 + prefs[code] = 0 72 72 + } 73 73 + for _, ctyp := range e.contentTypes { 74 74 + if ctyp.code == "*/*" { 75 75 + for code := range prefs { 76 76 + prefs[code] = ctyp.qval 77 77 + } 78 78 + } else if _, ok := prefs[ctyp.code]; ok { 79 79 + prefs[ctyp.code] = ctyp.qval 80 80 + } 81 81 + } 82 82 + ctyps := make([]httpAcceptOffer, len(offers)) 83 83 + for idx, code := range offers { 84 84 + ctyps[idx] = httpAcceptOffer{code, prefs[code]} 85 85 + } 86 86 + return preferredAcceptOffer(ctyps) 87 87 + } 88 88 + 89 89 + type HTTPEncodings struct { 90 90 + encodings []httpAcceptOffer 91 91 + } 92 92 + 93 93 + func ParseAcceptEncodingHeader(headerValue string) (result HTTPEncodings) { 94 94 + result = HTTPEncodings{parseGenericAcceptHeader(headerValue)} 42 95 if len(result.encodings) == 0 { 43 96 // RFC 9110 says (https://httpwg.org/specs/rfc9110.html#field.accept-encoding): 44 97 // "If no Accept-Encoding header field is in the request, any content ··· 51 104 52 105 // Negotiate returns the most preferred encoding that is acceptable by the 53 106 // client, or an empty string if no encodings are acceptable. 54 54 - func (e *httpEncodings) Negotiate(codes ...string) string { 55 55 - prefs := make(map[string]float64, len(codes)) 56 56 - for _, code := range codes { 107 107 + func (e *HTTPEncodings) Negotiate(offers ...string) string { 108 108 + prefs := make(map[string]float64, len(offers)) 109 109 + for _, code := range offers { 57 110 prefs[code] = 0 58 111 } 59 112 implicitIdentity := true ··· 73 126 if _, ok := prefs["identity"]; ok && implicitIdentity { 74 127 prefs["identity"] = -1 // sort last 75 128 } 76 76 - encs := make([]httpEncoding, len(codes)) 77 77 - for idx, code := range codes { 78 78 - encs[idx] = httpEncoding{code, prefs[code]} 129 129 + encs := make([]httpAcceptOffer, len(offers)) 130 130 + for idx, code := range offers { 131 131 + encs[idx] = httpAcceptOffer{code, prefs[code]} 79 132 } 80 80 - slices.SortStableFunc(encs, func(a, b httpEncoding) int { 81 81 - return -cmp.Compare(a.qval, b.qval) 82 82 - }) 83 83 - for _, enc := range encs { 84 84 - if enc.qval != 0 { 85 85 - return enc.code 133 133 + return preferredAcceptOffer(encs) 134 134 + } 135 135 + 136 136 + func chainHTTPMiddleware(middleware ...func(http.Handler) http.Handler) func(http.Handler) http.Handler { 137 137 + return func(handler http.Handler) http.Handler { 138 138 + for idx := len(middleware) - 1; idx >= 0; idx-- { 139 139 + handler = middleware[idx](handler) 86 140 } 141 141 + return handler 87 142 } 88 88 - return "" 143 143 + } 144 144 + 145 145 + func remoteAddrMiddleware(handler http.Handler) http.Handler { 146 146 + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 147 147 + var readXForwardedFor bool 148 148 + switch config.Audit.IncludeIPs { 149 149 + case "X-Forwarded-For": 150 150 + readXForwardedFor = true 151 151 + case "RemoteAddr", "": 152 152 + readXForwardedFor = false 153 153 + default: 154 154 + panic(fmt.Errorf("config.Audit.IncludeIPs is set to an unknown value (%q)", 155 155 + config.Audit.IncludeIPs)) 156 156 + } 157 157 + 158 158 + usingOriginalRemoteAddr := true 159 159 + if readXForwardedFor { 160 160 + forwardedFor := strings.Split(r.Header.Get("X-Forwarded-For"), ",") 161 161 + if len(forwardedFor) > 0 { 162 162 + remoteAddr := strings.TrimSpace(forwardedFor[len(forwardedFor)-1]) 163 163 + if remoteAddr != "" { 164 164 + r.RemoteAddr = remoteAddr 165 165 + usingOriginalRemoteAddr = false 166 166 + } 167 167 + } 168 168 + } 169 169 + if usingOriginalRemoteAddr { 170 170 + if ipAddress, _, err := net.SplitHostPort(r.RemoteAddr); err == nil { 171 171 + r.RemoteAddr = ipAddress 172 172 + } 173 173 + } 174 174 + 175 175 + handler.ServeHTTP(w, r) 176 176 + }) 89 177 }

+54

src/log.go

··· 1 1 + package git_pages 2 2 + 3 3 + import ( 4 4 + "context" 5 5 + "fmt" 6 6 + "log/slog" 7 7 + "os" 8 8 + "runtime" 9 9 + "strings" 10 10 + "time" 11 11 + ) 12 12 + 13 13 + var logc slogWithCtx 14 14 + 15 15 + type slogWithCtx struct{} 16 16 + 17 17 + func (l slogWithCtx) log(ctx context.Context, level slog.Level, msg string) { 18 18 + if ctx == nil { 19 19 + ctx = context.Background() 20 20 + } 21 21 + logger := slog.Default() 22 22 + if !logger.Enabled(ctx, level) { 23 23 + return 24 24 + } 25 25 + 26 26 + var pcs [1]uintptr 27 27 + // skip [runtime.Callers, this method, method calling this method] 28 28 + runtime.Callers(3, pcs[:]) 29 29 + 30 30 + record := slog.NewRecord(time.Now(), level, strings.TrimRight(msg, "\n"), pcs[0]) 31 31 + logger.Handler().Handle(ctx, record) 32 32 + } 33 33 + 34 34 + func (l slogWithCtx) Print(ctx context.Context, v ...any) { 35 35 + l.log(ctx, slog.LevelInfo, fmt.Sprint(v...)) 36 36 + } 37 37 + 38 38 + func (l slogWithCtx) Printf(ctx context.Context, format string, v ...any) { 39 39 + l.log(ctx, slog.LevelInfo, fmt.Sprintf(format, v...)) 40 40 + } 41 41 + 42 42 + func (l slogWithCtx) Println(ctx context.Context, v ...any) { 43 43 + l.log(ctx, slog.LevelInfo, fmt.Sprintln(v...)) 44 44 + } 45 45 + 46 46 + func (l slogWithCtx) Fatalf(ctx context.Context, format string, v ...any) { 47 47 + l.log(ctx, slog.LevelError, fmt.Sprintf(format, v...)) 48 48 + os.Exit(1) 49 49 + } 50 50 + 51 51 + func (l slogWithCtx) Fatalln(ctx context.Context, v ...any) { 52 52 + l.log(ctx, slog.LevelError, fmt.Sprintln(v...)) 53 53 + os.Exit(1) 54 54 + }

+318 -101

src/main.go

··· 2 2 3 3 import ( 4 4 "context" 5 5 + "crypto/tls" 5 6 "errors" 6 7 "flag" 7 8 "fmt" ··· 10 11 "log/slog" 11 12 "net" 12 13 "net/http" 14 14 + "net/http/httputil" 13 15 "net/url" 14 16 "os" 17 17 + "path" 15 18 "runtime/debug" 16 19 "strings" 20 20 + "time" 17 21 18 22 automemlimit "github.com/KimMachineGun/automemlimit/memlimit" 19 23 "github.com/c2h5oh/datasize" 24 24 + "github.com/fatih/color" 25 25 + "github.com/kankanreno/go-snowflake" 20 26 "github.com/prometheus/client_golang/prometheus/promhttp" 27 27 + "google.golang.org/protobuf/proto" 21 28 ) 22 29 23 30 var config *Config 24 31 var wildcards []*WildcardPattern 32 32 + var fallback http.Handler 25 33 var backend Backend 26 34 27 27 - func configureFeatures() (err error) { 35 35 + func configureFeatures(ctx context.Context) (err error) { 28 36 if len(config.Features) > 0 { 29 29 - log.Println("features:", strings.Join(config.Features, ", ")) 37 37 + logc.Println(ctx, "features:", strings.Join(config.Features, ", ")) 30 38 } 31 39 return 32 40 } 33 41 34 34 - func configureMemLimit() (err error) { 42 42 + func configureMemLimit(ctx context.Context) (err error) { 35 43 // Avoid being OOM killed by not garbage collecting early enough. 36 44 memlimitBefore := datasize.ByteSize(debug.SetMemoryLimit(-1)) 37 45 automemlimit.SetGoMemLimitWithOpts( ··· 46 54 ) 47 55 memlimitAfter := datasize.ByteSize(debug.SetMemoryLimit(-1)) 48 56 if memlimitBefore == memlimitAfter { 49 49 - log.Println("memlimit: now", memlimitBefore.HR()) 57 57 + logc.Println(ctx, "memlimit: now", memlimitBefore.HR()) 50 58 } else { 51 51 - log.Println("memlimit: was", memlimitBefore.HR(), "now", memlimitAfter.HR()) 59 59 + logc.Println(ctx, "memlimit: was", memlimitBefore.HR(), "now", memlimitAfter.HR()) 52 60 } 53 61 return 54 62 } 55 63 56 56 - func configureWildcards() (err error) { 64 64 + func configureWildcards(_ context.Context) (err error) { 57 65 newWildcards, err := TranslateWildcards(config.Wildcard) 58 66 if err != nil { 59 67 return err ··· 63 71 } 64 72 } 65 73 66 66 - func listen(name string, listen string) net.Listener { 74 74 + func configureFallback(_ context.Context) (err error) { 75 75 + if config.Fallback.ProxyTo != nil { 76 76 + fallbackURL := &config.Fallback.ProxyTo.URL 77 77 + fallback = &httputil.ReverseProxy{ 78 78 + Rewrite: func(r *httputil.ProxyRequest) { 79 79 + r.SetURL(fallbackURL) 80 80 + r.Out.Host = r.In.Host 81 81 + r.Out.Header["X-Forwarded-For"] = r.In.Header["X-Forwarded-For"] 82 82 + }, 83 83 + Transport: &http.Transport{ 84 84 + TLSClientConfig: &tls.Config{ 85 85 + InsecureSkipVerify: config.Fallback.Insecure, 86 86 + }, 87 87 + }, 88 88 + } 89 89 + } 90 90 + return 91 91 + } 92 92 + 93 93 + // Thread-unsafe, must be called only during initial configuration. 94 94 + func configureAudit(_ context.Context) (err error) { 95 95 + snowflake.SetStartTime(time.Date(2025, 12, 1, 0, 0, 0, 0, time.UTC)) 96 96 + snowflake.SetMachineID(config.Audit.NodeID) 97 97 + return 98 98 + } 99 99 + 100 100 + func listen(ctx context.Context, name string, listen string) net.Listener { 67 101 if listen == "-" { 68 102 return nil 69 103 } 70 104 71 105 protocol, address, ok := strings.Cut(listen, "/") 72 106 if !ok { 73 73 - log.Fatalf("%s: %s: malformed endpoint", name, listen) 107 107 + logc.Fatalf(ctx, "%s: %s: malformed endpoint", name, listen) 74 108 } 75 109 76 110 listener, err := net.Listen(protocol, address) 77 111 if err != nil { 78 78 - log.Fatalf("%s: %s\n", name, err) 112 112 + logc.Fatalf(ctx, "%s: %s\n", name, err) 79 113 } 80 114 81 115 return listener ··· 85 119 return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 86 120 defer func() { 87 121 if err := recover(); err != nil { 88 88 - log.Printf("panic: %s %s %s: %s\n%s", 122 122 + logc.Printf(r.Context(), "panic: %s %s %s: %s\n%s", 89 123 r.Method, r.Host, r.URL.Path, err, string(debug.Stack())) 90 124 http.Error(w, 91 125 fmt.Sprintf("internal server error: %s", err), ··· 97 131 }) 98 132 } 99 133 100 100 - func serve(listener net.Listener, handler http.Handler) { 134 134 + func serve(ctx context.Context, listener net.Listener, handler http.Handler) { 101 135 if listener != nil { 102 102 - handler = panicHandler(handler) 103 103 - 104 136 server := http.Server{Handler: handler} 105 137 server.Protocols = new(http.Protocols) 106 138 server.Protocols.SetHTTP1(true) 107 107 - if config.Feature("serve-h2c") { 108 108 - server.Protocols.SetUnencryptedHTTP2(true) 109 109 - } 110 110 - log.Fatalln(server.Serve(listener)) 139 139 + server.Protocols.SetUnencryptedHTTP2(true) 140 140 + logc.Fatalln(ctx, server.Serve(listener)) 111 141 } 112 142 } 113 143 ··· 118 148 case 1: 119 149 return arg 120 150 default: 121 121 - log.Fatalf("webroot argument must be either 'domain.tld' or 'domain.tld/dir") 151 151 + logc.Fatalln(context.Background(), 152 152 + "webroot argument must be either 'domain.tld' or 'domain.tld/dir") 122 153 return "" 123 154 } 124 155 } ··· 130 161 } else { 131 162 writer, err = os.Create(flag.Arg(0)) 132 163 if err != nil { 133 133 - log.Fatalln(err) 164 164 + logc.Fatalln(context.Background(), err) 134 165 } 135 166 } 136 167 return ··· 140 171 fmt.Fprintf(os.Stderr, "Usage:\n") 141 172 fmt.Fprintf(os.Stderr, "(server) "+ 142 173 "git-pages [-config <file>|-no-config]\n") 143 143 - fmt.Fprintf(os.Stderr, "(admin) "+ 144 144 - "git-pages {-run-migration <name>}\n") 145 174 fmt.Fprintf(os.Stderr, "(info) "+ 146 175 "git-pages {-print-config-env-vars|-print-config}\n") 147 147 - fmt.Fprintf(os.Stderr, "(cli) "+ 176 176 + fmt.Fprintf(os.Stderr, "(debug) "+ 177 177 + "git-pages {-list-blobs|-list-manifests}\n") 178 178 + fmt.Fprintf(os.Stderr, "(debug) "+ 148 179 "git-pages {-get-blob|-get-manifest|-get-archive|-update-site} <ref> [file]\n") 180 180 + fmt.Fprintf(os.Stderr, "(admin) "+ 181 181 + "git-pages {-freeze-domain <domain>|-unfreeze-domain <domain>}\n") 182 182 + fmt.Fprintf(os.Stderr, "(audit) "+ 183 183 + "git-pages {-audit-log|-audit-read <id>|-audit-server <endpoint> <program> [args...]}\n") 184 184 + fmt.Fprintf(os.Stderr, "(maint) "+ 185 185 + "git-pages {-run-migration <name>|-trace-garbage}\n") 149 186 flag.PrintDefaults() 150 187 } 151 188 152 189 func Main() { 190 190 + ctx := context.Background() 191 191 + 153 192 flag.Usage = usage 154 154 - printConfigEnvVars := flag.Bool("print-config-env-vars", false, 155 155 - "print every recognized configuration environment variable and exit") 156 156 - printConfig := flag.Bool("print-config", false, 157 157 - "print configuration as JSON and exit") 158 193 configTomlPath := flag.String("config", "", 159 194 "load configuration from `filename` (default: 'config.toml')") 160 195 noConfig := flag.Bool("no-config", false, 161 196 "run without configuration file (configure via environment variables)") 162 162 - runMigration := flag.String("run-migration", "", 163 163 - "run a store `migration` (one of: create-domain-markers)") 197 197 + printConfigEnvVars := flag.Bool("print-config-env-vars", false, 198 198 + "print every recognized configuration environment variable and exit") 199 199 + printConfig := flag.Bool("print-config", false, 200 200 + "print configuration as JSON and exit") 201 201 + listBlobs := flag.Bool("list-blobs", false, 202 202 + "enumerate every blob with its metadata") 203 203 + listManifests := flag.Bool("list-manifests", false, 204 204 + "enumerate every manifest with its metadata") 164 205 getBlob := flag.String("get-blob", "", 165 206 "write contents of `blob` ('sha256-xxxxxxx...xxx')") 166 207 getManifest := flag.String("get-manifest", "", ··· 169 210 "write archive for `site` (either 'domain.tld' or 'domain.tld/dir') in tar format") 170 211 updateSite := flag.String("update-site", "", 171 212 "update `site` (either 'domain.tld' or 'domain.tld/dir') from archive or repository URL") 213 213 + freezeDomain := flag.String("freeze-domain", "", 214 214 + "prevent any site uploads to a given `domain`") 215 215 + unfreezeDomain := flag.String("unfreeze-domain", "", 216 216 + "allow site uploads to a `domain` again after it has been frozen") 217 217 + auditLog := flag.Bool("audit-log", false, 218 218 + "display audit log") 219 219 + auditRead := flag.String("audit-read", "", 220 220 + "extract contents of audit record `id` to files '<id>-*'") 221 221 + auditRollback := flag.String("audit-rollback", "", 222 222 + "restore site from contents of audit record `id`") 223 223 + auditServer := flag.String("audit-server", "", 224 224 + "listen for notifications on `endpoint` and spawn a process for each audit event") 225 225 + runMigration := flag.String("run-migration", "", 226 226 + "run a store `migration` (one of: create-domain-markers)") 227 227 + traceGarbage := flag.Bool("trace-garbage", false, 228 228 + "estimate total size of unreachable blobs") 172 229 flag.Parse() 173 230 174 231 var cliOperations int 175 175 - if *getBlob != "" { 176 176 - cliOperations += 1 177 177 - } 178 178 - if *getManifest != "" { 179 179 - cliOperations += 1 180 180 - } 181 181 - if *getArchive != "" { 182 182 - cliOperations += 1 232 232 + for _, selected := range []bool{ 233 233 + *listBlobs, 234 234 + *listManifests, 235 235 + *getBlob != "", 236 236 + *getManifest != "", 237 237 + *getArchive != "", 238 238 + *updateSite != "", 239 239 + *freezeDomain != "", 240 240 + *unfreezeDomain != "", 241 241 + *auditLog, 242 242 + *auditRead != "", 243 243 + *auditRollback != "", 244 244 + *auditServer != "", 245 245 + *runMigration != "", 246 246 + *traceGarbage, 247 247 + } { 248 248 + if selected { 249 249 + cliOperations++ 250 250 + } 183 251 } 184 252 if cliOperations > 1 { 185 185 - log.Fatalln("-get-blob, -get-manifest, and -get-archive are mutually exclusive") 253 253 + logc.Fatalln(ctx, "-list-blobs, -list-manifests, -get-blob, -get-manifest, -get-archive, "+ 254 254 + "-update-site, -freeze-domain, -unfreeze-domain, -audit-log, -audit-read, "+ 255 255 + "-audit-rollback, -audit-server, -run-migration, and -trace-garbage are "+ 256 256 + "mutually exclusive") 186 257 } 187 258 188 259 if *configTomlPath != "" && *noConfig { 189 189 - log.Fatalln("-no-config and -config are mutually exclusive") 260 260 + logc.Fatalln(ctx, "-no-config and -config are mutually exclusive") 190 261 } 191 262 192 263 if *printConfigEnvVars { ··· 199 270 *configTomlPath = "config.toml" 200 271 } 201 272 if config, err = Configure(*configTomlPath); err != nil { 202 202 - log.Fatalln("config:", err) 273 273 + logc.Fatalln(ctx, "config:", err) 203 274 } 204 275 205 276 if *printConfig { 206 206 - fmt.Println(config.DebugJSON()) 277 277 + fmt.Println(config.TOML()) 207 278 return 208 279 } 209 280 ··· 211 282 defer FiniObservability() 212 283 213 284 if err = errors.Join( 214 214 - configureFeatures(), 215 215 - configureMemLimit(), 216 216 - configureWildcards(), 285 285 + configureFeatures(ctx), 286 286 + configureMemLimit(ctx), 287 287 + configureWildcards(ctx), 288 288 + configureFallback(ctx), 289 289 + configureAudit(ctx), 217 290 ); err != nil { 218 218 - log.Fatalln(err) 291 291 + logc.Fatalln(ctx, err) 219 292 } 220 293 221 221 - switch { 222 222 - case *runMigration != "": 223 223 - if backend, err = CreateBackend(&config.Storage); err != nil { 224 224 - log.Fatalln(err) 294 294 + // The server has its own logic for creating the backend. 295 295 + if cliOperations > 0 { 296 296 + if backend, err = CreateBackend(ctx, &config.Storage); err != nil { 297 297 + logc.Fatalln(ctx, err) 225 298 } 299 299 + } 226 300 227 227 - if err := RunMigration(context.Background(), *runMigration); err != nil { 228 228 - log.Fatalln(err) 301 301 + switch { 302 302 + case *listBlobs: 303 303 + for metadata, err := range backend.EnumerateBlobs(ctx) { 304 304 + if err != nil { 305 305 + logc.Fatalln(ctx, err) 306 306 + } 307 307 + fmt.Fprintf(color.Output, "%s %s %s\n", 308 308 + metadata.Name, 309 309 + color.HiWhiteString(metadata.LastModified.UTC().Format(time.RFC3339)), 310 310 + color.HiGreenString(fmt.Sprint(metadata.Size)), 311 311 + ) 229 312 } 230 313 231 231 - case *getBlob != "": 232 232 - if backend, err = CreateBackend(&config.Storage); err != nil { 233 233 - log.Fatalln(err) 314 314 + case *listManifests: 315 315 + for metadata, err := range backend.EnumerateManifests(ctx) { 316 316 + if err != nil { 317 317 + logc.Fatalln(ctx, err) 318 318 + } 319 319 + fmt.Fprintf(color.Output, "%s %s %s\n", 320 320 + metadata.Name, 321 321 + color.HiWhiteString(metadata.LastModified.UTC().Format(time.RFC3339)), 322 322 + color.HiGreenString(fmt.Sprint(metadata.Size)), 323 323 + ) 234 324 } 235 325 236 236 - reader, _, _, err := backend.GetBlob(context.Background(), *getBlob) 326 326 + case *getBlob != "": 327 327 + reader, _, err := backend.GetBlob(ctx, *getBlob) 237 328 if err != nil { 238 238 - log.Fatalln(err) 329 329 + logc.Fatalln(ctx, err) 239 330 } 240 331 io.Copy(fileOutputArg(), reader) 241 332 242 333 case *getManifest != "": 243 243 - if backend, err = CreateBackend(&config.Storage); err != nil { 244 244 - log.Fatalln(err) 245 245 - } 246 246 - 247 334 webRoot := webRootArg(*getManifest) 248 248 - manifest, _, err := backend.GetManifest(context.Background(), webRoot, GetManifestOptions{}) 335 335 + manifest, _, err := backend.GetManifest(ctx, webRoot, GetManifestOptions{}) 249 336 if err != nil { 250 250 - log.Fatalln(err) 337 337 + logc.Fatalln(ctx, err) 251 338 } 252 252 - fmt.Fprintln(fileOutputArg(), ManifestDebugJSON(manifest)) 339 339 + fmt.Fprintln(fileOutputArg(), string(ManifestJSON(manifest))) 253 340 254 341 case *getArchive != "": 255 255 - if backend, err = CreateBackend(&config.Storage); err != nil { 256 256 - log.Fatalln(err) 257 257 - } 258 258 - 259 342 webRoot := webRootArg(*getArchive) 260 260 - manifest, manifestMtime, err := 261 261 - backend.GetManifest(context.Background(), webRoot, GetManifestOptions{}) 343 343 + manifest, metadata, err := 344 344 + backend.GetManifest(ctx, webRoot, GetManifestOptions{}) 262 345 if err != nil { 263 263 - log.Fatalln(err) 346 346 + logc.Fatalln(ctx, err) 347 347 + } 348 348 + if err = CollectTar(ctx, fileOutputArg(), manifest, metadata); err != nil { 349 349 + logc.Fatalln(ctx, err) 264 350 } 265 265 - CollectTar(context.Background(), fileOutputArg(), manifest, manifestMtime) 266 351 267 352 case *updateSite != "": 268 268 - if backend, err = CreateBackend(&config.Storage); err != nil { 269 269 - log.Fatalln(err) 270 270 - } 353 353 + ctx = WithPrincipal(ctx) 354 354 + GetPrincipal(ctx).CliAdmin = proto.Bool(true) 271 355 272 356 if flag.NArg() != 1 { 273 273 - log.Fatalln("update source must be provided as the argument") 357 357 + logc.Fatalln(ctx, "update source must be provided as the argument") 274 358 } 275 359 276 360 sourceURL, err := url.Parse(flag.Arg(0)) 277 361 if err != nil { 278 278 - log.Fatalln(err) 362 362 + logc.Fatalln(ctx, err) 279 363 } 280 364 281 365 var result UpdateResult 282 366 if sourceURL.Scheme == "" { 283 367 file, err := os.Open(sourceURL.Path) 284 368 if err != nil { 285 285 - log.Fatalln(err) 369 369 + logc.Fatalln(ctx, err) 286 370 } 287 371 defer file.Close() 288 372 ··· 301 385 } 302 386 303 387 webRoot := webRootArg(*updateSite) 304 304 - result = UpdateFromArchive(context.Background(), webRoot, contentType, file) 388 388 + result = UpdateFromArchive(ctx, webRoot, contentType, file) 305 389 } else { 306 390 branch := "pages" 307 391 if sourceURL.Fragment != "" { ··· 309 393 } 310 394 311 395 webRoot := webRootArg(*updateSite) 312 312 - result = UpdateFromRepository(context.Background(), webRoot, sourceURL.String(), branch) 396 396 + result = UpdateFromRepository(ctx, webRoot, sourceURL.String(), branch) 313 397 } 314 398 315 399 switch result.outcome { 316 400 case UpdateError: 317 317 - log.Printf("error: %s\n", result.err) 401 401 + logc.Printf(ctx, "error: %s\n", result.err) 318 402 os.Exit(2) 319 403 case UpdateTimeout: 320 320 - log.Println("timeout") 404 404 + logc.Println(ctx, "timeout") 321 405 os.Exit(1) 322 406 case UpdateCreated: 323 323 - log.Println("created") 407 407 + logc.Println(ctx, "created") 324 408 case UpdateReplaced: 325 325 - log.Println("replaced") 409 409 + logc.Println(ctx, "replaced") 326 410 case UpdateDeleted: 327 327 - log.Println("deleted") 411 411 + logc.Println(ctx, "deleted") 328 412 case UpdateNoChange: 329 329 - log.Println("no-change") 413 413 + logc.Println(ctx, "no-change") 414 414 + } 415 415 + 416 416 + case *freezeDomain != "" || *unfreezeDomain != "": 417 417 + ctx = WithPrincipal(ctx) 418 418 + GetPrincipal(ctx).CliAdmin = proto.Bool(true) 419 419 + 420 420 + var domain string 421 421 + var freeze bool 422 422 + if *freezeDomain != "" { 423 423 + domain = *freezeDomain 424 424 + freeze = true 425 425 + } else { 426 426 + domain = *unfreezeDomain 427 427 + freeze = false 428 428 + } 429 429 + 430 430 + if freeze { 431 431 + if err = backend.FreezeDomain(ctx, domain); err != nil { 432 432 + logc.Fatalln(ctx, err) 433 433 + } 434 434 + logc.Println(ctx, "frozen") 435 435 + } else { 436 436 + if err = backend.UnfreezeDomain(ctx, domain); err != nil { 437 437 + logc.Fatalln(ctx, err) 438 438 + } 439 439 + logc.Println(ctx, "thawed") 440 440 + } 441 441 + 442 442 + case *auditLog: 443 443 + ch := make(chan *AuditRecord) 444 444 + ids := []AuditID{} 445 445 + for id, err := range backend.SearchAuditLog(ctx, SearchAuditLogOptions{}) { 446 446 + if err != nil { 447 447 + logc.Fatalln(ctx, err) 448 448 + } 449 449 + go func() { 450 450 + if record, err := backend.QueryAuditLog(ctx, id); err != nil { 451 451 + logc.Fatalln(ctx, err) 452 452 + } else { 453 453 + ch <- record 454 454 + } 455 455 + }() 456 456 + ids = append(ids, id) 457 457 + } 458 458 + 459 459 + records := map[AuditID]*AuditRecord{} 460 460 + for len(records) < len(ids) { 461 461 + record := <-ch 462 462 + records[record.GetAuditID()] = record 463 463 + } 464 464 + 465 465 + for _, id := range ids { 466 466 + record := records[id] 467 467 + fmt.Fprintf(color.Output, "%s %s %s %s %s\n", 468 468 + record.GetAuditID().String(), 469 469 + color.HiWhiteString(record.GetTimestamp().AsTime().UTC().Format(time.RFC3339)), 470 470 + color.HiMagentaString(record.DescribePrincipal()), 471 471 + color.HiGreenString(record.DescribeResource()), 472 472 + record.GetEvent(), 473 473 + ) 474 474 + } 475 475 + 476 476 + case *auditRead != "": 477 477 + id, err := ParseAuditID(*auditRead) 478 478 + if err != nil { 479 479 + logc.Fatalln(ctx, err) 480 480 + } 481 481 + 482 482 + record, err := backend.QueryAuditLog(ctx, id) 483 483 + if err != nil { 484 484 + logc.Fatalln(ctx, err) 485 485 + } 486 486 + 487 487 + if err = ExtractAuditRecord(ctx, id, record, "."); err != nil { 488 488 + logc.Fatalln(ctx, err) 489 489 + } 490 490 + 491 491 + case *auditRollback != "": 492 492 + ctx = WithPrincipal(ctx) 493 493 + GetPrincipal(ctx).CliAdmin = proto.Bool(true) 494 494 + 495 495 + id, err := ParseAuditID(*auditRollback) 496 496 + if err != nil { 497 497 + logc.Fatalln(ctx, err) 498 498 + } 499 499 + 500 500 + record, err := backend.QueryAuditLog(ctx, id) 501 501 + if err != nil { 502 502 + logc.Fatalln(ctx, err) 503 503 + } 504 504 + 505 505 + if record.GetManifest() == nil || record.GetDomain() == "" || record.GetProject() == "" { 506 506 + logc.Fatalln(ctx, "no manifest in audit record") 507 507 + } 508 508 + 509 509 + webRoot := path.Join(record.GetDomain(), record.GetProject()) 510 510 + err = backend.StageManifest(ctx, record.GetManifest()) 511 511 + if err != nil { 512 512 + logc.Fatalln(ctx, err) 513 513 + } 514 514 + err = backend.CommitManifest(ctx, webRoot, record.GetManifest(), ModifyManifestOptions{}) 515 515 + if err != nil { 516 516 + logc.Fatalln(ctx, err) 517 517 + } 518 518 + 519 519 + case *auditServer != "": 520 520 + if flag.NArg() < 1 { 521 521 + logc.Fatalln(ctx, "handler path not provided") 522 522 + } 523 523 + 524 524 + processor, err := AuditEventProcessor(flag.Arg(0), flag.Args()[1:]) 525 525 + if err != nil { 526 526 + logc.Fatalln(ctx, err) 527 527 + } 528 528 + 529 529 + serve(ctx, listen(ctx, "audit", *auditServer), ObserveHTTPHandler(processor)) 530 530 + 531 531 + case *runMigration != "": 532 532 + if err = RunMigration(ctx, *runMigration); err != nil { 533 533 + logc.Fatalln(ctx, err) 534 534 + } 535 535 + 536 536 + case *traceGarbage: 537 537 + if err = TraceGarbage(ctx); err != nil { 538 538 + logc.Fatalln(ctx, err) 330 539 } 331 540 332 541 default: ··· 339 548 // The backend is not recreated (this is intentional as it allows preserving the cache). 340 549 OnReload(func() { 341 550 if newConfig, err := Configure(*configTomlPath); err != nil { 342 342 - log.Println("config: reload err:", err) 551 551 + logc.Println(ctx, "config: reload err:", err) 343 552 } else { 344 553 // From https://go.dev/ref/mem: 345 554 // > A read r of a memory location x holding a value that is not larger than ··· 349 558 // > concurrent write. 350 559 config = newConfig 351 560 if err = errors.Join( 352 352 - configureFeatures(), 353 353 - configureMemLimit(), 354 354 - configureWildcards(), 561 561 + configureFeatures(ctx), 562 562 + configureMemLimit(ctx), 563 563 + configureWildcards(ctx), 564 564 + configureFallback(ctx), 355 565 ); err != nil { 356 566 // At this point the configuration is in an in-between, corrupted state, so 357 567 // the only reasonable choice is to crash. 358 358 - log.Fatalln("config: reload fail:", err) 568 568 + logc.Fatalln(ctx, "config: reload fail:", err) 359 569 } else { 360 360 - log.Println("config: reload ok") 570 570 + logc.Println(ctx, "config: reload ok") 361 571 } 362 572 } 363 573 }) ··· 366 576 // spends some time initializing (which the S3 backend does) a proxy like Caddy can race 367 577 // with git-pages on startup and return errors for requests that would have been served 368 578 // just 0.5s later. 369 369 - pagesListener := listen("pages", config.Server.Pages) 370 370 - caddyListener := listen("caddy", config.Server.Caddy) 371 371 - metricsListener := listen("metrics", config.Server.Metrics) 579 579 + pagesListener := listen(ctx, "pages", config.Server.Pages) 580 580 + caddyListener := listen(ctx, "caddy", config.Server.Caddy) 581 581 + metricsListener := listen(ctx, "metrics", config.Server.Metrics) 372 582 373 373 - if backend, err = CreateBackend(&config.Storage); err != nil { 374 374 - log.Fatalln(err) 583 583 + if backend, err = CreateBackend(ctx, &config.Storage); err != nil { 584 584 + logc.Fatalln(ctx, err) 375 585 } 376 586 backend = NewObservedBackend(backend) 377 587 378 378 - go serve(pagesListener, ObserveHTTPHandler(http.HandlerFunc(ServePages))) 379 379 - go serve(caddyListener, ObserveHTTPHandler(http.HandlerFunc(ServeCaddy))) 380 380 - go serve(metricsListener, promhttp.Handler()) 588 588 + middleware := chainHTTPMiddleware( 589 589 + panicHandler, 590 590 + remoteAddrMiddleware, 591 591 + ObserveHTTPHandler, 592 592 + ) 593 593 + go serve(ctx, pagesListener, middleware(http.HandlerFunc(ServePages))) 594 594 + go serve(ctx, caddyListener, middleware(http.HandlerFunc(ServeCaddy))) 595 595 + go serve(ctx, metricsListener, promhttp.Handler()) 381 596 382 597 if config.Insecure { 383 383 - log.Println("serve: ready (INSECURE)") 598 598 + logc.Println(ctx, "serve: ready (INSECURE)") 384 599 } else { 385 385 - log.Println("serve: ready") 600 600 + logc.Println(ctx, "serve: ready") 386 601 } 387 387 - select {} 602 602 + 603 603 + WaitForInterrupt() 604 604 + logc.Println(ctx, "serve: exiting") 388 605 } 389 606 }

+150 -53

src/manifest.go

··· 8 8 "crypto/sha256" 9 9 "errors" 10 10 "fmt" 11 11 - "log" 12 11 "mime" 13 12 "net/http" 14 13 "path" ··· 18 17 "time" 19 18 20 19 "github.com/c2h5oh/datasize" 20 20 + "github.com/go-git/go-git/v6/plumbing" 21 21 + format "github.com/go-git/go-git/v6/plumbing/format/config" 21 22 "github.com/klauspost/compress/zstd" 22 23 "github.com/prometheus/client_golang/prometheus" 23 24 "github.com/prometheus/client_golang/prometheus/promauto" ··· 37 38 }) 38 39 ) 39 40 41 41 + func NewManifest() *Manifest { 42 42 + return &Manifest{ 43 43 + Contents: map[string]*Entry{ 44 44 + "": {Type: Type_Directory.Enum()}, 45 45 + }, 46 46 + } 47 47 + } 48 48 + 40 49 func IsManifestEmpty(manifest *Manifest) bool { 41 50 if len(manifest.Contents) > 1 { 42 51 return false ··· 69 78 return true 70 79 } 71 80 72 72 - func EncodeManifest(manifest *Manifest) []byte { 73 73 - result, err := proto.MarshalOptions{Deterministic: true}.Marshal(manifest) 81 81 + func EncodeManifest(manifest *Manifest) (data []byte) { 82 82 + data, err := proto.MarshalOptions{Deterministic: true}.Marshal(manifest) 74 83 if err != nil { 75 84 panic(err) 76 85 } 77 77 - return result 86 86 + return 87 87 + } 88 88 + 89 89 + func DecodeManifest(data []byte) (manifest *Manifest, err error) { 90 90 + manifest = &Manifest{} 91 91 + err = proto.Unmarshal(data, manifest) 92 92 + return 93 93 + } 94 94 + 95 95 + func NewManifestEntry(type_ Type, data []byte) *Entry { 96 96 + entry := &Entry{} 97 97 + entry.Type = type_.Enum() 98 98 + if data != nil { 99 99 + entry.Data = data 100 100 + entry.Transform = Transform_Identity.Enum() 101 101 + entry.OriginalSize = proto.Int64(int64(len(data))) 102 102 + entry.CompressedSize = proto.Int64(int64(len(data))) 103 103 + } 104 104 + return entry 78 105 } 79 106 80 80 - func DecodeManifest(data []byte) (*Manifest, error) { 81 81 - manifest := Manifest{} 82 82 - err := proto.Unmarshal(data, &manifest) 83 83 - return &manifest, err 107 107 + func AddFile(manifest *Manifest, fileName string, data []byte) *Entry { 108 108 + // Fill in `git_hash` even for files not originating from git using the SHA256 algorithm; 109 109 + // we use this primarily for incremental archive uploads, but when support for git SHA256 110 110 + // repositories is complete, archive uploads and git checkouts will have cross-support for 111 111 + // incremental updates. 112 112 + hasher := plumbing.NewHasher(format.SHA256, plumbing.BlobObject, int64(len(data))) 113 113 + hasher.Write(data) 114 114 + entry := NewManifestEntry(Type_InlineFile, data) 115 115 + entry.GitHash = proto.String(hasher.Sum().String()) 116 116 + manifest.Contents[fileName] = entry 117 117 + return entry 84 118 } 85 119 86 86 - func AddProblem(manifest *Manifest, path, format string, args ...any) error { 120 120 + func AddSymlink(manifest *Manifest, fileName string, target string) *Entry { 121 121 + if path.IsAbs(target) { 122 122 + AddProblem(manifest, fileName, "absolute symlink: %s", target) 123 123 + return nil 124 124 + } else { 125 125 + entry := NewManifestEntry(Type_Symlink, []byte(target)) 126 126 + manifest.Contents[fileName] = entry 127 127 + return entry 128 128 + } 129 129 + } 130 130 + 131 131 + func AddDirectory(manifest *Manifest, dirName string) *Entry { 132 132 + dirName = strings.TrimSuffix(dirName, "/") 133 133 + entry := NewManifestEntry(Type_Directory, nil) 134 134 + manifest.Contents[dirName] = entry 135 135 + return entry 136 136 + } 137 137 + 138 138 + func AddProblem(manifest *Manifest, pathName, format string, args ...any) error { 87 139 cause := fmt.Sprintf(format, args...) 88 140 manifest.Problems = append(manifest.Problems, &Problem{ 89 89 - Path: proto.String(path), 141 141 + Path: proto.String(pathName), 90 142 Cause: proto.String(cause), 91 143 }) 92 92 - return fmt.Errorf("%s: %s", path, cause) 144 144 + return fmt.Errorf("%s: %s", pathName, cause) 145 145 + } 146 146 + 147 147 + // EnsureLeadingDirectories adds directory entries for any parent directories 148 148 + // that are implicitly referenced by files in the manifest but don't have 149 149 + // explicit directory entries. (This can be the case if an archive is created 150 150 + // via globs rather than including a whole directory.) 151 151 + func EnsureLeadingDirectories(manifest *Manifest) { 152 152 + for name := range manifest.Contents { 153 153 + for dir := path.Dir(name); dir != "." && dir != ""; dir = path.Dir(dir) { 154 154 + if _, exists := manifest.Contents[dir]; !exists { 155 155 + AddDirectory(manifest, dir) 156 156 + } 157 157 + } 158 158 + } 93 159 } 94 160 95 161 func GetProblemReport(manifest *Manifest) []string { 96 162 var report []string 97 163 for _, problem := range manifest.Problems { 98 164 report = append(report, 99 99 - fmt.Sprintf("%s: %s", problem.GetPath(), problem.GetCause())) 165 165 + fmt.Sprintf("/%s: %s", problem.GetPath(), problem.GetCause())) 100 166 } 101 167 return report 102 168 } 103 169 104 104 - func ManifestDebugJSON(manifest *Manifest) string { 105 105 - result, err := protojson.MarshalOptions{ 170 170 + func ManifestJSON(manifest *Manifest) []byte { 171 171 + json, err := protojson.MarshalOptions{ 106 172 Multiline: true, 107 173 EmitDefaultValues: true, 108 174 }.Marshal(manifest) 109 175 if err != nil { 110 176 panic(err) 111 177 } 112 112 - return string(result) 178 178 + return json 113 179 } 114 180 115 181 var ErrSymlinkLoop = errors.New("symbolic link loop") ··· 145 211 for path, entry := range manifest.Contents { 146 212 if entry.GetType() == Type_Directory || entry.GetType() == Type_Symlink { 147 213 // no Content-Type 148 148 - } else if entry.GetType() == Type_InlineFile && entry.GetTransform() == Transform_None { 214 214 + } else if entry.GetType() == Type_InlineFile && entry.GetTransform() == Transform_Identity { 149 215 contentType := mime.TypeByExtension(filepath.Ext(path)) 150 216 if contentType == "" { 151 151 - contentType = http.DetectContentType(entry.Data[:512]) 217 217 + contentType = http.DetectContentType(entry.Data[:min(512, len(entry.Data))]) 152 218 } 153 219 entry.ContentType = proto.String(contentType) 154 154 - } else { 220 220 + } else if entry.GetContentType() == "" { 155 221 panic(fmt.Errorf("DetectContentType encountered invalid entry: %v, %v", 156 222 entry.GetType(), entry.GetTransform())) 157 223 } 158 224 } 159 225 } 160 226 161 161 - // The `clauspost/compress/zstd` package recommends reusing a compressor to avoid repeated 227 227 + // The `klauspost/compress/zstd` package recommends reusing a compressor to avoid repeated 162 228 // allocations of internal buffers. 163 229 var zstdEncoder, _ = zstd.NewWriter(nil, zstd.WithEncoderLevel(zstd.SpeedBetterCompression)) 164 230 ··· 167 233 span, _ := ObserveFunction(ctx, "CompressFiles") 168 234 defer span.Finish() 169 235 170 170 - var originalSize, compressedSize int64 236 236 + var originalSize int64 237 237 + var compressedSize int64 171 238 for _, entry := range manifest.Contents { 172 172 - if entry.GetType() == Type_InlineFile && entry.GetTransform() == Transform_None { 173 173 - mtype := getMediaType(entry.GetContentType()) 174 174 - if strings.HasPrefix(mtype, "video/") || strings.HasPrefix(mtype, "audio/") { 239 239 + if entry.GetType() == Type_InlineFile && entry.GetTransform() == Transform_Identity { 240 240 + mediaType := getMediaType(entry.GetContentType()) 241 241 + if strings.HasPrefix(mediaType, "video/") || strings.HasPrefix(mediaType, "audio/") { 175 242 continue 176 243 } 177 177 - originalSize += entry.GetSize() 178 178 - compressedData := zstdEncoder.EncodeAll(entry.GetData(), make([]byte, 0, entry.GetSize())) 179 179 - if len(compressedData) < int(*entry.Size) { 244 244 + compressedData := zstdEncoder.EncodeAll(entry.GetData(), 245 245 + make([]byte, 0, entry.GetOriginalSize())) 246 246 + if int64(len(compressedData)) < entry.GetOriginalSize() { 180 247 entry.Data = compressedData 181 181 - entry.Size = proto.Int64(int64(len(entry.Data))) 182 182 - entry.Transform = Transform_Zstandard.Enum() 248 248 + entry.Transform = Transform_Zstd.Enum() 249 249 + entry.CompressedSize = proto.Int64(int64(len(entry.Data))) 183 250 } 184 184 - compressedSize += entry.GetSize() 185 251 } 252 252 + originalSize += entry.GetOriginalSize() 253 253 + compressedSize += entry.GetCompressedSize() 186 254 } 187 255 manifest.OriginalSize = proto.Int64(originalSize) 188 256 manifest.CompressedSize = proto.Int64(compressedSize) 189 257 190 258 if originalSize != 0 { 191 259 spaceSaving := (float64(originalSize) - float64(compressedSize)) / float64(originalSize) 192 192 - log.Printf("compress: saved %.2f percent (%s to %s)", 260 260 + logc.Printf(ctx, "compress: saved %.2f percent (%s to %s)", 193 261 spaceSaving*100.0, 194 262 datasize.ByteSize(originalSize).HR(), 195 263 datasize.ByteSize(compressedSize).HR(), ··· 203 271 // At the moment, there isn't a good way to report errors except to log them on the terminal. 204 272 // (Perhaps in the future they could be exposed at `.git-pages/status.txt`?) 205 273 func PrepareManifest(ctx context.Context, manifest *Manifest) error { 206 206 - // Parse Netlify-style `_redirects` 274 274 + // Parse Netlify-style `_redirects`. 207 275 if err := ProcessRedirectsFile(manifest); err != nil { 208 208 - log.Printf("redirects err: %s\n", err) 276 276 + logc.Printf(ctx, "redirects err: %s\n", err) 209 277 } else if len(manifest.Redirects) > 0 { 210 210 - log.Printf("redirects ok: %d rules\n", len(manifest.Redirects)) 278 278 + logc.Printf(ctx, "redirects ok: %d rules\n", len(manifest.Redirects)) 211 279 } 212 280 213 213 - // Parse Netlify-style `_headers` 281 281 + // Check if any redirects are unreachable. 282 282 + LintRedirects(manifest) 283 283 + 284 284 + // Parse Netlify-style `_headers`. 214 285 if err := ProcessHeadersFile(manifest); err != nil { 215 215 - log.Printf("headers err: %s\n", err) 286 286 + logc.Printf(ctx, "headers err: %s\n", err) 216 287 } else if len(manifest.Headers) > 0 { 217 217 - log.Printf("headers ok: %d rules\n", len(manifest.Headers)) 288 288 + logc.Printf(ctx, "headers ok: %d rules\n", len(manifest.Headers)) 218 289 } 219 290 220 220 - // Sniff content type like `http.ServeContent` 291 291 + // Sniff content type like `http.ServeContent`. 221 292 DetectContentType(manifest) 222 293 223 223 - // Opportunistically compress blobs (must be done last) 294 294 + // Opportunistically compress blobs (must be done last). 224 295 CompressFiles(ctx, manifest) 225 296 226 297 return nil 227 298 } 228 299 300 300 + var ErrSiteTooLarge = errors.New("site too large") 229 301 var ErrManifestTooLarge = errors.New("manifest too large") 230 302 231 303 // Uploads inline file data over certain size to the storage backend. Returns a copy of 232 304 // the manifest updated to refer to an external content-addressable store. 233 233 - func StoreManifest(ctx context.Context, name string, manifest *Manifest) (*Manifest, error) { 305 305 + func StoreManifest( 306 306 + ctx context.Context, name string, manifest *Manifest, opts ModifyManifestOptions, 307 307 + ) (*Manifest, error) { 234 308 span, ctx := ObserveFunction(ctx, "StoreManifest", "manifest.name", name) 235 309 defer span.Finish() 236 310 ··· 247 321 CompressedSize: manifest.CompressedSize, 248 322 StoredSize: proto.Int64(0), 249 323 } 250 250 - extObjectSizes := make(map[string]int64) 251 324 for name, entry := range manifest.Contents { 252 325 cannotBeInlined := entry.GetType() == Type_InlineFile && 253 253 - entry.GetSize() > int64(config.Limits.MaxInlineFileSize.Bytes()) 326 326 + entry.GetCompressedSize() > int64(config.Limits.MaxInlineFileSize.Bytes()) 254 327 if cannotBeInlined { 255 328 dataHash := sha256.Sum256(entry.Data) 256 329 extManifest.Contents[name] = &Entry{ 257 257 - Type: Type_ExternalFile.Enum(), 258 258 - Size: entry.Size, 259 259 - Data: fmt.Appendf(nil, "sha256-%x", dataHash), 260 260 - Transform: entry.Transform, 261 261 - ContentType: entry.ContentType, 330 330 + Type: Type_ExternalFile.Enum(), 331 331 + OriginalSize: entry.OriginalSize, 332 332 + CompressedSize: entry.CompressedSize, 333 333 + Data: fmt.Appendf(nil, "sha256-%x", dataHash), 334 334 + Transform: entry.Transform, 335 335 + ContentType: entry.ContentType, 336 336 + GitHash: entry.GitHash, 262 337 } 263 263 - extObjectSizes[string(dataHash[:])] = entry.GetSize() 264 338 } else { 265 339 extManifest.Contents[name] = entry 266 340 } 267 341 } 268 268 - // `extObjectMap` stores size once per object, deduplicating it 269 269 - for _, storedSize := range extObjectSizes { 270 270 - *extManifest.StoredSize += storedSize 342 342 + 343 343 + // Compute the total and deduplicated storage size. 344 344 + totalSize := int64(0) 345 345 + blobSizes := map[string]int64{} 346 346 + for _, entry := range extManifest.Contents { 347 347 + totalSize += entry.GetOriginalSize() 348 348 + if entry.GetType() == Type_ExternalFile { 349 349 + blobSizes[string(entry.Data)] = entry.GetCompressedSize() 350 350 + } 351 351 + } 352 352 + if uint64(totalSize) > config.Limits.MaxSiteSize.Bytes() { 353 353 + return nil, fmt.Errorf("%w: contents size %s exceeds %s limit", 354 354 + ErrSiteTooLarge, 355 355 + datasize.ByteSize(totalSize).HR(), 356 356 + config.Limits.MaxSiteSize.HR(), 357 357 + ) 358 358 + } 359 359 + for _, blobSize := range blobSizes { 360 360 + *extManifest.StoredSize += blobSize 271 361 } 272 362 273 363 // Upload the resulting manifest and the blob it references. ··· 287 377 wg := sync.WaitGroup{} 288 378 ch := make(chan error, len(extManifest.Contents)) 289 379 for name, entry := range extManifest.Contents { 290 290 - if entry.GetType() == Type_ExternalFile { 380 380 + // Upload external entries (those that were decided as ineligible for being stored inline). 381 381 + // If the entry in the original manifest is already an external reference, there's no need 382 382 + // to externalize it (and no way for us to do so, since the entry only contains the blob name). 383 383 + if entry.GetType() == Type_ExternalFile && manifest.Contents[name].GetType() == Type_InlineFile { 291 384 wg.Go(func() { 292 385 err := backend.PutBlob(ctx, string(entry.Data), manifest.Contents[name].Data) 293 386 if err != nil { ··· 302 395 return nil, err // currently ignores all but 1st error 303 396 } 304 397 305 305 - if err := backend.CommitManifest(ctx, name, &extManifest); err != nil { 306 306 - return nil, fmt.Errorf("commit manifest: %w", err) 398 398 + if err := backend.CommitManifest(ctx, name, &extManifest, opts); err != nil { 399 399 + if errors.Is(err, ErrDomainFrozen) { 400 400 + return nil, err 401 401 + } else { 402 402 + return nil, fmt.Errorf("commit manifest: %w", err) 403 403 + } 307 404 } 308 405 309 406 return &extManifest, nil

+10 -8

src/migrate.go

··· 3 3 import ( 4 4 "context" 5 5 "fmt" 6 6 - "log" 7 6 "slices" 8 7 "strings" 9 8 ) ··· 19 18 20 19 func createDomainMarkers(ctx context.Context) error { 21 20 if backend.HasFeature(ctx, FeatureCheckDomainMarker) { 22 22 - log.Print("store already has domain markers") 21 21 + logc.Print(ctx, "store already has domain markers") 23 22 return nil 24 23 } 25 24 26 26 - var manifests, domains []string 27 27 - manifests, err := backend.ListManifests(ctx) 28 28 - if err != nil { 29 29 - return fmt.Errorf("list manifests: %w", err) 25 25 + var manifests []string 26 26 + for metadata, err := range backend.EnumerateManifests(ctx) { 27 27 + if err != nil { 28 28 + return fmt.Errorf("enum manifests: %w", err) 29 29 + } 30 30 + manifests = append(manifests, metadata.Name) 30 31 } 31 32 slices.Sort(manifests) 33 33 + var domains []string 32 34 for _, manifest := range manifests { 33 35 domain, _, _ := strings.Cut(manifest, "/") 34 36 if len(domains) == 0 || domains[len(domains)-1] != domain { ··· 36 38 } 37 39 } 38 40 for idx, domain := range domains { 39 39 - log.Printf("(%d / %d) creating domain %s", idx+1, len(domains), domain) 41 41 + logc.Printf(ctx, "(%d / %d) creating domain %s", idx+1, len(domains), domain) 40 42 if err := backend.CreateDomain(ctx, domain); err != nil { 41 43 return fmt.Errorf("creating domain %s: %w", domain, err) 42 44 } ··· 44 46 if err := backend.EnableFeature(ctx, FeatureCheckDomainMarker); err != nil { 45 47 return err 46 48 } 47 47 - log.Printf("created markers for %d domains", len(domains)) 49 49 + logc.Printf(ctx, "created markers for %d domains", len(domains)) 48 50 return nil 49 51 }

+176 -46

src/observe.go

··· 5 5 "errors" 6 6 "fmt" 7 7 "io" 8 8 + "iter" 8 9 "log" 9 10 "log/slog" 10 11 "math/rand/v2" ··· 12 13 "os" 13 14 "runtime/debug" 14 15 "strconv" 16 16 + "sync" 15 17 "time" 16 18 17 19 slogmulti "github.com/samber/slog-multi" 20 20 + 21 21 + syslog "codeberg.org/git-pages/go-slog-syslog" 18 22 19 23 "github.com/prometheus/client_golang/prometheus" 20 24 "github.com/prometheus/client_golang/prometheus/promauto" ··· 41 45 }, []string{"method"}) 42 46 ) 43 47 48 48 + var syslogHandler syslog.Handler 49 49 + 44 50 func hasSentry() bool { 45 51 return os.Getenv("SENTRY_DSN") != "" 46 52 } 47 53 54 54 + func chainSentryMiddleware( 55 55 + middleware ...func(event *sentry.Event, hint *sentry.EventHint) *sentry.Event, 56 56 + ) func(event *sentry.Event, hint *sentry.EventHint) *sentry.Event { 57 57 + return func(event *sentry.Event, hint *sentry.EventHint) *sentry.Event { 58 58 + for idx := 0; idx < len(middleware) && event != nil; idx++ { 59 59 + event = middleware[idx](event, hint) 60 60 + } 61 61 + return event 62 62 + } 63 63 + } 64 64 + 65 65 + // sensitiveHTTPHeaders extends the list of sensitive headers defined in the Sentry Go SDK with our 66 66 + // own application-specific header field names. 67 67 + var sensitiveHTTPHeaders = map[string]struct{}{ 68 68 + "Forge-Authorization": {}, 69 69 + } 70 70 + 71 71 + // scrubSentryEvent removes sensitive HTTP header fields from the Sentry event. 72 72 + func scrubSentryEvent(event *sentry.Event, hint *sentry.EventHint) *sentry.Event { 73 73 + if event.Request != nil && event.Request.Headers != nil { 74 74 + for key := range event.Request.Headers { 75 75 + if _, ok := sensitiveHTTPHeaders[key]; ok { 76 76 + delete(event.Request.Headers, key) 77 77 + } 78 78 + } 79 79 + } 80 80 + return event 81 81 + } 82 82 + 83 83 + // sampleSentryEvent returns a function that discards a Sentry event according to the sample rate, 84 84 + // unless the associated HTTP request triggers a mutation or it took too long to produce a response, 85 85 + // in which case the event is never discarded. 86 86 + func sampleSentryEvent(sampleRate float64) func(*sentry.Event, *sentry.EventHint) *sentry.Event { 87 87 + return func(event *sentry.Event, hint *sentry.EventHint) *sentry.Event { 88 88 + newSampleRate := sampleRate 89 89 + if event.Request != nil { 90 90 + switch event.Request.Method { 91 91 + case "PUT", "POST", "DELETE": 92 92 + newSampleRate = 1 93 93 + } 94 94 + } 95 95 + duration := event.Timestamp.Sub(event.StartTime) 96 96 + threshold := time.Duration(config.Observability.SlowResponseThreshold) 97 97 + if duration >= threshold { 98 98 + newSampleRate = 1 99 99 + } 100 100 + if rand.Float64() < newSampleRate { 101 101 + return event 102 102 + } 103 103 + return nil 104 104 + } 105 105 + } 106 106 + 48 107 func InitObservability() { 49 108 debug.SetPanicOnFault(true) 50 109 ··· 68 127 log.Println("unknown log format", config.LogFormat) 69 128 } 70 129 130 130 + if syslogAddr := os.Getenv("SYSLOG_ADDR"); syslogAddr != "" { 131 131 + var err error 132 132 + syslogHandler, err = syslog.NewHandler(&syslog.HandlerOptions{ 133 133 + Address: syslogAddr, 134 134 + AppName: "git-pages", 135 135 + StructuredDataID: "git-pages", 136 136 + }) 137 137 + if err != nil { 138 138 + log.Fatalf("syslog: %v", err) 139 139 + } 140 140 + logHandlers = append(logHandlers, syslogHandler) 141 141 + } 142 142 + 71 143 if hasSentry() { 72 144 enableLogs := false 73 145 if value, err := strconv.ParseBool(os.Getenv("SENTRY_LOGS")); err == nil { ··· 79 151 enableTracing = value 80 152 } 81 153 154 154 + tracesSampleRate := 1.00 155 155 + switch environment { 156 156 + case "development", "staging": 157 157 + default: 158 158 + tracesSampleRate = 0.05 159 159 + } 160 160 + 82 161 options := sentry.ClientOptions{} 162 162 + options.DisableTelemetryBuffer = !config.Feature("sentry-telemetry-buffer") 83 163 options.Environment = environment 84 164 options.EnableLogs = enableLogs 85 165 options.EnableTracing = enableTracing 86 86 - options.TracesSampleRate = 1 87 87 - switch environment { 88 88 - case "development", "staging": 89 89 - default: 90 90 - options.BeforeSendTransaction = func(event *sentry.Event, hint *sentry.EventHint) *sentry.Event { 91 91 - sampleRate := 0.05 92 92 - if trace, ok := event.Contexts["trace"]; ok { 93 93 - if data, ok := trace["data"].(map[string]any); ok { 94 94 - if method, ok := data["http.request.method"].(string); ok { 95 95 - switch method { 96 96 - case "PUT", "DELETE", "POST": 97 97 - sampleRate = 1 98 98 - default: 99 99 - duration := event.Timestamp.Sub(event.StartTime) 100 100 - threshold := time.Duration(config.Observability.SlowResponseThreshold) 101 101 - if duration >= threshold { 102 102 - sampleRate = 1 103 103 - } 104 104 - } 105 105 - } 106 106 - } 107 107 - } 108 108 - if rand.Float64() < sampleRate { 109 109 - return event 110 110 - } 111 111 - return nil 112 112 - } 113 113 - } 166 166 + options.TracesSampleRate = 1 // use our own custom sampling logic 167 167 + options.BeforeSend = scrubSentryEvent 168 168 + options.BeforeSendTransaction = chainSentryMiddleware( 169 169 + sampleSentryEvent(tracesSampleRate), 170 170 + scrubSentryEvent, 171 171 + ) 114 172 if err := sentry.Init(options); err != nil { 115 173 log.Fatalf("sentry: %s\n", err) 116 174 } ··· 126 184 } 127 185 128 186 func FiniObservability() { 187 187 + var wg sync.WaitGroup 188 188 + timeout := 2 * time.Second 189 189 + if syslogHandler != nil { 190 190 + wg.Go(func() { syslogHandler.Flush(timeout) }) 191 191 + } 129 192 if hasSentry() { 130 130 - sentry.Flush(2 * time.Second) 193 193 + wg.Go(func() { sentry.Flush(timeout) }) 131 194 } 195 195 + wg.Wait() 132 196 } 133 197 134 198 func ObserveError(err error) { ··· 290 354 func (backend *observedBackend) GetBlob( 291 355 ctx context.Context, name string, 292 356 ) ( 293 293 - reader io.ReadSeeker, size uint64, mtime time.Time, err error, 357 357 + reader io.ReadSeeker, metadata BlobMetadata, err error, 294 358 ) { 295 359 span, ctx := ObserveFunction(ctx, "GetBlob", "blob.name", name) 296 296 - if reader, size, mtime, err = backend.inner.GetBlob(ctx, name); err == nil { 297 297 - ObserveData(ctx, "blob.size", size) 360 360 + if reader, metadata, err = backend.inner.GetBlob(ctx, name); err == nil { 361 361 + ObserveData(ctx, "blob.size", metadata.Size) 298 362 blobsRetrievedCount.Inc() 299 299 - blobsRetrievedBytes.Add(float64(size)) 363 363 + blobsRetrievedBytes.Add(float64(metadata.Size)) 300 364 } 301 365 span.Finish() 302 366 return ··· 319 383 return 320 384 } 321 385 322 322 - func (backend *observedBackend) ListManifests(ctx context.Context) (manifests []string, err error) { 323 323 - span, ctx := ObserveFunction(ctx, "ListManifests") 324 324 - manifests, err = backend.inner.ListManifests(ctx) 325 325 - span.Finish() 326 326 - return 386 386 + func (backend *observedBackend) EnumerateBlobs(ctx context.Context) iter.Seq2[BlobMetadata, error] { 387 387 + return func(yield func(BlobMetadata, error) bool) { 388 388 + span, ctx := ObserveFunction(ctx, "EnumerateBlobs") 389 389 + for metadata, err := range backend.inner.EnumerateBlobs(ctx) { 390 390 + if !yield(metadata, err) { 391 391 + break 392 392 + } 393 393 + } 394 394 + span.Finish() 395 395 + } 327 396 } 328 397 329 398 func (backend *observedBackend) GetManifest( 330 399 ctx context.Context, name string, opts GetManifestOptions, 331 400 ) ( 332 332 - manifest *Manifest, mtime time.Time, err error, 401 401 + manifest *Manifest, metadata ManifestMetadata, err error, 333 402 ) { 334 403 span, ctx := ObserveFunction(ctx, "GetManifest", 335 404 "manifest.name", name, 336 405 "manifest.bypass_cache", opts.BypassCache, 337 406 ) 338 338 - if manifest, mtime, err = backend.inner.GetManifest(ctx, name, opts); err == nil { 407 407 + if manifest, metadata, err = backend.inner.GetManifest(ctx, name, opts); err == nil { 339 408 manifestsRetrievedCount.Inc() 340 409 } 341 410 span.Finish() ··· 349 418 return 350 419 } 351 420 352 352 - func (backend *observedBackend) CommitManifest(ctx context.Context, name string, manifest *Manifest) (err error) { 421 421 + func (backend *observedBackend) HasAtomicCAS(ctx context.Context) bool { 422 422 + return backend.inner.HasAtomicCAS(ctx) 423 423 + } 424 424 + 425 425 + func (backend *observedBackend) CommitManifest(ctx context.Context, name string, manifest *Manifest, opts ModifyManifestOptions) (err error) { 353 426 span, ctx := ObserveFunction(ctx, "CommitManifest", "manifest.name", name) 354 354 - err = backend.inner.CommitManifest(ctx, name, manifest) 427 427 + err = backend.inner.CommitManifest(ctx, name, manifest, opts) 355 428 span.Finish() 356 429 return 357 430 } 358 431 359 359 - func (backend *observedBackend) DeleteManifest(ctx context.Context, name string) (err error) { 432 432 + func (backend *observedBackend) DeleteManifest(ctx context.Context, name string, opts ModifyManifestOptions) (err error) { 360 433 span, ctx := ObserveFunction(ctx, "DeleteManifest", "manifest.name", name) 361 361 - err = backend.inner.DeleteManifest(ctx, name) 434 434 + err = backend.inner.DeleteManifest(ctx, name, opts) 362 435 span.Finish() 363 436 return 364 437 } 365 438 439 439 + func (backend *observedBackend) EnumerateManifests(ctx context.Context) iter.Seq2[ManifestMetadata, error] { 440 440 + return func(yield func(ManifestMetadata, error) bool) { 441 441 + span, ctx := ObserveFunction(ctx, "EnumerateManifests") 442 442 + for metadata, err := range backend.inner.EnumerateManifests(ctx) { 443 443 + if !yield(metadata, err) { 444 444 + break 445 445 + } 446 446 + } 447 447 + span.Finish() 448 448 + } 449 449 + } 450 450 + 366 451 func (backend *observedBackend) CheckDomain(ctx context.Context, domain string) (found bool, err error) { 367 367 - span, ctx := ObserveFunction(ctx, "CheckDomain", "manifest.domain", domain) 452 452 + span, ctx := ObserveFunction(ctx, "CheckDomain", "domain.name", domain) 368 453 found, err = backend.inner.CheckDomain(ctx, domain) 369 454 span.Finish() 370 455 return 371 456 } 372 457 373 458 func (backend *observedBackend) CreateDomain(ctx context.Context, domain string) (err error) { 374 374 - span, ctx := ObserveFunction(ctx, "CreateDomain", "manifest.domain", domain) 459 459 + span, ctx := ObserveFunction(ctx, "CreateDomain", "domain.name", domain) 375 460 err = backend.inner.CreateDomain(ctx, domain) 376 461 span.Finish() 377 462 return 378 463 } 464 464 + 465 465 + func (backend *observedBackend) FreezeDomain(ctx context.Context, domain string) (err error) { 466 466 + span, ctx := ObserveFunction(ctx, "FreezeDomain", "domain.name", domain) 467 467 + err = backend.inner.FreezeDomain(ctx, domain) 468 468 + span.Finish() 469 469 + return 470 470 + } 471 471 + 472 472 + func (backend *observedBackend) UnfreezeDomain(ctx context.Context, domain string) (err error) { 473 473 + span, ctx := ObserveFunction(ctx, "UnfreezeDomain", "domain.name", domain) 474 474 + err = backend.inner.UnfreezeDomain(ctx, domain) 475 475 + span.Finish() 476 476 + return 477 477 + } 478 478 + 479 479 + func (backend *observedBackend) AppendAuditLog(ctx context.Context, id AuditID, record *AuditRecord) (err error) { 480 480 + span, ctx := ObserveFunction(ctx, "AppendAuditLog", "audit.id", id) 481 481 + err = backend.inner.AppendAuditLog(ctx, id, record) 482 482 + span.Finish() 483 483 + return 484 484 + } 485 485 + 486 486 + func (backend *observedBackend) QueryAuditLog(ctx context.Context, id AuditID) (record *AuditRecord, err error) { 487 487 + span, ctx := ObserveFunction(ctx, "QueryAuditLog", "audit.id", id) 488 488 + record, err = backend.inner.QueryAuditLog(ctx, id) 489 489 + span.Finish() 490 490 + return 491 491 + } 492 492 + 493 493 + func (backend *observedBackend) SearchAuditLog( 494 494 + ctx context.Context, opts SearchAuditLogOptions, 495 495 + ) iter.Seq2[AuditID, error] { 496 496 + return func(yield func(AuditID, error) bool) { 497 497 + span, ctx := ObserveFunction(ctx, "SearchAuditLog", 498 498 + "audit.search.since", opts.Since, 499 499 + "audit.search.until", opts.Until, 500 500 + ) 501 501 + for id, err := range backend.inner.SearchAuditLog(ctx, opts) { 502 502 + if !yield(id, err) { 503 503 + break 504 504 + } 505 505 + } 506 506 + span.Finish() 507 507 + } 508 508 + }

+281 -99

src/pages.go

··· 8 8 "errors" 9 9 "fmt" 10 10 "io" 11 11 - "log" 12 11 "maps" 13 12 "net/http" 14 13 "net/url" 15 14 "os" 16 15 "path" 16 16 + "slices" 17 17 "strconv" 18 18 "strings" 19 19 "time" ··· 22 22 "github.com/pquerna/cachecontrol/cacheobject" 23 23 "github.com/prometheus/client_golang/prometheus" 24 24 "github.com/prometheus/client_golang/prometheus/promauto" 25 25 + "google.golang.org/protobuf/proto" 25 26 ) 26 27 27 28 const notFoundPage = "404.html" 28 29 29 30 var ( 31 31 + serveEncodingCount = promauto.NewCounterVec(prometheus.CounterOpts{ 32 32 + Name: "git_pages_serve_encoding_count", 33 33 + Help: "Count of blob transform vs negotiated encoding", 34 34 + }, []string{"transform", "negotiated"}) 35 35 + 30 36 siteUpdatesCount = promauto.NewCounterVec(prometheus.CounterOpts{ 31 37 Name: "git_pages_site_updates", 32 38 Help: "Count of site updates in total", ··· 41 47 }, []string{"cause"}) 42 48 ) 43 49 44 44 - func reportSiteUpdate(via string, result *UpdateResult) { 50 50 + func observeSiteUpdate(via string, result *UpdateResult) { 45 51 siteUpdatesCount.With(prometheus.Labels{"via": via}).Inc() 46 46 - 47 52 switch result.outcome { 48 53 case UpdateError: 49 54 siteUpdateErrorCount.With(prometheus.Labels{"cause": "other"}).Inc() ··· 61 66 } 62 67 63 68 func makeWebRoot(host string, projectName string) string { 64 64 - return fmt.Sprintf("%s/%s", strings.ToLower(host), projectName) 69 69 + return path.Join(strings.ToLower(host), projectName) 70 70 + } 71 71 + 72 72 + func getWebRoot(r *http.Request) (string, error) { 73 73 + host, err := GetHost(r) 74 74 + if err != nil { 75 75 + return "", err 76 76 + } 77 77 + 78 78 + projectName, err := GetProjectName(r) 79 79 + if err != nil { 80 80 + return "", err 81 81 + } 82 82 + 83 83 + return makeWebRoot(host, projectName), nil 65 84 } 66 85 67 86 func writeRedirect(w http.ResponseWriter, code int, path string) { ··· 78 97 var err error 79 98 var sitePath string 80 99 var manifest *Manifest 81 81 - var manifestMtime time.Time 100 100 + var metadata ManifestMetadata 82 101 83 102 cacheControl, err := cacheobject.ParseRequestCacheControl(r.Header.Get("Cache-Control")) 84 103 if err != nil { ··· 97 116 } 98 117 99 118 type indexManifestResult struct { 100 100 - manifest *Manifest 101 101 - manifestMtime time.Time 102 102 - err error 119 119 + manifest *Manifest 120 120 + metadata ManifestMetadata 121 121 + err error 103 122 } 104 123 indexManifestCh := make(chan indexManifestResult, 1) 105 124 go func() { 106 106 - manifest, mtime, err := backend.GetManifest( 125 125 + manifest, metadata, err := backend.GetManifest( 107 126 r.Context(), makeWebRoot(host, ".index"), 108 127 GetManifestOptions{BypassCache: bypassCache}, 109 128 ) 110 110 - indexManifestCh <- (indexManifestResult{manifest, mtime, err}) 129 129 + indexManifestCh <- (indexManifestResult{manifest, metadata, err}) 111 130 }() 112 131 113 132 err = nil 114 133 sitePath = strings.TrimPrefix(r.URL.Path, "/") 115 134 if projectName, projectPath, hasProjectSlash := strings.Cut(sitePath, "/"); projectName != "" { 116 116 - var projectManifest *Manifest 117 117 - var projectManifestMtime time.Time 118 118 - projectManifest, projectManifestMtime, err = backend.GetManifest( 119 119 - r.Context(), makeWebRoot(host, projectName), 120 120 - GetManifestOptions{BypassCache: bypassCache}, 121 121 - ) 122 122 - if err == nil { 123 123 - if !hasProjectSlash { 124 124 - writeRedirect(w, http.StatusFound, r.URL.Path+"/") 125 125 - return nil 135 135 + if IsValidProjectName(projectName) { 136 136 + var projectManifest *Manifest 137 137 + var projectMetadata ManifestMetadata 138 138 + projectManifest, projectMetadata, err = backend.GetManifest( 139 139 + r.Context(), makeWebRoot(host, projectName), 140 140 + GetManifestOptions{BypassCache: bypassCache}, 141 141 + ) 142 142 + if err == nil { 143 143 + if !hasProjectSlash { 144 144 + writeRedirect(w, http.StatusFound, r.URL.Path+"/") 145 145 + return nil 146 146 + } 147 147 + sitePath, manifest, metadata = projectPath, projectManifest, projectMetadata 126 148 } 127 127 - sitePath, manifest, manifestMtime = projectPath, projectManifest, projectManifestMtime 128 149 } 129 150 } 130 151 if manifest == nil && (err == nil || errors.Is(err, ErrObjectNotFound)) { 131 152 result := <-indexManifestCh 132 132 - manifest, manifestMtime, err = result.manifest, result.manifestMtime, result.err 153 153 + manifest, metadata, err = result.manifest, result.metadata, result.err 133 154 if manifest == nil && errors.Is(err, ErrObjectNotFound) { 134 134 - if found, fallbackErr := HandleWildcardFallback(w, r); found { 135 135 - return fallbackErr 155 155 + if fallback != nil { 156 156 + logc.Printf(r.Context(), "fallback: %s via %s", host, config.Fallback.ProxyTo) 157 157 + fallback.ServeHTTP(w, r) 158 158 + return nil 136 159 } else { 137 160 w.WriteHeader(http.StatusNotFound) 138 161 fmt.Fprintf(w, "site not found\n") ··· 159 182 return nil 160 183 } 161 184 if metadataPath, found := strings.CutPrefix(sitePath, ".git-pages/"); found { 162 162 - lastModified := manifestMtime.UTC().Format(http.TimeFormat) 185 185 + lastModified := metadata.LastModified.UTC().Format(http.TimeFormat) 163 186 switch { 164 187 case metadataPath == "health": 165 188 w.Header().Add("Last-Modified", lastModified) 189 189 + w.Header().Add("ETag", fmt.Sprintf("\"%s\"", metadata.ETag)) 166 190 w.WriteHeader(http.StatusOK) 167 191 fmt.Fprintf(w, "ok\n") 168 192 return nil ··· 177 201 178 202 w.Header().Add("Content-Type", "application/json; charset=utf-8") 179 203 w.Header().Add("Last-Modified", lastModified) 204 204 + w.Header().Add("ETag", fmt.Sprintf("\"%s-manifest\"", metadata.ETag)) 180 205 w.WriteHeader(http.StatusOK) 181 181 - w.Write([]byte(ManifestDebugJSON(manifest))) 206 206 + w.Write(ManifestJSON(manifest)) 182 207 return nil 183 208 184 184 - case metadataPath == "archive.tar" && config.Feature("archive-site"): 209 209 + case metadataPath == "archive.tar": 185 210 // same as above 186 211 _, err := AuthorizeMetadataRetrieval(r) 187 212 if err != nil { ··· 190 215 191 216 // we only offer `/.git-pages/archive.tar` and not the `.tar.gz`/`.tar.zst` variants 192 217 // because HTTP can already request compression using the `Content-Encoding` mechanism 193 193 - acceptedEncodings := parseHTTPEncodings(r.Header.Get("Accept-Encoding")) 218 218 + acceptedEncodings := ParseAcceptEncodingHeader(r.Header.Get("Accept-Encoding")) 219 219 + w.Header().Add("Vary", "Accept-Encoding") 194 220 negotiated := acceptedEncodings.Negotiate("zstd", "gzip", "identity") 195 221 if negotiated != "" { 196 222 w.Header().Set("Content-Encoding", negotiated) 197 223 } 198 224 w.Header().Add("Content-Type", "application/x-tar") 199 225 w.Header().Add("Last-Modified", lastModified) 226 226 + w.Header().Add("ETag", fmt.Sprintf("\"%s-archive\"", metadata.ETag)) 200 227 w.Header().Add("Transfer-Encoding", "chunked") 201 228 w.WriteHeader(http.StatusOK) 202 229 var iow io.Writer ··· 208 235 case "zstd": 209 236 iow, _ = zstd.NewWriter(w) 210 237 } 211 211 - return CollectTar(r.Context(), iow, manifest, manifestMtime) 238 238 + return CollectTar(r.Context(), iow, manifest, metadata) 212 239 213 240 default: 214 241 w.WriteHeader(http.StatusNotFound) ··· 220 247 entryPath := sitePath 221 248 entry := (*Entry)(nil) 222 249 appliedRedirect := false 223 223 - status := 200 250 250 + status := http.StatusOK 224 251 reader := io.ReadSeeker(nil) 225 252 mtime := time.Time{} 226 253 for { ··· 234 261 entry = manifest.Contents[entryPath] 235 262 if !appliedRedirect { 236 263 redirectKind := RedirectAny 237 237 - if entry != nil && entry.GetType() != Type_Invalid { 264 264 + if entry != nil && entry.GetType() != Type_InvalidEntry { 238 265 redirectKind = RedirectForce 239 266 } 240 267 originalURL := (&url.URL{Host: r.Host}).ResolveReference(r.URL) 241 241 - redirectURL, redirectStatus := ApplyRedirectRules(manifest, originalURL, redirectKind) 268 268 + _, redirectURL, redirectStatus := ApplyRedirectRules(manifest, originalURL, redirectKind) 242 269 if Is3xxHTTPStatus(redirectStatus) { 243 270 writeRedirect(w, redirectStatus, redirectURL.String()) 244 271 return nil ··· 251 278 continue 252 279 } 253 280 } 254 254 - if entry == nil || entry.GetType() == Type_Invalid { 255 255 - status = 404 281 281 + if entry == nil || entry.GetType() == Type_InvalidEntry { 282 282 + status = http.StatusNotFound 256 283 if entryPath != notFoundPage { 257 284 entryPath = notFoundPage 258 285 continue ··· 268 295 w.WriteHeader(http.StatusNotModified) 269 296 return nil 270 297 } else { 271 271 - reader, _, mtime, err = backend.GetBlob(r.Context(), string(entry.Data)) 298 298 + var metadata BlobMetadata 299 299 + reader, metadata, err = backend.GetBlob(r.Context(), string(entry.Data)) 272 300 if err != nil { 273 301 ObserveError(err) // all storage errors must be reported 274 302 w.WriteHeader(http.StatusInternalServerError) 275 303 fmt.Fprintf(w, "internal server error: %s\n", err) 276 304 return err 277 305 } 306 306 + mtime = metadata.LastModified 278 307 w.Header().Set("ETag", etag) 279 308 } 280 309 } else if entry.GetType() == Type_Directory { ··· 297 326 defer closer.Close() 298 327 } 299 328 300 300 - acceptedEncodings := parseHTTPEncodings(r.Header.Get("Accept-Encoding")) 329 329 + var offeredEncodings []string 330 330 + acceptedEncodings := ParseAcceptEncodingHeader(r.Header.Get("Accept-Encoding")) 331 331 + w.Header().Add("Vary", "Accept-Encoding") 301 332 negotiatedEncoding := true 302 333 switch entry.GetTransform() { 303 303 - case Transform_None: 304 304 - if acceptedEncodings.Negotiate("identity") != "identity" { 334 334 + case Transform_Identity: 335 335 + offeredEncodings = []string{"identity"} 336 336 + switch acceptedEncodings.Negotiate(offeredEncodings...) { 337 337 + case "identity": 338 338 + serveEncodingCount. 339 339 + With(prometheus.Labels{"transform": "identity", "negotiated": "identity"}). 340 340 + Inc() 341 341 + default: 305 342 negotiatedEncoding = false 343 343 + serveEncodingCount. 344 344 + With(prometheus.Labels{"transform": "identity", "negotiated": "failure"}). 345 345 + Inc() 306 346 } 307 307 - case Transform_Zstandard: 308 308 - supported := []string{"zstd", "identity"} 347 347 + case Transform_Zstd: 348 348 + offeredEncodings = []string{"zstd", "identity"} 309 349 if entry.ContentType == nil { 310 350 // If Content-Type is unset, `http.ServeContent` will try to sniff 311 351 // the file contents. That won't work if it's compressed. 312 312 - supported = []string{"identity"} 352 352 + offeredEncodings = []string{"identity"} 313 353 } 314 314 - switch acceptedEncodings.Negotiate(supported...) { 354 354 + switch acceptedEncodings.Negotiate(offeredEncodings...) { 315 355 case "zstd": 316 356 // Set Content-Length ourselves since `http.ServeContent` only sets 317 357 // it if Content-Encoding is unset or if it's a range request. 318 318 - w.Header().Set("Content-Length", strconv.FormatInt(*entry.Size, 10)) 358 358 + w.Header().Set("Content-Length", strconv.FormatInt(entry.GetCompressedSize(), 10)) 319 359 w.Header().Set("Content-Encoding", "zstd") 360 360 + serveEncodingCount. 361 361 + With(prometheus.Labels{"transform": "zstd", "negotiated": "zstd"}). 362 362 + Inc() 320 363 case "identity": 321 364 compressedData, _ := io.ReadAll(reader) 322 365 decompressedData, err := zstdDecoder.DecodeAll(compressedData, []byte{}) ··· 326 369 return err 327 370 } 328 371 reader = bytes.NewReader(decompressedData) 372 372 + serveEncodingCount. 373 373 + With(prometheus.Labels{"transform": "zstd", "negotiated": "identity"}). 374 374 + Inc() 329 375 default: 330 376 negotiatedEncoding = false 377 377 + serveEncodingCount. 378 378 + With(prometheus.Labels{"transform": "zstd", "negotiated": "failure"}). 379 379 + Inc() 331 380 } 332 381 default: 333 382 return fmt.Errorf("unexpected transform") 334 383 } 335 384 if !negotiatedEncoding { 385 385 + w.Header().Set("Accept-Encoding", strings.Join(offeredEncodings, ", ")) 336 386 w.WriteHeader(http.StatusNotAcceptable) 337 337 - return fmt.Errorf("no supported content encodings (accept-encoding: %q)", 387 387 + return fmt.Errorf("no supported content encodings (Accept-Encoding: %s)", 338 388 r.Header.Get("Accept-Encoding")) 339 389 } 340 390 ··· 369 419 io.Copy(w, reader) 370 420 } 371 421 } else { 372 372 - // consider content fresh for 60 seconds (the same as the freshness interval of 373 373 - // manifests in the S3 backend), and use stale content anyway as long as it's not 374 374 - // older than a hour; while it is cheap to handle If-Modified-Since queries 375 375 - // server-side, on the client `max-age=0, must-revalidate` causes every resource 376 376 - // to block the page load every time 377 377 - w.Header().Set("Cache-Control", "max-age=60, stale-while-revalidate=3600") 378 378 - // see https://web.dev/articles/stale-while-revalidate for details 422 422 + if _, hasCacheControl := w.Header()["Cache-Control"]; !hasCacheControl { 423 423 + // consider content fresh for 60 seconds (the same as the freshness interval of 424 424 + // manifests in the S3 backend), and use stale content anyway as long as it's not 425 425 + // older than a hour; while it is cheap to handle If-Modified-Since queries 426 426 + // server-side, on the client `max-age=0, must-revalidate` causes every resource 427 427 + // to block the page load every time 428 428 + w.Header().Set("Cache-Control", "max-age=60, stale-while-revalidate=3600") 429 429 + // see https://web.dev/articles/stale-while-revalidate for details 430 430 + } 379 431 380 432 // http.ServeContent handles conditional requests and range requests 381 433 http.ServeContent(w, r, entryPath, mtime, reader) ··· 383 435 return nil 384 436 } 385 437 438 438 + func checkDryRun(w http.ResponseWriter, r *http.Request) bool { 439 439 + // "Dry run" requests are used to non-destructively check if the request would have 440 440 + // successfully been authorized. 441 441 + if r.Header.Get("Dry-Run") != "" { 442 442 + fmt.Fprintln(w, "dry-run ok") 443 443 + return true 444 444 + } 445 445 + return false 446 446 + } 447 447 + 386 448 func putPage(w http.ResponseWriter, r *http.Request) error { 387 449 var result UpdateResult 388 450 389 389 - host, err := GetHost(r) 390 390 - if err != nil { 391 391 - return err 451 451 + for _, header := range []string{ 452 452 + "If-Modified-Since", "If-Unmodified-Since", "If-Match", "If-None-Match", 453 453 + } { 454 454 + if r.Header.Get(header) != "" { 455 455 + http.Error(w, fmt.Sprintf("unsupported precondition %s", header), http.StatusBadRequest) 456 456 + return nil 457 457 + } 392 458 } 393 459 394 394 - projectName, err := GetProjectName(r) 460 460 + webRoot, err := getWebRoot(r) 395 461 if err != nil { 396 462 return err 397 463 } 398 464 399 399 - webRoot := makeWebRoot(host, projectName) 400 400 - 401 401 - updateCtx, cancel := context.WithTimeout(r.Context(), time.Duration(config.Limits.UpdateTimeout)) 465 465 + ctx, cancel := context.WithTimeout(r.Context(), time.Duration(config.Limits.UpdateTimeout)) 402 466 defer cancel() 403 467 404 468 contentType := getMediaType(r.Header.Get("Content-Type")) 405 405 - 406 406 - if contentType == "application/x-www-form-urlencoded" { 469 469 + switch contentType { 470 470 + case "application/x-www-form-urlencoded": 407 471 auth, err := AuthorizeUpdateFromRepository(r) 408 472 if err != nil { 409 473 return err ··· 428 492 return err 429 493 } 430 494 431 431 - result = UpdateFromRepository(updateCtx, webRoot, repoURL, branch) 432 432 - } else { 495 495 + if checkDryRun(w, r) { 496 496 + return nil 497 497 + } 498 498 + 499 499 + result = UpdateFromRepository(ctx, webRoot, repoURL, branch) 500 500 + 501 501 + default: 433 502 _, err := AuthorizeUpdateFromArchive(r) 434 503 if err != nil { 435 504 return err 436 505 } 437 506 507 507 + if checkDryRun(w, r) { 508 508 + return nil 509 509 + } 510 510 + 438 511 // request body contains archive 439 512 reader := http.MaxBytesReader(w, r.Body, int64(config.Limits.MaxSiteSize.Bytes())) 440 440 - result = UpdateFromArchive(updateCtx, webRoot, contentType, reader) 513 513 + result = UpdateFromArchive(ctx, webRoot, contentType, reader) 514 514 + } 515 515 + 516 516 + return reportUpdateResult(w, r, result) 517 517 + } 518 518 + 519 519 + func patchPage(w http.ResponseWriter, r *http.Request) error { 520 520 + for _, header := range []string{ 521 521 + "If-Modified-Since", "If-Unmodified-Since", "If-Match", "If-None-Match", 522 522 + } { 523 523 + if r.Header.Get(header) != "" { 524 524 + http.Error(w, fmt.Sprintf("unsupported precondition %s", header), http.StatusBadRequest) 525 525 + return nil 526 526 + } 527 527 + } 528 528 + 529 529 + webRoot, err := getWebRoot(r) 530 530 + if err != nil { 531 531 + return err 532 532 + } 533 533 + 534 534 + if _, err = AuthorizeUpdateFromArchive(r); err != nil { 535 535 + return err 536 536 + } 537 537 + 538 538 + if checkDryRun(w, r) { 539 539 + return nil 540 540 + } 541 541 + 542 542 + // Providing atomic compare-and-swap operations might be difficult or impossible depending 543 543 + // on the backend in use and its configuration, but for applications where a mostly-atomic 544 544 + // compare-and-swap operation is good enough (e.g. generating page previews) we don't want 545 545 + // to prevent the use of partial updates. 546 546 + wantAtomicCAS := r.Header.Get("Atomic") 547 547 + hasAtomicCAS := backend.HasAtomicCAS(r.Context()) 548 548 + switch { 549 549 + case wantAtomicCAS == "yes" && hasAtomicCAS || wantAtomicCAS == "no": 550 550 + // all good 551 551 + case wantAtomicCAS == "yes": 552 552 + http.Error(w, "atomic partial updates unsupported", http.StatusPreconditionFailed) 553 553 + return nil 554 554 + case wantAtomicCAS == "": 555 555 + http.Error(w, "must provide \"Atomic: yes|no\" header", http.StatusPreconditionRequired) 556 556 + return nil 557 557 + default: 558 558 + http.Error(w, "malformed Atomic: header", http.StatusBadRequest) 559 559 + return nil 560 560 + } 561 561 + 562 562 + var parents CreateParentsMode 563 563 + switch r.Header.Get("Create-Parents") { 564 564 + case "", "no": 565 565 + parents = RequireParents 566 566 + case "yes": 567 567 + parents = CreateParents 568 568 + default: 569 569 + http.Error(w, "malformed Create-Parents: header", http.StatusBadRequest) 570 570 + return nil 571 571 + } 572 572 + 573 573 + ctx, cancel := context.WithTimeout(r.Context(), time.Duration(config.Limits.UpdateTimeout)) 574 574 + defer cancel() 575 575 + 576 576 + contentType := getMediaType(r.Header.Get("Content-Type")) 577 577 + reader := http.MaxBytesReader(w, r.Body, int64(config.Limits.MaxSiteSize.Bytes())) 578 578 + result := PartialUpdateFromArchive(ctx, webRoot, contentType, reader, parents) 579 579 + return reportUpdateResult(w, r, result) 580 580 + } 581 581 + 582 582 + func reportUpdateResult(w http.ResponseWriter, r *http.Request, result UpdateResult) error { 583 583 + var unresolvedRefErr UnresolvedRefError 584 584 + if result.outcome == UpdateError && errors.As(result.err, &unresolvedRefErr) { 585 585 + offeredContentTypes := []string{"text/plain", "application/vnd.git-pages.unresolved"} 586 586 + acceptedContentTypes := ParseAcceptHeader(r.Header.Get("Accept")) 587 587 + switch acceptedContentTypes.Negotiate(offeredContentTypes...) { 588 588 + default: 589 589 + w.Header().Set("Accept", strings.Join(offeredContentTypes, ", ")) 590 590 + w.WriteHeader(http.StatusNotAcceptable) 591 591 + return fmt.Errorf("no supported content types (Accept: %s)", r.Header.Get("Accept")) 592 592 + case "application/vnd.git-pages.unresolved": 593 593 + w.Header().Set("Content-Type", "application/vnd.git-pages.unresolved") 594 594 + w.WriteHeader(http.StatusUnprocessableEntity) 595 595 + for _, missingRef := range unresolvedRefErr.missing { 596 596 + fmt.Fprintln(w, missingRef) 597 597 + } 598 598 + return nil 599 599 + case "text/plain": 600 600 + // handled below 601 601 + } 441 602 } 442 603 443 604 switch result.outcome { 444 605 case UpdateError: 445 445 - if errors.Is(result.err, ErrManifestTooLarge) { 446 446 - w.WriteHeader(http.StatusRequestEntityTooLarge) 606 606 + if errors.Is(result.err, ErrSiteTooLarge) { 607 607 + w.WriteHeader(http.StatusUnprocessableEntity) 608 608 + } else if errors.Is(result.err, ErrManifestTooLarge) { 609 609 + w.WriteHeader(http.StatusUnprocessableEntity) 447 610 } else if errors.Is(result.err, errArchiveFormat) { 448 611 w.WriteHeader(http.StatusUnsupportedMediaType) 449 612 } else if errors.Is(result.err, ErrArchiveTooLarge) { 450 613 w.WriteHeader(http.StatusRequestEntityTooLarge) 614 614 + } else if errors.Is(result.err, ErrRepositoryTooLarge) { 615 615 + w.WriteHeader(http.StatusUnprocessableEntity) 616 616 + } else if errors.Is(result.err, ErrMalformedPatch) { 617 617 + w.WriteHeader(http.StatusUnprocessableEntity) 618 618 + } else if errors.Is(result.err, ErrPreconditionFailed) { 619 619 + w.WriteHeader(http.StatusPreconditionFailed) 620 620 + } else if errors.Is(result.err, ErrWriteConflict) { 621 621 + w.WriteHeader(http.StatusConflict) 622 622 + } else if errors.Is(result.err, ErrDomainFrozen) { 623 623 + w.WriteHeader(http.StatusForbidden) 624 624 + } else if errors.As(result.err, &unresolvedRefErr) { 625 625 + w.WriteHeader(http.StatusUnprocessableEntity) 451 626 } else { 452 627 w.WriteHeader(http.StatusServiceUnavailable) 453 628 } ··· 476 651 } else { 477 652 fmt.Fprintln(w, "internal error") 478 653 } 479 479 - reportSiteUpdate("rest", &result) 654 654 + observeSiteUpdate("rest", &result) 480 655 return nil 481 656 } 482 657 483 658 func deletePage(w http.ResponseWriter, r *http.Request) error { 484 484 - _, err := AuthorizeUpdateFromRepository(r) 659 659 + webRoot, err := getWebRoot(r) 485 660 if err != nil { 486 661 return err 487 662 } 488 663 489 489 - host, err := GetHost(r) 664 664 + _, err = AuthorizeUpdateFromRepository(r) 490 665 if err != nil { 491 666 return err 492 667 } 493 668 494 494 - projectName, err := GetProjectName(r) 495 495 - if err != nil { 496 496 - return err 669 669 + if checkDryRun(w, r) { 670 670 + return nil 497 671 } 498 672 499 499 - err = backend.DeleteManifest(r.Context(), makeWebRoot(host, projectName)) 500 500 - if err != nil { 673 673 + if err = backend.DeleteManifest(r.Context(), webRoot, ModifyManifestOptions{}); err != nil { 501 674 w.WriteHeader(http.StatusInternalServerError) 675 675 + fmt.Fprintln(w, err) 502 676 } else { 503 677 w.Header().Add("Update-Result", "deleted") 504 678 w.WriteHeader(http.StatusOK) 505 679 } 506 506 - if err != nil { 507 507 - fmt.Fprintln(w, err) 508 508 - } 509 680 return err 510 681 } 511 682 ··· 514 685 requestTimeout := 3 * time.Second 515 686 requestTimer := time.NewTimer(requestTimeout) 516 687 517 517 - auth, err := AuthorizeUpdateFromRepository(r) 518 518 - if err != nil { 519 519 - return err 520 520 - } 521 521 - 522 522 - host, err := GetHost(r) 688 688 + webRoot, err := getWebRoot(r) 523 689 if err != nil { 524 690 return err 525 691 } 526 692 527 527 - projectName, err := GetProjectName(r) 693 693 + auth, err := AuthorizeUpdateFromRepository(r) 528 694 if err != nil { 529 695 return err 530 696 } 531 697 532 532 - webRoot := makeWebRoot(host, projectName) 533 533 - 534 698 eventName := "" 535 699 for _, header := range []string{ 536 700 "X-Forgejo-Event", ··· 578 742 return err 579 743 } 580 744 581 581 - if event.Ref != fmt.Sprintf("refs/heads/%s", auth.branch) { 745 745 + if event.Ref != path.Join("refs", "heads", auth.branch) { 582 746 code := http.StatusUnauthorized 583 747 if strings.Contains(r.Header.Get("User-Agent"), "GitHub-Hookshot") { 584 748 // GitHub has no way to restrict branches for a webhook, and responding with 401 ··· 596 760 return err 597 761 } 598 762 763 763 + if checkDryRun(w, r) { 764 764 + return nil 765 765 + } 766 766 + 599 767 resultChan := make(chan UpdateResult) 600 768 go func(ctx context.Context) { 601 769 ctx, cancel := context.WithTimeout(ctx, time.Duration(config.Limits.UpdateTimeout)) ··· 603 771 604 772 result := UpdateFromRepository(ctx, webRoot, repoURL, auth.branch) 605 773 resultChan <- result 606 606 - reportSiteUpdate("webhook", &result) 607 607 - }(context.Background()) 774 774 + observeSiteUpdate("webhook", &result) 775 775 + }(context.WithoutCancel(r.Context())) 608 776 609 777 var result UpdateResult 610 778 select { ··· 623 791 w.WriteHeader(http.StatusGatewayTimeout) 624 792 fmt.Fprintln(w, "update timeout") 625 793 case UpdateNoChange: 626 626 - w.WriteHeader(http.StatusOK) 627 794 fmt.Fprintln(w, "unchanged") 628 795 case UpdateCreated: 629 629 - w.WriteHeader(http.StatusOK) 630 796 fmt.Fprintln(w, "created") 631 797 case UpdateReplaced: 632 632 - w.WriteHeader(http.StatusOK) 633 798 fmt.Fprintln(w, "replaced") 634 799 case UpdateDeleted: 635 635 - w.WriteHeader(http.StatusOK) 636 800 fmt.Fprintln(w, "deleted") 637 801 } 638 802 if result.manifest != nil { ··· 648 812 } 649 813 650 814 func ServePages(w http.ResponseWriter, r *http.Request) { 815 815 + r = r.WithContext(WithPrincipal(r.Context())) 816 816 + if config.Audit.IncludeIPs != "" { 817 817 + GetPrincipal(r.Context()).IpAddress = proto.String(r.RemoteAddr) 818 818 + } 651 819 // We want upstream health checks to be done as closely to the normal flow as possible; 652 820 // any intentional deviation is an opportunity to miss an issue that will affect our 653 821 // visitors but not our health checks. 654 822 if r.Header.Get("Health-Check") == "" { 655 655 - log.Println("pages:", r.Method, r.Host, r.URL, r.Header.Get("Content-Type")) 823 823 + var mediaType string 824 824 + switch r.Method { 825 825 + case "HEAD", "GET": 826 826 + mediaType = r.Header.Get("Accept") 827 827 + default: 828 828 + mediaType = r.Header.Get("Content-Type") 829 829 + } 830 830 + logc.Println(r.Context(), "pages:", r.Method, r.Host, r.URL, mediaType) 656 831 if region := os.Getenv("FLY_REGION"); region != "" { 657 832 machine_id := os.Getenv("FLY_MACHINE_ID") 658 833 w.Header().Add("Server", fmt.Sprintf("git-pages (fly.io; %s; %s)", region, machine_id)) ··· 667 842 } 668 843 } 669 844 } 845 845 + allowedMethods := []string{"OPTIONS", "HEAD", "GET", "PUT", "PATCH", "DELETE", "POST"} 846 846 + if r.Method == "OPTIONS" || !slices.Contains(allowedMethods, r.Method) { 847 847 + w.Header().Add("Allow", strings.Join(allowedMethods, ", ")) 848 848 + } 670 849 err := error(nil) 671 850 switch r.Method { 672 851 // REST API 673 673 - case http.MethodHead, http.MethodGet: 852 852 + case "OPTIONS": 853 853 + // no preflight options 854 854 + case "HEAD", "GET": 674 855 err = getPage(w, r) 675 675 - case http.MethodPut: 856 856 + case "PUT": 676 857 err = putPage(w, r) 677 677 - case http.MethodDelete: 858 858 + case "PATCH": 859 859 + err = patchPage(w, r) 860 860 + case "DELETE": 678 861 err = deletePage(w, r) 679 862 // webhook API 680 680 - case http.MethodPost: 863 863 + case "POST": 681 864 err = postPage(w, r) 682 865 default: 683 683 - w.Header().Add("Allow", "HEAD, GET, PUT, DELETE, POST") 684 866 http.Error(w, "method not allowed", http.StatusMethodNotAllowed) 685 867 err = fmt.Errorf("method %s not allowed", r.Method) 686 868 } ··· 695 877 http.Error(w, message, http.StatusRequestEntityTooLarge) 696 878 err = errors.New(message) 697 879 } 698 698 - log.Println("pages err:", err) 880 880 + logc.Println(r.Context(), "pages err:", err) 699 881 } 700 882 }

+55

src/pages_test.go

··· 1 1 + package git_pages 2 2 + 3 3 + import ( 4 4 + "net/http" 5 5 + "strings" 6 6 + "testing" 7 7 + ) 8 8 + 9 9 + func checkHost(t *testing.T, host string, expectOk string, expectErr string) { 10 10 + host, err := GetHost(&http.Request{Host: host}) 11 11 + if expectErr != "" { 12 12 + if err == nil || !strings.HasPrefix(err.Error(), expectErr) { 13 13 + t.Errorf("%s: expect err %s, got err %s", host, expectErr, err) 14 14 + } 15 15 + } 16 16 + if expectOk != "" { 17 17 + if err != nil { 18 18 + t.Errorf("%s: expect ok %s, got err %s", host, expectOk, err) 19 19 + } else if host != expectOk { 20 20 + t.Errorf("%s: expect ok %s, got ok %s", host, expectOk, host) 21 21 + } 22 22 + } 23 23 + } 24 24 + 25 25 + func TestHelloName(t *testing.T) { 26 26 + config = &Config{Features: []string{}} 27 27 + 28 28 + checkHost(t, "foo.bar", "foo.bar", "") 29 29 + checkHost(t, "foo-baz.bar", "foo-baz.bar", "") 30 30 + checkHost(t, "foo--baz.bar", "foo--baz.bar", "") 31 31 + checkHost(t, "foo.bar.", "foo.bar", "") 32 32 + checkHost(t, ".foo.bar", "", "reserved host name") 33 33 + checkHost(t, "..foo.bar", "", "reserved host name") 34 34 + 35 35 + checkHost(t, "ß.bar", "xn--zca.bar", "") 36 36 + checkHost(t, "xn--zca.bar", "xn--zca.bar", "") 37 37 + 38 38 + checkHost(t, "foo-.bar", "", "malformed host name") 39 39 + checkHost(t, "-foo.bar", "", "malformed host name") 40 40 + checkHost(t, "foo_.bar", "", "malformed host name") 41 41 + checkHost(t, "_foo.bar", "", "malformed host name") 42 42 + checkHost(t, "foo_baz.bar", "", "malformed host name") 43 43 + checkHost(t, "foo__baz.bar", "", "malformed host name") 44 44 + checkHost(t, "*.foo.bar", "", "malformed host name") 45 45 + 46 46 + config = &Config{Features: []string{"relaxed-idna"}} 47 47 + 48 48 + checkHost(t, "foo-.bar", "", "malformed host name") 49 49 + checkHost(t, "-foo.bar", "", "malformed host name") 50 50 + checkHost(t, "foo_.bar", "foo_.bar", "") 51 51 + checkHost(t, "_foo.bar", "", "reserved host name") 52 52 + checkHost(t, "foo_baz.bar", "foo_baz.bar", "") 53 53 + checkHost(t, "foo__baz.bar", "foo__baz.bar", "") 54 54 + checkHost(t, "*.foo.bar", "", "malformed host name") 55 55 + }

+142

src/patch.go

··· 1 1 + package git_pages 2 2 + 3 3 + import ( 4 4 + "archive/tar" 5 5 + "errors" 6 6 + "fmt" 7 7 + "io" 8 8 + "maps" 9 9 + "slices" 10 10 + "strings" 11 11 + ) 12 12 + 13 13 + var ErrMalformedPatch = errors.New("malformed patch") 14 14 + 15 15 + type CreateParentsMode int 16 16 + 17 17 + const ( 18 18 + RequireParents CreateParentsMode = iota 19 19 + CreateParents 20 20 + ) 21 21 + 22 22 + // Mutates `manifest` according to a tar stream and the following rules: 23 23 + // - A character device with major 0 and minor 0 is a "whiteout marker". When placed 24 24 + // at a given path, this path and its entire subtree (if any) are removed from the manifest. 25 25 + // - When a directory is placed at a given path, this path and its entire subtree (if any) are 26 26 + // removed from the manifest and replaced with the contents of the directory. 27 27 + func ApplyTarPatch(manifest *Manifest, reader io.Reader, parents CreateParentsMode) error { 28 28 + type Node struct { 29 29 + entry *Entry 30 30 + children map[string]*Node 31 31 + } 32 32 + 33 33 + // Extract the manifest contents (which is using a flat hash map) into a directory tree 34 34 + // so that recursive delete operations have O(1) complexity. s 35 35 + var root *Node 36 36 + sortedNames := slices.Sorted(maps.Keys(manifest.GetContents())) 37 37 + for _, name := range sortedNames { 38 38 + entry := manifest.Contents[name] 39 39 + node := &Node{entry: entry} 40 40 + if entry.GetType() == Type_Directory { 41 41 + node.children = map[string]*Node{} 42 42 + } 43 43 + if name == "" { 44 44 + root = node 45 45 + } else { 46 46 + segments := strings.Split(name, "/") 47 47 + fileName := segments[len(segments)-1] 48 48 + iter := root 49 49 + for _, segment := range segments[:len(segments)-1] { 50 50 + if iter.children == nil { 51 51 + panic("malformed manifest") 52 52 + } else if _, exists := iter.children[segment]; !exists { 53 53 + panic("malformed manifest") 54 54 + } else { 55 55 + iter = iter.children[segment] 56 56 + } 57 57 + } 58 58 + iter.children[fileName] = node 59 59 + } 60 60 + } 61 61 + manifest.Contents = map[string]*Entry{} 62 62 + 63 63 + // Process the archive as a patch operation. 64 64 + archive := tar.NewReader(reader) 65 65 + for { 66 66 + header, err := archive.Next() 67 67 + if err == io.EOF { 68 68 + break 69 69 + } else if err != nil { 70 70 + return err 71 71 + } 72 72 + 73 73 + segments := strings.Split(strings.TrimRight(header.Name, "/"), "/") 74 74 + fileName := segments[len(segments)-1] 75 75 + node := root 76 76 + for index, segment := range segments[:len(segments)-1] { 77 77 + if node.children == nil { 78 78 + dirName := strings.Join(segments[:index], "/") 79 79 + return fmt.Errorf("%w: %s: not a directory", ErrMalformedPatch, dirName) 80 80 + } 81 81 + if _, exists := node.children[segment]; !exists { 82 82 + switch parents { 83 83 + case RequireParents: 84 84 + nodeName := strings.Join(segments[:index+1], "/") 85 85 + return fmt.Errorf("%w: %s: path not found", ErrMalformedPatch, nodeName) 86 86 + case CreateParents: 87 87 + node.children[segment] = &Node{ 88 88 + entry: NewManifestEntry(Type_Directory, nil), 89 89 + children: map[string]*Node{}, 90 90 + } 91 91 + } 92 92 + } 93 93 + node = node.children[segment] 94 94 + } 95 95 + if node.children == nil { 96 96 + dirName := strings.Join(segments[:len(segments)-1], "/") 97 97 + return fmt.Errorf("%w: %s: not a directory", ErrMalformedPatch, dirName) 98 98 + } 99 99 + 100 100 + switch header.Typeflag { 101 101 + case tar.TypeReg: 102 102 + fileData, err := io.ReadAll(archive) 103 103 + if err != nil { 104 104 + return fmt.Errorf("tar: %s: %w", header.Name, err) 105 105 + } 106 106 + node.children[fileName] = &Node{ 107 107 + entry: NewManifestEntry(Type_InlineFile, fileData), 108 108 + } 109 109 + case tar.TypeSymlink: 110 110 + node.children[fileName] = &Node{ 111 111 + entry: NewManifestEntry(Type_Symlink, []byte(header.Linkname)), 112 112 + } 113 113 + case tar.TypeDir: 114 114 + node.children[fileName] = &Node{ 115 115 + entry: NewManifestEntry(Type_Directory, nil), 116 116 + children: map[string]*Node{}, 117 117 + } 118 118 + case tar.TypeChar: 119 119 + if header.Devmajor == 0 && header.Devminor == 0 { 120 120 + delete(node.children, fileName) 121 121 + } else { 122 122 + AddProblem(manifest, header.Name, 123 123 + "tar: unsupported chardev %d,%d", header.Devmajor, header.Devminor) 124 124 + } 125 125 + default: 126 126 + AddProblem(manifest, header.Name, 127 127 + "tar: unsupported type '%c'", header.Typeflag) 128 128 + continue 129 129 + } 130 130 + } 131 131 + 132 132 + // Repopulate manifest contents with the updated directory tree. 133 133 + var traverse func([]string, *Node) 134 134 + traverse = func(segments []string, node *Node) { 135 135 + manifest.Contents[strings.Join(segments, "/")] = node.entry 136 136 + for fileName, childNode := range node.children { 137 137 + traverse(append(segments, fileName), childNode) 138 138 + } 139 139 + } 140 140 + traverse([]string{}, root) 141 141 + return nil 142 142 + }

+58 -14

src/redirects.go

··· 13 13 14 14 const RedirectsFileName string = "_redirects" 15 15 16 16 - func unparseRule(rule redirects.Rule) string { 16 16 + // Converts our Protobuf representation to tj/go-redirects. 17 17 + func exportRedirectRule(rule *RedirectRule) *redirects.Rule { 18 18 + return &redirects.Rule{ 19 19 + From: rule.GetFrom(), 20 20 + To: rule.GetTo(), 21 21 + Status: int(rule.GetStatus()), 22 22 + Force: rule.GetForce(), 23 23 + } 24 24 + } 25 25 + 26 26 + func unparseRedirectRule(rule *redirects.Rule) string { 17 27 var statusPart string 18 28 if rule.Force { 19 29 statusPart = fmt.Sprintf("%d!", rule.Status) ··· 49 59 return status >= 300 && status <= 399 50 60 } 51 61 52 52 - func validateRedirectRule(rule redirects.Rule) error { 62 62 + func validateRedirectRule(rule *redirects.Rule) error { 53 63 if len(rule.Params) > 0 { 54 64 return fmt.Errorf("rules with parameters are not supported") 55 65 } ··· 103 113 } 104 114 105 115 for index, rule := range rules { 106 106 - if err := validateRedirectRule(rule); err != nil { 116 116 + if err := validateRedirectRule(&rule); err != nil { 107 117 AddProblem(manifest, RedirectsFileName, 108 108 - "rule #%d %q: %s", index+1, unparseRule(rule), err) 118 118 + "rule #%d %q: %s", index+1, unparseRedirectRule(&rule), err) 109 119 continue 110 120 } 111 121 manifest.Redirects = append(manifest.Redirects, &RedirectRule{ ··· 121 131 func CollectRedirectsFile(manifest *Manifest) string { 122 132 var rules []string 123 133 for _, rule := range manifest.GetRedirects() { 124 124 - rules = append(rules, unparseRule(redirects.Rule{ 125 125 - From: rule.GetFrom(), 126 126 - To: rule.GetTo(), 127 127 - Status: int(rule.GetStatus()), 128 128 - Force: rule.GetForce(), 129 129 - })+"\n") 134 134 + rules = append(rules, unparseRedirectRule(exportRedirectRule(rule))+"\n") 130 135 } 131 136 return strings.Join(rules, "") 132 137 } ··· 147 152 148 153 const ( 149 154 RedirectAny RedirectKind = iota 155 155 + RedirectNormal 150 156 RedirectForce 151 157 ) 152 158 153 159 func ApplyRedirectRules( 154 160 manifest *Manifest, fromURL *url.URL, kind RedirectKind, 155 161 ) ( 156 156 - toURL *url.URL, status int, 162 162 + rule *RedirectRule, toURL *url.URL, status int, 157 163 ) { 158 164 fromSegments := pathSegments(fromURL.Path) 159 165 next: 160 160 - for _, rule := range manifest.Redirects { 161 161 - if kind == RedirectForce && !*rule.Force { 166 166 + for _, rule = range manifest.Redirects { 167 167 + switch { 168 168 + case kind == RedirectNormal && *rule.Force: 169 169 + continue 170 170 + case kind == RedirectForce && !*rule.Force: 162 171 continue 163 172 } 164 173 // check if the rule matches fromURL ··· 205 214 RawQuery: fromURL.RawQuery, 206 215 } 207 216 status = int(*rule.Status) 208 208 - break 217 217 + return 209 218 } 210 219 // no redirect found 220 220 + rule = nil 211 221 return 212 222 } 223 223 + 224 224 + func redirectHasSplat(rule *RedirectRule) bool { 225 225 + ruleFromURL, _ := url.Parse(*rule.From) // pre-validated in `validateRedirectRule` 226 226 + ruleFromSegments := pathSegments(ruleFromURL.Path) 227 227 + return slices.Contains(ruleFromSegments, "*") 228 228 + } 229 229 + 230 230 + func LintRedirects(manifest *Manifest) { 231 231 + for name, entry := range manifest.GetContents() { 232 232 + nameURL, err := url.Parse("/" + name) 233 233 + if err != nil { 234 234 + continue 235 235 + } 236 236 + 237 237 + // Check if the entry URL would trigger a non-forced redirect if the entry didn't exist. 238 238 + // If the redirect matches exactly one URL (i.e. has no splat) then it will never be 239 239 + // triggered and an issue is reported; if the rule has a splat, it will always be possible 240 240 + // to trigger it, as it matches an infinite number of URLs. 241 241 + rule, _, _ := ApplyRedirectRules(manifest, nameURL, RedirectNormal) 242 242 + if rule != nil && !redirectHasSplat(rule) { 243 243 + entryDesc := "file" 244 244 + if entry.GetType() == Type_Directory { 245 245 + entryDesc = "directory" 246 246 + } 247 247 + AddProblem(manifest, name, 248 248 + "%s shadows redirect %q; remove the %s or use a %d! forced redirect instead", 249 249 + entryDesc, 250 250 + unparseRedirectRule(exportRedirectRule(rule)), 251 251 + entryDesc, 252 252 + rule.GetStatus(), 253 253 + ) 254 254 + } 255 255 + } 256 256 + }

+326 -61

src/schema.pb.go

··· 9 9 import ( 10 10 protoreflect "google.golang.org/protobuf/reflect/protoreflect" 11 11 protoimpl "google.golang.org/protobuf/runtime/protoimpl" 12 12 + timestamppb "google.golang.org/protobuf/types/known/timestamppb" 12 13 reflect "reflect" 13 14 sync "sync" 14 15 unsafe "unsafe" ··· 25 26 26 27 const ( 27 28 // Invalid entry. 28 28 - Type_Invalid Type = 0 29 29 + Type_InvalidEntry Type = 0 29 30 // Directory. 30 31 Type_Directory Type = 1 31 32 // Inline file. `Blob.Data` contains file contents. ··· 39 40 // Enum value maps for Type. 40 41 var ( 41 42 Type_name = map[int32]string{ 42 42 - 0: "Invalid", 43 43 + 0: "InvalidEntry", 43 44 1: "Directory", 44 45 2: "InlineFile", 45 46 3: "ExternalFile", 46 47 4: "Symlink", 47 48 } 48 49 Type_value = map[string]int32{ 49 49 - "Invalid": 0, 50 50 + "InvalidEntry": 0, 50 51 "Directory": 1, 51 52 "InlineFile": 2, 52 53 "ExternalFile": 3, ··· 81 82 return file_schema_proto_rawDescGZIP(), []int{0} 82 83 } 83 84 85 85 + // Transformation names should match HTTP `Accept-Encoding:` header. 84 86 type Transform int32 85 87 86 88 const ( 87 89 // No transformation. 88 88 - Transform_None Transform = 0 90 90 + Transform_Identity Transform = 0 89 91 // Zstandard compression. 90 90 - Transform_Zstandard Transform = 1 92 92 + Transform_Zstd Transform = 1 91 93 ) 92 94 93 95 // Enum value maps for Transform. 94 96 var ( 95 97 Transform_name = map[int32]string{ 96 96 - 0: "None", 97 97 - 1: "Zstandard", 98 98 + 0: "Identity", 99 99 + 1: "Zstd", 98 100 } 99 101 Transform_value = map[string]int32{ 100 100 - "None": 0, 101 101 - "Zstandard": 1, 102 102 + "Identity": 0, 103 103 + "Zstd": 1, 102 104 } 103 105 ) 104 106 ··· 129 131 return file_schema_proto_rawDescGZIP(), []int{1} 130 132 } 131 133 134 134 + type AuditEvent int32 135 135 + 136 136 + const ( 137 137 + // Invalid event. 138 138 + AuditEvent_InvalidEvent AuditEvent = 0 139 139 + // A manifest was committed (a site was created or updated). 140 140 + AuditEvent_CommitManifest AuditEvent = 1 141 141 + // A manifest was deleted (a site was deleted). 142 142 + AuditEvent_DeleteManifest AuditEvent = 2 143 143 + // A domain was frozen. 144 144 + AuditEvent_FreezeDomain AuditEvent = 3 145 145 + // A domain was thawed. 146 146 + AuditEvent_UnfreezeDomain AuditEvent = 4 147 147 + ) 148 148 + 149 149 + // Enum value maps for AuditEvent. 150 150 + var ( 151 151 + AuditEvent_name = map[int32]string{ 152 152 + 0: "InvalidEvent", 153 153 + 1: "CommitManifest", 154 154 + 2: "DeleteManifest", 155 155 + 3: "FreezeDomain", 156 156 + 4: "UnfreezeDomain", 157 157 + } 158 158 + AuditEvent_value = map[string]int32{ 159 159 + "InvalidEvent": 0, 160 160 + "CommitManifest": 1, 161 161 + "DeleteManifest": 2, 162 162 + "FreezeDomain": 3, 163 163 + "UnfreezeDomain": 4, 164 164 + } 165 165 + ) 166 166 + 167 167 + func (x AuditEvent) Enum() *AuditEvent { 168 168 + p := new(AuditEvent) 169 169 + *p = x 170 170 + return p 171 171 + } 172 172 + 173 173 + func (x AuditEvent) String() string { 174 174 + return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x)) 175 175 + } 176 176 + 177 177 + func (AuditEvent) Descriptor() protoreflect.EnumDescriptor { 178 178 + return file_schema_proto_enumTypes[2].Descriptor() 179 179 + } 180 180 + 181 181 + func (AuditEvent) Type() protoreflect.EnumType { 182 182 + return &file_schema_proto_enumTypes[2] 183 183 + } 184 184 + 185 185 + func (x AuditEvent) Number() protoreflect.EnumNumber { 186 186 + return protoreflect.EnumNumber(x) 187 187 + } 188 188 + 189 189 + // Deprecated: Use AuditEvent.Descriptor instead. 190 190 + func (AuditEvent) EnumDescriptor() ([]byte, []int) { 191 191 + return file_schema_proto_rawDescGZIP(), []int{2} 192 192 + } 193 193 + 132 194 type Entry struct { 133 195 state protoimpl.MessageState `protogen:"open.v1"` 134 196 Type *Type `protobuf:"varint,1,opt,name=type,enum=Type" json:"type,omitempty"` 135 197 // Only present for `type == InlineFile` and `type == ExternalFile`. 136 136 - // For transformed entries, refers to the post-transformation (compressed) size. 137 137 - Size *int64 `protobuf:"varint,2,opt,name=size" json:"size,omitempty"` 198 198 + // For transformed entries, refers to the pre-transformation (decompressed) size; otherwise 199 199 + // equal to `compressed_size`. 200 200 + OriginalSize *int64 `protobuf:"varint,7,opt,name=original_size,json=originalSize" json:"original_size,omitempty"` 201 201 + // Only present for `type == InlineFile` and `type == ExternalFile`. 202 202 + // For transformed entries, refers to the post-transformation (compressed) size; otherwise 203 203 + // equal to `original_size`. 204 204 + CompressedSize *int64 `protobuf:"varint,2,opt,name=compressed_size,json=compressedSize" json:"compressed_size,omitempty"` 138 205 // Meaning depends on `type`: 139 206 // - If `type == InlineFile`, contains file data. 140 207 // - If `type == ExternalFile`, contains blob name (an otherwise unspecified ··· 147 214 Transform *Transform `protobuf:"varint,4,opt,name=transform,enum=Transform" json:"transform,omitempty"` 148 215 // Only present for `type == InlineFile` and `type == ExternalFile`. 149 216 // Currently, optional (not present on certain legacy manifests). 150 150 - ContentType *string `protobuf:"bytes,5,opt,name=content_type,json=contentType" json:"content_type,omitempty"` 217 217 + ContentType *string `protobuf:"bytes,5,opt,name=content_type,json=contentType" json:"content_type,omitempty"` 218 218 + // May be present for `type == InlineFile` and `type == ExternalFile`. 219 219 + // Used to reduce the amount of work being done during git checkouts. 220 220 + // The type of hash used is determined by the length: 221 221 + // - 40 bytes: SHA1DC (as hex) 222 222 + // - 64 bytes: SHA256 (as hex) 223 223 + GitHash *string `protobuf:"bytes,6,opt,name=git_hash,json=gitHash" json:"git_hash,omitempty"` 151 224 unknownFields protoimpl.UnknownFields 152 225 sizeCache protoimpl.SizeCache 153 226 } ··· 186 259 if x != nil && x.Type != nil { 187 260 return *x.Type 188 261 } 189 189 - return Type_Invalid 262 262 + return Type_InvalidEntry 190 263 } 191 264 192 192 - func (x *Entry) GetSize() int64 { 193 193 - if x != nil && x.Size != nil { 194 194 - return *x.Size 265 265 + func (x *Entry) GetOriginalSize() int64 { 266 266 + if x != nil && x.OriginalSize != nil { 267 267 + return *x.OriginalSize 268 268 + } 269 269 + return 0 270 270 + } 271 271 + 272 272 + func (x *Entry) GetCompressedSize() int64 { 273 273 + if x != nil && x.CompressedSize != nil { 274 274 + return *x.CompressedSize 195 275 } 196 276 return 0 197 277 } ··· 207 287 if x != nil && x.Transform != nil { 208 288 return *x.Transform 209 289 } 210 210 - return Transform_None 290 290 + return Transform_Identity 211 291 } 212 292 213 293 func (x *Entry) GetContentType() string { 214 294 if x != nil && x.ContentType != nil { 215 295 return *x.ContentType 296 296 + } 297 297 + return "" 298 298 + } 299 299 + 300 300 + func (x *Entry) GetGitHash() string { 301 301 + if x != nil && x.GitHash != nil { 302 302 + return *x.GitHash 216 303 } 217 304 return "" 218 305 } ··· 446 533 447 534 type Manifest struct { 448 535 state protoimpl.MessageState `protogen:"open.v1"` 449 449 - // Source metadata 536 536 + // Source metadata. 450 537 RepoUrl *string `protobuf:"bytes,1,opt,name=repo_url,json=repoUrl" json:"repo_url,omitempty"` 451 538 Branch *string `protobuf:"bytes,2,opt,name=branch" json:"branch,omitempty"` 452 539 Commit *string `protobuf:"bytes,3,opt,name=commit" json:"commit,omitempty"` 453 453 - // Contents 540 540 + // Site contents. 454 541 Contents map[string]*Entry `protobuf:"bytes,4,rep,name=contents" json:"contents,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"` 455 455 - OriginalSize *int64 `protobuf:"varint,10,opt,name=original_size,json=originalSize" json:"original_size,omitempty"` // total size of entries before compression 456 456 - CompressedSize *int64 `protobuf:"varint,5,opt,name=compressed_size,json=compressedSize" json:"compressed_size,omitempty"` // simple sum of each `entry.size` 457 457 - StoredSize *int64 `protobuf:"varint,8,opt,name=stored_size,json=storedSize" json:"stored_size,omitempty"` // total size of (deduplicated) external objects 458 458 - // Netlify-style `_redirects` and `_headers` 542 542 + OriginalSize *int64 `protobuf:"varint,10,opt,name=original_size,json=originalSize" json:"original_size,omitempty"` // sum of each `entry.original_size` 543 543 + CompressedSize *int64 `protobuf:"varint,5,opt,name=compressed_size,json=compressedSize" json:"compressed_size,omitempty"` // sum of each `entry.compressed_size` 544 544 + StoredSize *int64 `protobuf:"varint,8,opt,name=stored_size,json=storedSize" json:"stored_size,omitempty"` // sum of deduplicated `entry.compressed_size` for external files only 545 545 + // Netlify-style `_redirects` and `_headers` rules. 459 546 Redirects []*RedirectRule `protobuf:"bytes,6,rep,name=redirects" json:"redirects,omitempty"` 460 547 Headers []*HeaderRule `protobuf:"bytes,9,rep,name=headers" json:"headers,omitempty"` 461 461 - // Diagnostics for non-fatal errors 548 548 + // Diagnostics for non-fatal errors. 462 549 Problems []*Problem `protobuf:"bytes,7,rep,name=problems" json:"problems,omitempty"` 463 550 unknownFields protoimpl.UnknownFields 464 551 sizeCache protoimpl.SizeCache ··· 564 651 return nil 565 652 } 566 653 654 654 + type AuditRecord struct { 655 655 + state protoimpl.MessageState `protogen:"open.v1"` 656 656 + // Audit event metadata. 657 657 + Id *int64 `protobuf:"varint,1,opt,name=id" json:"id,omitempty"` 658 658 + Timestamp *timestamppb.Timestamp `protobuf:"bytes,2,opt,name=timestamp" json:"timestamp,omitempty"` 659 659 + Event *AuditEvent `protobuf:"varint,3,opt,name=event,enum=AuditEvent" json:"event,omitempty"` 660 660 + Principal *Principal `protobuf:"bytes,4,opt,name=principal" json:"principal,omitempty"` 661 661 + // Affected resource. 662 662 + Domain *string `protobuf:"bytes,10,opt,name=domain" json:"domain,omitempty"` 663 663 + Project *string `protobuf:"bytes,11,opt,name=project" json:"project,omitempty"` // only for `*Manifest` events 664 664 + // Snapshot of site manifest. 665 665 + Manifest *Manifest `protobuf:"bytes,12,opt,name=manifest" json:"manifest,omitempty"` // only for `*Manifest` events 666 666 + unknownFields protoimpl.UnknownFields 667 667 + sizeCache protoimpl.SizeCache 668 668 + } 669 669 + 670 670 + func (x *AuditRecord) Reset() { 671 671 + *x = AuditRecord{} 672 672 + mi := &file_schema_proto_msgTypes[6] 673 673 + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) 674 674 + ms.StoreMessageInfo(mi) 675 675 + } 676 676 + 677 677 + func (x *AuditRecord) String() string { 678 678 + return protoimpl.X.MessageStringOf(x) 679 679 + } 680 680 + 681 681 + func (*AuditRecord) ProtoMessage() {} 682 682 + 683 683 + func (x *AuditRecord) ProtoReflect() protoreflect.Message { 684 684 + mi := &file_schema_proto_msgTypes[6] 685 685 + if x != nil { 686 686 + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) 687 687 + if ms.LoadMessageInfo() == nil { 688 688 + ms.StoreMessageInfo(mi) 689 689 + } 690 690 + return ms 691 691 + } 692 692 + return mi.MessageOf(x) 693 693 + } 694 694 + 695 695 + // Deprecated: Use AuditRecord.ProtoReflect.Descriptor instead. 696 696 + func (*AuditRecord) Descriptor() ([]byte, []int) { 697 697 + return file_schema_proto_rawDescGZIP(), []int{6} 698 698 + } 699 699 + 700 700 + func (x *AuditRecord) GetId() int64 { 701 701 + if x != nil && x.Id != nil { 702 702 + return *x.Id 703 703 + } 704 704 + return 0 705 705 + } 706 706 + 707 707 + func (x *AuditRecord) GetTimestamp() *timestamppb.Timestamp { 708 708 + if x != nil { 709 709 + return x.Timestamp 710 710 + } 711 711 + return nil 712 712 + } 713 713 + 714 714 + func (x *AuditRecord) GetEvent() AuditEvent { 715 715 + if x != nil && x.Event != nil { 716 716 + return *x.Event 717 717 + } 718 718 + return AuditEvent_InvalidEvent 719 719 + } 720 720 + 721 721 + func (x *AuditRecord) GetPrincipal() *Principal { 722 722 + if x != nil { 723 723 + return x.Principal 724 724 + } 725 725 + return nil 726 726 + } 727 727 + 728 728 + func (x *AuditRecord) GetDomain() string { 729 729 + if x != nil && x.Domain != nil { 730 730 + return *x.Domain 731 731 + } 732 732 + return "" 733 733 + } 734 734 + 735 735 + func (x *AuditRecord) GetProject() string { 736 736 + if x != nil && x.Project != nil { 737 737 + return *x.Project 738 738 + } 739 739 + return "" 740 740 + } 741 741 + 742 742 + func (x *AuditRecord) GetManifest() *Manifest { 743 743 + if x != nil { 744 744 + return x.Manifest 745 745 + } 746 746 + return nil 747 747 + } 748 748 + 749 749 + type Principal struct { 750 750 + state protoimpl.MessageState `protogen:"open.v1"` 751 751 + IpAddress *string `protobuf:"bytes,1,opt,name=ip_address,json=ipAddress" json:"ip_address,omitempty"` 752 752 + CliAdmin *bool `protobuf:"varint,2,opt,name=cli_admin,json=cliAdmin" json:"cli_admin,omitempty"` 753 753 + unknownFields protoimpl.UnknownFields 754 754 + sizeCache protoimpl.SizeCache 755 755 + } 756 756 + 757 757 + func (x *Principal) Reset() { 758 758 + *x = Principal{} 759 759 + mi := &file_schema_proto_msgTypes[7] 760 760 + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) 761 761 + ms.StoreMessageInfo(mi) 762 762 + } 763 763 + 764 764 + func (x *Principal) String() string { 765 765 + return protoimpl.X.MessageStringOf(x) 766 766 + } 767 767 + 768 768 + func (*Principal) ProtoMessage() {} 769 769 + 770 770 + func (x *Principal) ProtoReflect() protoreflect.Message { 771 771 + mi := &file_schema_proto_msgTypes[7] 772 772 + if x != nil { 773 773 + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) 774 774 + if ms.LoadMessageInfo() == nil { 775 775 + ms.StoreMessageInfo(mi) 776 776 + } 777 777 + return ms 778 778 + } 779 779 + return mi.MessageOf(x) 780 780 + } 781 781 + 782 782 + // Deprecated: Use Principal.ProtoReflect.Descriptor instead. 783 783 + func (*Principal) Descriptor() ([]byte, []int) { 784 784 + return file_schema_proto_rawDescGZIP(), []int{7} 785 785 + } 786 786 + 787 787 + func (x *Principal) GetIpAddress() string { 788 788 + if x != nil && x.IpAddress != nil { 789 789 + return *x.IpAddress 790 790 + } 791 791 + return "" 792 792 + } 793 793 + 794 794 + func (x *Principal) GetCliAdmin() bool { 795 795 + if x != nil && x.CliAdmin != nil { 796 796 + return *x.CliAdmin 797 797 + } 798 798 + return false 799 799 + } 800 800 + 567 801 var File_schema_proto protoreflect.FileDescriptor 568 802 569 803 const file_schema_proto_rawDesc = "" + 570 804 "\n" + 571 571 - "\fschema.proto\"\x97\x01\n" + 805 805 + "\fschema.proto\x1a\x1fgoogle/protobuf/timestamp.proto\"\xec\x01\n" + 572 806 "\x05Entry\x12\x19\n" + 573 573 - "\x04type\x18\x01 \x01(\x0e2\x05.TypeR\x04type\x12\x12\n" + 574 574 - "\x04size\x18\x02 \x01(\x03R\x04size\x12\x12\n" + 807 807 + "\x04type\x18\x01 \x01(\x0e2\x05.TypeR\x04type\x12#\n" + 808 808 + "\roriginal_size\x18\a \x01(\x03R\foriginalSize\x12'\n" + 809 809 + "\x0fcompressed_size\x18\x02 \x01(\x03R\x0ecompressedSize\x12\x12\n" + 575 810 "\x04data\x18\x03 \x01(\fR\x04data\x12(\n" + 576 811 "\ttransform\x18\x04 \x01(\x0e2\n" + 577 812 ".TransformR\ttransform\x12!\n" + 578 578 - "\fcontent_type\x18\x05 \x01(\tR\vcontentType\"`\n" + 813 813 + "\fcontent_type\x18\x05 \x01(\tR\vcontentType\x12\x19\n" + 814 814 + "\bgit_hash\x18\x06 \x01(\tR\agitHash\"`\n" + 579 815 "\fRedirectRule\x12\x12\n" + 580 816 "\x04from\x18\x01 \x01(\tR\x04from\x12\x0e\n" + 581 817 "\x02to\x18\x02 \x01(\tR\x02to\x12\x16\n" + ··· 607 843 "\bproblems\x18\a \x03(\v2\b.ProblemR\bproblems\x1aC\n" + 608 844 "\rContentsEntry\x12\x10\n" + 609 845 "\x03key\x18\x01 \x01(\tR\x03key\x12\x1c\n" + 610 610 - "\x05value\x18\x02 \x01(\v2\x06.EntryR\x05value:\x028\x01*Q\n" + 611 611 - "\x04Type\x12\v\n" + 612 612 - "\aInvalid\x10\x00\x12\r\n" + 846 846 + "\x05value\x18\x02 \x01(\v2\x06.EntryR\x05value:\x028\x01\"\xfd\x01\n" + 847 847 + "\vAuditRecord\x12\x0e\n" + 848 848 + "\x02id\x18\x01 \x01(\x03R\x02id\x128\n" + 849 849 + "\ttimestamp\x18\x02 \x01(\v2\x1a.google.protobuf.TimestampR\ttimestamp\x12!\n" + 850 850 + "\x05event\x18\x03 \x01(\x0e2\v.AuditEventR\x05event\x12(\n" + 851 851 + "\tprincipal\x18\x04 \x01(\v2\n" + 852 852 + ".PrincipalR\tprincipal\x12\x16\n" + 853 853 + "\x06domain\x18\n" + 854 854 + " \x01(\tR\x06domain\x12\x18\n" + 855 855 + "\aproject\x18\v \x01(\tR\aproject\x12%\n" + 856 856 + "\bmanifest\x18\f \x01(\v2\t.ManifestR\bmanifest\"G\n" + 857 857 + "\tPrincipal\x12\x1d\n" + 858 858 + "\n" + 859 859 + "ip_address\x18\x01 \x01(\tR\tipAddress\x12\x1b\n" + 860 860 + "\tcli_admin\x18\x02 \x01(\bR\bcliAdmin*V\n" + 861 861 + "\x04Type\x12\x10\n" + 862 862 + "\fInvalidEntry\x10\x00\x12\r\n" + 613 863 "\tDirectory\x10\x01\x12\x0e\n" + 614 864 "\n" + 615 865 "InlineFile\x10\x02\x12\x10\n" + 616 866 "\fExternalFile\x10\x03\x12\v\n" + 617 617 - "\aSymlink\x10\x04*$\n" + 618 618 - "\tTransform\x12\b\n" + 619 619 - "\x04None\x10\x00\x12\r\n" + 620 620 - "\tZstandard\x10\x01B,Z*codeberg.org/git-pages/git-pages/git_pagesb\beditionsp\xe8\a" 867 867 + "\aSymlink\x10\x04*#\n" + 868 868 + "\tTransform\x12\f\n" + 869 869 + "\bIdentity\x10\x00\x12\b\n" + 870 870 + "\x04Zstd\x10\x01*l\n" + 871 871 + "\n" + 872 872 + "AuditEvent\x12\x10\n" + 873 873 + "\fInvalidEvent\x10\x00\x12\x12\n" + 874 874 + "\x0eCommitManifest\x10\x01\x12\x12\n" + 875 875 + "\x0eDeleteManifest\x10\x02\x12\x10\n" + 876 876 + "\fFreezeDomain\x10\x03\x12\x12\n" + 877 877 + "\x0eUnfreezeDomain\x10\x04B,Z*codeberg.org/git-pages/git-pages/git_pagesb\beditionsp\xe8\a" 621 878 622 879 var ( 623 880 file_schema_proto_rawDescOnce sync.Once ··· 631 888 return file_schema_proto_rawDescData 632 889 } 633 890 634 634 - var file_schema_proto_enumTypes = make([]protoimpl.EnumInfo, 2) 635 635 - var file_schema_proto_msgTypes = make([]protoimpl.MessageInfo, 7) 891 891 + var file_schema_proto_enumTypes = make([]protoimpl.EnumInfo, 3) 892 892 + var file_schema_proto_msgTypes = make([]protoimpl.MessageInfo, 9) 636 893 var file_schema_proto_goTypes = []any{ 637 637 - (Type)(0), // 0: Type 638 638 - (Transform)(0), // 1: Transform 639 639 - (*Entry)(nil), // 2: Entry 640 640 - (*RedirectRule)(nil), // 3: RedirectRule 641 641 - (*Header)(nil), // 4: Header 642 642 - (*HeaderRule)(nil), // 5: HeaderRule 643 643 - (*Problem)(nil), // 6: Problem 644 644 - (*Manifest)(nil), // 7: Manifest 645 645 - nil, // 8: Manifest.ContentsEntry 894 894 + (Type)(0), // 0: Type 895 895 + (Transform)(0), // 1: Transform 896 896 + (AuditEvent)(0), // 2: AuditEvent 897 897 + (*Entry)(nil), // 3: Entry 898 898 + (*RedirectRule)(nil), // 4: RedirectRule 899 899 + (*Header)(nil), // 5: Header 900 900 + (*HeaderRule)(nil), // 6: HeaderRule 901 901 + (*Problem)(nil), // 7: Problem 902 902 + (*Manifest)(nil), // 8: Manifest 903 903 + (*AuditRecord)(nil), // 9: AuditRecord 904 904 + (*Principal)(nil), // 10: Principal 905 905 + nil, // 11: Manifest.ContentsEntry 906 906 + (*timestamppb.Timestamp)(nil), // 12: google.protobuf.Timestamp 646 907 } 647 908 var file_schema_proto_depIdxs = []int32{ 648 648 - 0, // 0: Entry.type:type_name -> Type 649 649 - 1, // 1: Entry.transform:type_name -> Transform 650 650 - 4, // 2: HeaderRule.header_map:type_name -> Header 651 651 - 8, // 3: Manifest.contents:type_name -> Manifest.ContentsEntry 652 652 - 3, // 4: Manifest.redirects:type_name -> RedirectRule 653 653 - 5, // 5: Manifest.headers:type_name -> HeaderRule 654 654 - 6, // 6: Manifest.problems:type_name -> Problem 655 655 - 2, // 7: Manifest.ContentsEntry.value:type_name -> Entry 656 656 - 8, // [8:8] is the sub-list for method output_type 657 657 - 8, // [8:8] is the sub-list for method input_type 658 658 - 8, // [8:8] is the sub-list for extension type_name 659 659 - 8, // [8:8] is the sub-list for extension extendee 660 660 - 0, // [0:8] is the sub-list for field type_name 909 909 + 0, // 0: Entry.type:type_name -> Type 910 910 + 1, // 1: Entry.transform:type_name -> Transform 911 911 + 5, // 2: HeaderRule.header_map:type_name -> Header 912 912 + 11, // 3: Manifest.contents:type_name -> Manifest.ContentsEntry 913 913 + 4, // 4: Manifest.redirects:type_name -> RedirectRule 914 914 + 6, // 5: Manifest.headers:type_name -> HeaderRule 915 915 + 7, // 6: Manifest.problems:type_name -> Problem 916 916 + 12, // 7: AuditRecord.timestamp:type_name -> google.protobuf.Timestamp 917 917 + 2, // 8: AuditRecord.event:type_name -> AuditEvent 918 918 + 10, // 9: AuditRecord.principal:type_name -> Principal 919 919 + 8, // 10: AuditRecord.manifest:type_name -> Manifest 920 920 + 3, // 11: Manifest.ContentsEntry.value:type_name -> Entry 921 921 + 12, // [12:12] is the sub-list for method output_type 922 922 + 12, // [12:12] is the sub-list for method input_type 923 923 + 12, // [12:12] is the sub-list for extension type_name 924 924 + 12, // [12:12] is the sub-list for extension extendee 925 925 + 0, // [0:12] is the sub-list for field type_name 661 926 } 662 927 663 928 func init() { file_schema_proto_init() } ··· 670 935 File: protoimpl.DescBuilder{ 671 936 GoPackagePath: reflect.TypeOf(x{}).PkgPath(), 672 937 RawDescriptor: unsafe.Slice(unsafe.StringData(file_schema_proto_rawDesc), len(file_schema_proto_rawDesc)), 673 673 - NumEnums: 2, 674 674 - NumMessages: 7, 938 938 + NumEnums: 3, 939 939 + NumMessages: 9, 675 940 NumExtensions: 0, 676 941 NumServices: 0, 677 942 },

+59 -12

src/schema.proto

··· 2 2 3 3 option go_package = "codeberg.org/git-pages/git-pages/git_pages"; 4 4 5 5 + import "google/protobuf/timestamp.proto"; 6 6 + 5 7 enum Type { 6 8 // Invalid entry. 7 7 - Invalid = 0; 9 9 + InvalidEntry = 0; 8 10 // Directory. 9 11 Directory = 1; 10 12 // Inline file. `Blob.Data` contains file contents. ··· 15 17 Symlink = 4; 16 18 } 17 19 20 20 + // Transformation names should match HTTP `Accept-Encoding:` header. 18 21 enum Transform { 19 22 // No transformation. 20 20 - None = 0; 23 23 + Identity = 0; 21 24 // Zstandard compression. 22 22 - Zstandard = 1; 25 25 + Zstd = 1; 23 26 } 24 27 25 28 message Entry { 26 29 Type type = 1; 27 30 // Only present for `type == InlineFile` and `type == ExternalFile`. 28 28 - // For transformed entries, refers to the post-transformation (compressed) size. 29 29 - int64 size = 2; 31 31 + // For transformed entries, refers to the pre-transformation (decompressed) size; otherwise 32 32 + // equal to `compressed_size`. 33 33 + int64 original_size = 7; 34 34 + // Only present for `type == InlineFile` and `type == ExternalFile`. 35 35 + // For transformed entries, refers to the post-transformation (compressed) size; otherwise 36 36 + // equal to `original_size`. 37 37 + int64 compressed_size = 2; 30 38 // Meaning depends on `type`: 31 39 // * If `type == InlineFile`, contains file data. 32 40 // * If `type == ExternalFile`, contains blob name (an otherwise unspecified ··· 40 48 // Only present for `type == InlineFile` and `type == ExternalFile`. 41 49 // Currently, optional (not present on certain legacy manifests). 42 50 string content_type = 5; 51 51 + // May be present for `type == InlineFile` and `type == ExternalFile`. 52 52 + // Used to reduce the amount of work being done during git checkouts. 53 53 + // The type of hash used is determined by the length: 54 54 + // * 40 bytes: SHA1DC (as hex) 55 55 + // * 64 bytes: SHA256 (as hex) 56 56 + string git_hash = 6; 43 57 } 44 58 45 59 // See https://docs.netlify.com/manage/routing/redirects/overview/ for details. ··· 68 82 } 69 83 70 84 message Manifest { 71 71 - // Source metadata 85 85 + // Source metadata. 72 86 string repo_url = 1; 73 87 string branch = 2; 74 88 string commit = 3; 75 89 76 76 - // Contents 90 90 + // Site contents. 77 91 map<string, Entry> contents = 4; 78 78 - int64 original_size = 10; // total size of entries before compression 79 79 - int64 compressed_size = 5; // simple sum of each `entry.size` 80 80 - int64 stored_size = 8; // total size of (deduplicated) external objects 92 92 + int64 original_size = 10; // sum of each `entry.original_size` 93 93 + int64 compressed_size = 5; // sum of each `entry.compressed_size` 94 94 + int64 stored_size = 8; // sum of deduplicated `entry.compressed_size` for external files only 81 95 82 82 - // Netlify-style `_redirects` and `_headers` 96 96 + // Netlify-style `_redirects` and `_headers` rules. 83 97 repeated RedirectRule redirects = 6; 84 98 repeated HeaderRule headers = 9; 85 99 86 86 - // Diagnostics for non-fatal errors 100 100 + // Diagnostics for non-fatal errors. 87 101 repeated Problem problems = 7; 88 102 } 103 103 + 104 104 + enum AuditEvent { 105 105 + // Invalid event. 106 106 + InvalidEvent = 0; 107 107 + // A manifest was committed (a site was created or updated). 108 108 + CommitManifest = 1; 109 109 + // A manifest was deleted (a site was deleted). 110 110 + DeleteManifest = 2; 111 111 + // A domain was frozen. 112 112 + FreezeDomain = 3; 113 113 + // A domain was thawed. 114 114 + UnfreezeDomain = 4; 115 115 + } 116 116 + 117 117 + message AuditRecord { 118 118 + // Audit event metadata. 119 119 + int64 id = 1; 120 120 + google.protobuf.Timestamp timestamp = 2; 121 121 + AuditEvent event = 3; 122 122 + Principal principal = 4; 123 123 + 124 124 + // Affected resource. 125 125 + string domain = 10; 126 126 + string project = 11; // only for `*Manifest` events 127 127 + 128 128 + // Snapshot of site manifest. 129 129 + Manifest manifest = 12; // only for `*Manifest` events 130 130 + } 131 131 + 132 132 + message Principal { 133 133 + string ip_address = 1; 134 134 + bool cli_admin = 2; 135 135 + }

+29

src/signal.go

··· 1 1 + // See https://pkg.go.dev/os/signal#hdr-Windows for a description of what this module 2 2 + // will do on Windows (tl;dr nothing calls the reload handler, the interrupt handler works 3 3 + // more or less how you'd expect). 4 4 + 5 5 + package git_pages 6 6 + 7 7 + import ( 8 8 + "os" 9 9 + "os/signal" 10 10 + "syscall" 11 11 + ) 12 12 + 13 13 + func OnReload(handler func()) { 14 14 + sighup := make(chan os.Signal, 1) 15 15 + signal.Notify(sighup, syscall.SIGHUP) 16 16 + go func() { 17 17 + for { 18 18 + <-sighup 19 19 + handler() 20 20 + } 21 21 + }() 22 22 + } 23 23 + 24 24 + func WaitForInterrupt() { 25 25 + sigint := make(chan os.Signal, 1) 26 26 + signal.Notify(sigint, syscall.SIGINT, syscall.SIGTERM) 27 27 + <-sigint 28 28 + signal.Stop(sigint) 29 29 + }

-7

src/signal_other.go

··· 1 1 - //go:build !unix 2 2 - 3 3 - package git_pages 4 4 - 5 5 - func OnReload(handler func()) { 6 6 - // not implemented 7 7 - }

-20

src/signal_posix.go

··· 1 1 - //go:build unix 2 2 - 3 3 - package git_pages 4 4 - 5 5 - import ( 6 6 - "os" 7 7 - "os/signal" 8 8 - "syscall" 9 9 - ) 10 10 - 11 11 - func OnReload(handler func()) { 12 12 - sighup := make(chan os.Signal, 1) 13 13 - signal.Notify(sighup, syscall.SIGHUP) 14 14 - go func() { 15 15 - for { 16 16 - <-sighup 17 17 - handler() 18 18 - } 19 19 - }() 20 20 - }

+117 -29

src/update.go

··· 5 5 "errors" 6 6 "fmt" 7 7 "io" 8 8 - "log" 9 8 "strings" 9 9 + 10 10 + "google.golang.org/protobuf/proto" 10 11 ) 11 12 12 13 type UpdateOutcome int ··· 26 27 err error 27 28 } 28 29 29 29 - func Update(ctx context.Context, webRoot string, manifest *Manifest) UpdateResult { 30 30 - var oldManifest, newManifest *Manifest 30 30 + func Update( 31 31 + ctx context.Context, webRoot string, oldManifest, newManifest *Manifest, 32 32 + opts ModifyManifestOptions, 33 33 + ) UpdateResult { 31 34 var err error 35 35 + var storedManifest *Manifest 32 36 33 37 outcome := UpdateError 34 34 - oldManifest, _, _ = backend.GetManifest(ctx, webRoot, GetManifestOptions{}) 35 35 - if IsManifestEmpty(manifest) { 36 36 - newManifest, err = manifest, backend.DeleteManifest(ctx, webRoot) 38 38 + if IsManifestEmpty(newManifest) { 39 39 + storedManifest, err = newManifest, backend.DeleteManifest(ctx, webRoot, opts) 37 40 if err == nil { 38 41 if oldManifest == nil { 39 42 outcome = UpdateNoChange ··· 41 44 outcome = UpdateDeleted 42 45 } 43 46 } 44 44 - } else if err = PrepareManifest(ctx, manifest); err == nil { 45 45 - newManifest, err = StoreManifest(ctx, webRoot, manifest) 47 47 + } else if err = PrepareManifest(ctx, newManifest); err == nil { 48 48 + storedManifest, err = StoreManifest(ctx, webRoot, newManifest, opts) 46 49 if err == nil { 47 50 domain, _, _ := strings.Cut(webRoot, "/") 48 51 err = backend.CreateDomain(ctx, domain) ··· 50 53 if err == nil { 51 54 if oldManifest == nil { 52 55 outcome = UpdateCreated 53 53 - } else if CompareManifest(oldManifest, newManifest) { 56 56 + } else if CompareManifest(oldManifest, storedManifest) { 54 57 outcome = UpdateNoChange 55 58 } else { 56 59 outcome = UpdateReplaced ··· 70 73 case UpdateNoChange: 71 74 status = "unchanged" 72 75 } 73 73 - if newManifest.Commit != nil { 74 74 - log.Printf("update %s ok: %s %s", webRoot, status, *newManifest.Commit) 76 76 + if storedManifest.Commit != nil { 77 77 + logc.Printf(ctx, "update %s ok: %s %s", webRoot, *storedManifest.Commit, status) 75 78 } else { 76 76 - log.Printf("update %s ok: %s", webRoot, status) 79 79 + logc.Printf(ctx, "update %s ok: %s", webRoot, status) 77 80 } 78 81 } else { 79 79 - log.Printf("update %s err: %s", webRoot, err) 82 82 + logc.Printf(ctx, "update %s err: %s", webRoot, err) 80 83 } 81 84 82 82 - return UpdateResult{outcome, newManifest, err} 85 85 + return UpdateResult{outcome, storedManifest, err} 83 86 } 84 87 85 88 func UpdateFromRepository( ··· 91 94 span, ctx := ObserveFunction(ctx, "UpdateFromRepository", "repo.url", repoURL) 92 95 defer span.Finish() 93 96 94 94 - log.Printf("update %s: %s %s\n", webRoot, repoURL, branch) 97 97 + logc.Printf(ctx, "update %s: %s %s\n", webRoot, repoURL, branch) 95 98 96 96 - manifest, err := FetchRepository(ctx, repoURL, branch) 99 99 + // Ignore errors; worst case we have to re-fetch all of the blobs. 100 100 + oldManifest, _, _ := backend.GetManifest(ctx, webRoot, GetManifestOptions{}) 101 101 + 102 102 + newManifest, err := FetchRepository(ctx, repoURL, branch, oldManifest) 97 103 if errors.Is(err, context.DeadlineExceeded) { 98 104 result = UpdateResult{UpdateTimeout, nil, fmt.Errorf("update timeout")} 99 105 } else if err != nil { 100 106 result = UpdateResult{UpdateError, nil, err} 101 107 } else { 102 102 - result = Update(ctx, webRoot, manifest) 108 108 + result = Update(ctx, webRoot, oldManifest, newManifest, ModifyManifestOptions{}) 103 109 } 104 110 105 111 observeUpdateResult(result) ··· 114 120 contentType string, 115 121 reader io.Reader, 116 122 ) (result UpdateResult) { 117 117 - var manifest *Manifest 118 123 var err error 119 124 125 125 + // Ignore errors; worst case we have to re-fetch all of the blobs. 126 126 + oldManifest, _, _ := backend.GetManifest(ctx, webRoot, GetManifestOptions{}) 127 127 + 128 128 + extractTar := func(ctx context.Context, reader io.Reader) (*Manifest, error) { 129 129 + return ExtractTar(ctx, reader, oldManifest) 130 130 + } 131 131 + 132 132 + var newManifest *Manifest 120 133 switch contentType { 121 134 case "application/x-tar": 122 122 - log.Printf("update %s: (tar)", webRoot) 123 123 - manifest, err = ExtractTar(reader) // yellow? 135 135 + logc.Printf(ctx, "update %s: (tar)", webRoot) 136 136 + newManifest, err = extractTar(ctx, reader) // yellow? 124 137 case "application/x-tar+gzip": 125 125 - log.Printf("update %s: (tar.gz)", webRoot) 126 126 - manifest, err = ExtractTarGzip(reader) // definitely yellow. 138 138 + logc.Printf(ctx, "update %s: (tar.gz)", webRoot) 139 139 + newManifest, err = ExtractGzip(ctx, reader, extractTar) // definitely yellow. 127 140 case "application/x-tar+zstd": 128 128 - log.Printf("update %s: (tar.zst)", webRoot) 129 129 - manifest, err = ExtractTarZstd(reader) 141 141 + logc.Printf(ctx, "update %s: (tar.zst)", webRoot) 142 142 + newManifest, err = ExtractZstd(ctx, reader, extractTar) 130 143 case "application/zip": 131 131 - log.Printf("update %s: (zip)", webRoot) 132 132 - manifest, err = ExtractZip(reader) 144 144 + logc.Printf(ctx, "update %s: (zip)", webRoot) 145 145 + newManifest, err = ExtractZip(ctx, reader, oldManifest) 133 146 default: 134 147 err = errArchiveFormat 135 148 } 136 149 137 150 if err != nil { 138 138 - log.Printf("update %s err: %s", webRoot, err) 151 151 + logc.Printf(ctx, "update %s err: %s", webRoot, err) 139 152 result = UpdateResult{UpdateError, nil, err} 140 153 } else { 141 141 - result = Update(ctx, webRoot, manifest) 154 154 + result = Update(ctx, webRoot, oldManifest, newManifest, ModifyManifestOptions{}) 155 155 + } 156 156 + 157 157 + observeUpdateResult(result) 158 158 + return 159 159 + } 160 160 + 161 161 + func PartialUpdateFromArchive( 162 162 + ctx context.Context, 163 163 + webRoot string, 164 164 + contentType string, 165 165 + reader io.Reader, 166 166 + parents CreateParentsMode, 167 167 + ) (result UpdateResult) { 168 168 + var err error 169 169 + 170 170 + // Here the old manifest is used both as a substrate to which a patch is applied, as well 171 171 + // as a "load linked" operation for a future "store conditional" update which, taken together, 172 172 + // create an atomic compare-and-swap operation. 173 173 + oldManifest, oldMetadata, err := backend.GetManifest(ctx, webRoot, 174 174 + GetManifestOptions{BypassCache: true}) 175 175 + if err != nil { 176 176 + logc.Printf(ctx, "patch %s err: %s", webRoot, err) 177 177 + return UpdateResult{UpdateError, nil, err} 178 178 + } 179 179 + 180 180 + applyTarPatch := func(ctx context.Context, reader io.Reader) (*Manifest, error) { 181 181 + // Clone the manifest before starting to mutate it. `GetManifest` may return cached 182 182 + // `*Manifest` objects, which should never be mutated. 183 183 + newManifest := &Manifest{} 184 184 + proto.Merge(newManifest, oldManifest) 185 185 + newManifest.RepoUrl = nil 186 186 + newManifest.Branch = nil 187 187 + newManifest.Commit = nil 188 188 + if err := ApplyTarPatch(newManifest, reader, parents); err != nil { 189 189 + return nil, err 190 190 + } else { 191 191 + return newManifest, nil 192 192 + } 193 193 + } 194 194 + 195 195 + var newManifest *Manifest 196 196 + switch contentType { 197 197 + case "application/x-tar": 198 198 + logc.Printf(ctx, "patch %s: (tar)", webRoot) 199 199 + newManifest, err = applyTarPatch(ctx, reader) 200 200 + case "application/x-tar+gzip": 201 201 + logc.Printf(ctx, "patch %s: (tar.gz)", webRoot) 202 202 + newManifest, err = ExtractGzip(ctx, reader, applyTarPatch) 203 203 + case "application/x-tar+zstd": 204 204 + logc.Printf(ctx, "patch %s: (tar.zst)", webRoot) 205 205 + newManifest, err = ExtractZstd(ctx, reader, applyTarPatch) 206 206 + default: 207 207 + err = errArchiveFormat 208 208 + } 209 209 + 210 210 + if err != nil { 211 211 + logc.Printf(ctx, "patch %s err: %s", webRoot, err) 212 212 + result = UpdateResult{UpdateError, nil, err} 213 213 + } else { 214 214 + result = Update(ctx, webRoot, oldManifest, newManifest, 215 215 + ModifyManifestOptions{ 216 216 + IfUnmodifiedSince: oldMetadata.LastModified, 217 217 + IfMatch: oldMetadata.ETag, 218 218 + }) 219 219 + // The `If-Unmodified-Since` precondition is internally generated here, which means its 220 220 + // failure shouldn't be surfaced as-is in the HTTP response. If we also accepted options 221 221 + // from the client, then that precondition failure should surface in the response. 222 222 + if errors.Is(result.err, ErrPreconditionFailed) { 223 223 + result.err = ErrWriteConflict 224 224 + } 142 225 } 143 226 144 227 observeUpdateResult(result) ··· 146 229 } 147 230 148 231 func observeUpdateResult(result UpdateResult) { 149 149 - if result.err != nil { 232 232 + var unresolvedRefErr UnresolvedRefError 233 233 + if errors.As(result.err, &unresolvedRefErr) { 234 234 + // This error is an expected outcome of an incremental update's probe phase. 235 235 + } else if errors.Is(result.err, ErrWriteConflict) { 236 236 + // This error is an expected outcome of an incremental update losing a race. 237 237 + } else if result.err != nil { 150 238 ObserveError(result.err) 151 239 } 152 240 }

-55

src/wildcard.go

··· 1 1 package git_pages 2 2 3 3 import ( 4 4 - "crypto/tls" 5 4 "fmt" 6 6 - "log" 7 7 - "net/http" 8 8 - "net/http/httputil" 9 9 - "net/url" 10 5 "slices" 11 6 "strings" 12 7 ··· 19 14 IndexRepos []*fasttemplate.Template 20 15 IndexBranch string 21 16 Authorization bool 22 22 - FallbackURL *url.URL 23 23 - Fallback http.Handler 24 17 } 25 18 26 19 func (pattern *WildcardPattern) GetHost() string { ··· 79 72 return repoURLs, branch 80 73 } 81 74 82 82 - func (pattern *WildcardPattern) IsFallbackFor(host string) bool { 83 83 - if pattern.Fallback == nil { 84 84 - return false 85 85 - } 86 86 - _, found := pattern.Matches(host) 87 87 - return found 88 88 - } 89 89 - 90 90 - func HandleWildcardFallback(w http.ResponseWriter, r *http.Request) (bool, error) { 91 91 - host, err := GetHost(r) 92 92 - if err != nil { 93 93 - return false, err 94 94 - } 95 95 - 96 96 - for _, pattern := range wildcards { 97 97 - if pattern.IsFallbackFor(host) { 98 98 - log.Printf("proxy: %s via %s", pattern.GetHost(), pattern.FallbackURL) 99 99 - pattern.Fallback.ServeHTTP(w, r) 100 100 - return true, nil 101 101 - } 102 102 - } 103 103 - return false, nil 104 104 - } 105 105 - 106 75 func TranslateWildcards(configs []WildcardConfig) ([]*WildcardPattern, error) { 107 76 var wildcardPatterns []*WildcardPattern 108 77 for _, config := range configs { ··· 135 104 } 136 105 } 137 106 138 138 - var fallbackURL *url.URL 139 139 - var fallback http.Handler 140 140 - if config.FallbackProxyTo != "" { 141 141 - fallbackURL, err = url.Parse(config.FallbackProxyTo) 142 142 - if err != nil { 143 143 - return nil, fmt.Errorf("wildcard pattern: fallback URL: %w", err) 144 144 - } 145 145 - 146 146 - fallback = &httputil.ReverseProxy{ 147 147 - Rewrite: func(r *httputil.ProxyRequest) { 148 148 - r.SetURL(fallbackURL) 149 149 - r.Out.Host = r.In.Host 150 150 - r.Out.Header["X-Forwarded-For"] = r.In.Header["X-Forwarded-For"] 151 151 - }, 152 152 - Transport: &http.Transport{ 153 153 - TLSClientConfig: &tls.Config{ 154 154 - InsecureSkipVerify: config.FallbackInsecure, 155 155 - }, 156 156 - }, 157 157 - } 158 158 - } 159 159 - 160 107 wildcardPatterns = append(wildcardPatterns, &WildcardPattern{ 161 108 Domain: strings.Split(config.Domain, "."), 162 109 CloneURL: cloneURLTemplate, 163 110 IndexRepos: indexRepoTemplates, 164 111 IndexBranch: indexRepoBranch, 165 112 Authorization: authorization, 166 166 - FallbackURL: fallbackURL, 167 167 - Fallback: fallback, 168 113 }) 169 114 } 170 115 return wildcardPatterns, nil

+112

test/stresspatch/main.go

··· 1 1 + package main 2 2 + 3 3 + import ( 4 4 + "archive/tar" 5 5 + "bytes" 6 6 + "flag" 7 7 + "fmt" 8 8 + "io" 9 9 + "net/http" 10 10 + "sync" 11 11 + "time" 12 12 + ) 13 13 + 14 14 + func makeInit() []byte { 15 15 + writer := bytes.NewBuffer(nil) 16 16 + archive := tar.NewWriter(writer) 17 17 + archive.WriteHeader(&tar.Header{ 18 18 + Typeflag: tar.TypeReg, 19 19 + Name: "index.html", 20 20 + }) 21 21 + archive.Write([]byte{}) 22 22 + archive.Flush() 23 23 + return writer.Bytes() 24 24 + } 25 25 + 26 26 + func initSite() { 27 27 + req, err := http.NewRequest(http.MethodPut, "http://localhost:3000", 28 28 + bytes.NewReader(makeInit())) 29 29 + if err != nil { 30 30 + panic(err) 31 31 + } 32 32 + 33 33 + req.Header.Add("Content-Type", "application/x-tar") 34 34 + resp, err := http.DefaultClient.Do(req) 35 35 + if err != nil { 36 36 + panic(err) 37 37 + } 38 38 + defer resp.Body.Close() 39 39 + } 40 40 + 41 41 + func makePatch(n int) []byte { 42 42 + writer := bytes.NewBuffer(nil) 43 43 + archive := tar.NewWriter(writer) 44 44 + archive.WriteHeader(&tar.Header{ 45 45 + Typeflag: tar.TypeReg, 46 46 + Name: fmt.Sprintf("%d.txt", n), 47 47 + }) 48 48 + archive.Write([]byte{}) 49 49 + archive.Flush() 50 50 + return writer.Bytes() 51 51 + } 52 52 + 53 53 + func patchRequest(n int) int { 54 54 + req, err := http.NewRequest(http.MethodPatch, "http://localhost:3000", 55 55 + bytes.NewReader(makePatch(n))) 56 56 + if err != nil { 57 57 + panic(err) 58 58 + } 59 59 + 60 60 + req.Header.Add("Atomic", "no") 61 61 + req.Header.Add("Content-Type", "application/x-tar") 62 62 + resp, err := http.DefaultClient.Do(req) 63 63 + if err != nil { 64 64 + panic(err) 65 65 + } 66 66 + defer resp.Body.Close() 67 67 + 68 68 + data, err := io.ReadAll(resp.Body) 69 69 + if err != nil { 70 70 + panic(err) 71 71 + } 72 72 + 73 73 + fmt.Printf("%d: %s %q\n", n, resp.Status, string(data)) 74 74 + return resp.StatusCode 75 75 + } 76 76 + 77 77 + func concurrentWriter(wg *sync.WaitGroup, n int) { 78 78 + for { 79 79 + if patchRequest(n) == 200 { 80 80 + break 81 81 + } 82 82 + } 83 83 + wg.Done() 84 84 + } 85 85 + 86 86 + var count = flag.Int("count", 10, "request count") 87 87 + 88 88 + func main() { 89 89 + flag.Parse() 90 90 + 91 91 + initSite() 92 92 + time.Sleep(time.Second) 93 93 + 94 94 + wg := &sync.WaitGroup{} 95 95 + for n := range *count { 96 96 + wg.Add(1) 97 97 + go concurrentWriter(wg, n) 98 98 + } 99 99 + wg.Wait() 100 100 + 101 101 + success := 0 102 102 + for n := range *count { 103 103 + resp, err := http.Get(fmt.Sprintf("http://localhost:3000/%d.txt", n)) 104 104 + if err != nil { 105 105 + panic(err) 106 106 + } 107 107 + if resp.StatusCode == 200 { 108 108 + success++ 109 109 + } 110 110 + } 111 111 + fmt.Printf("written: %d of %d\n", success, *count) 112 112 + }