Merge pull request #9 from opencourseworld/maint.dev-system-v2

··· 1 1 - # # [jneen] DO NOT EDIT THIS FILE 2 2 - # # instead, override values in a separate private.env 3 3 - # 4 4 - # export APP_PORT=9000 5 5 - # export DB_PORT=5432 6 6 - # export HTTP_PORT=6001 7 7 - # export HTTPS_PORT=6000 8 8 - # # note: must be localhost for login to work 9 9 - # export LISTEN_HOST=localhost 10 10 - # 11 11 - # export ENABLE_API_DEV_USER='true' 12 12 - # export FORCE_HTTP='false' 13 13 - # export DEBUG_LEVEL='light' 14 14 - # export COURSE_IMPORT_REQUIRE_VERIFY='false' 15 15 - # export COURSE_IMPORT_ENABLE_CACHE='true' 16 16 - # export USE_DUMP_FOR_SEARCHES='false' 17 17 - # export DUMP_FILENAME='fake.db' 18 18 - # export MIXED_DIFFICULTY='false' 19 19 - # export JWT_SECRET='localdev' 20 20 - # export DATA_VIEW_TYPE='combined_dyna' 21 21 - # 22 22 - # export TWITCH_CLIENT_ID=ufm5q62qafbnkp3leizjli8z89xo9w 23 23 - # export TWITCH_CLIENT_SECRET="set this in private.env" 24 24 - # export DISCORD_CLIENT_ID=1080395755581349908 25 25 - # export DISCORD_CLIENT_SECRET="set this in private.env" 26 26 - # export ENABLE_TAGS=false 27 27 - # 28 28 - # # load override values 29 29 - # . private.env 30 30 - # 31 31 - # export APP_URLS="http://$LISTEN_HOST:$APP_PORT" 32 32 - # export API_URLS="http://$LISTEN_HOST:$HTTP_PORT,https://$LISTEN_HOST:$HTTPS_PORT" 33 33 - # export REDIRECT_URLS="http://$LISTEN_HOST:$HTTP_PORT/api/connect/discord/callback" 34 34 - # export DYNA_URL="postgres://$LISTEN_HOST:$DB_PORT/opencourseworld?sslmode=disable&user=root&options=-c%20search_path=ocw,smm2srv,public" 35 35 - # export LISTEN_ADDRESS="$LISTEN_HOST:$HTTPS_PORT" 36 36 - # export LISTEN_ADDRESS_HTTP="$LISTEN_HOST:$HTTP_PORT" 37 37 - # export DNS_WHITELIST="$LISTEN_HOST:$HTTPS_PORT,$LISTEN_HOST:$HTTP_PORT"

+7 -7

.gitignore

reviewed

··· 1 1 _vendor 2 2 - data/*/*.json 3 3 - data/*/*.bcd 4 4 - data/*/*.jpg 5 5 - data/*/*.jpeg 6 6 - data/*.db 2 2 + 3 3 + /data 7 4 __pycache__ 8 5 9 6 nex/.git_old ··· 20 17 env/ 21 18 22 19 *.bin 20 20 + *.zip 23 21 24 22 .DS_Store 25 23 ··· 27 25 *.dll 28 26 29 27 /private.env 30 30 - /dev-db/data 28 28 + /.env 31 29 /smm2_gameserver 32 32 - certs 30 30 + 31 31 + # unused, but still ignore just in case, as it may contain secrets 32 32 + /private.env

+48 -18

Dockerfile

reviewed

··· 1 1 - FROM golang:latest as builder 1 1 + # This Dockerfile is for both local development and production. 2 2 + # See docker-compose.yml for local dev configuration. 3 3 + FROM golang:latest AS builder 4 4 + RUN go install -tags 'cockroachdb' github.com/golang-migrate/migrate/v4/cmd/migrate@latest 2 5 3 6 WORKDIR /app 4 7 5 8 COPY go.mod go.sum ./ 6 9 RUN go mod download && go mod verify 7 10 8 8 - COPY main.go ./ 9 9 - COPY config config 10 10 - COPY db db 11 11 - COPY nex nex 12 12 - COPY util util 13 13 - COPY api api 14 14 - COPY orm orm 11 11 + COPY main.go /app/main.go 12 12 + COPY config /app/config 13 13 + COPY db /app/db 14 14 + COPY nex /app/nex 15 15 + COPY util /app/util 16 16 + COPY api /app/api 17 17 + COPY orm /app/orm 15 18 16 16 - RUN go build -v -o /app/server *.go 17 17 - RUN go build -v -o /app/serve_cache util/serve_dump_cache/*.go 18 18 - RUN echo '{}' > /app/config.json 19 19 + ENV GOCACHE=/go-cache 20 20 + # use a docker cache mount to avoid recompiling all dependencies each time 21 21 + RUN --mount=type=cache,target=/go-cache \ 22 22 + go build -v -o /app/server *.go 23 23 + RUN --mount=type=cache,target=/go-cache \ 24 24 + go build -v -o /app/serve_cache util/serve_dump_cache/*.go 19 25 20 20 - # be sure to have config.json inside /app/ mounted 21 21 - CMD ["/app/server"] 26 26 + CMD bash 22 27 23 23 - #FROM gcr.io/distroless/base-debian10 24 24 - FROM golang:latest 28 28 + FROM debian:bookworm-slim 29 29 + RUN apt update && apt install -y curl 30 30 + 31 31 + ARG TARGETARCH 32 32 + ENV COCKROACH_VERSION=v22.2.6.linux-$TARGETARCH 33 33 + ENV CRDB_CKSM_ARM64=aa3b695171235d143d8d0dc6ebdbec9908b7009b0e6f5b2f750cb3126d7e9298 34 34 + ENV CRDB_CKSM_AMD64=de670f9823d0794880ef0c72dfec213470cf6ea3fc96ed27e1706170bae6573b 35 35 + 36 36 + # install the cockroach-sql standalone binary 37 37 + RUN curl -q https://binaries.cockroachdb.com/cockroach-sql-$COCKROACH_VERSION.tgz | \ 38 38 + tar -xzvf - -C /usr/local/bin --strip-components 1 cockroach-sql-$COCKROACH_VERSION/cockroach-sql \ 39 39 + && if [ $TARGETARCH = "arm64" ]; then checksum=$CRDB_CKSM_ARM64; elif [ $TARGETARCH = "amd64" ]; then checksum=$CRDB_CKSM_AMD64; fi \ 40 40 + && echo "$checksum /usr/local/bin/cockroach-sql" | sha256sum --check 41 41 + 42 42 + COPY --from=builder /go/bin/migrate /bin/migrate 25 43 COPY --from=builder /app/server /bin/server 26 44 COPY --from=builder /app/serve_cache /bin/serve_cache 27 27 - COPY --from=builder /app/config.json /app/config.json 28 45 COPY --from=builder /app/db/migrations /app/db/migrations 46 46 + 47 47 + # only scripts that can be used in production are listed here. 48 48 + # in dev, the entire scripts/ directory is mounted. 49 49 + COPY \ 50 50 + ./scripts/db-console.sh \ 51 51 + ./scripts/db-restore-aws.sh \ 52 52 + ./scripts/env.sh \ 53 53 + ./scripts/migrate.sh \ 54 54 + ./scripts/util.sh \ 55 55 + /app/scripts/ 29 56 WORKDIR /app 30 30 - ENV HOSTNAME localhost 31 31 - CMD ["server"] 57 57 + 58 58 + ENV IS_DOCKER=1 59 59 + ENV DB_CONSOLE_TYPE=cockroach-sql 60 60 + 61 61 + CMD /bin/server

+29 -11

README.md

reviewed

··· 10 10 11 11 ### Set up a dev environment 12 12 13 13 - Most of the configuration options can be found in `.env`, and can be overridden by creating a file called `private.env`. This can be used to set up secrets, or override ports or binding hosts. 13 13 + Documentation of the configuration options can be found in `scripts/defaults.sh`. Each of these can be overridden using the syntax `VAR=value` in a `.env` file, or overridden in your environment by other means, such as `direnv`. 14 14 15 15 #### The Database 16 16 17 17 We use CockroachDB for our database, which should be run through Docker. Once you have set up docker, run: 18 18 19 19 ```bash 20 20 - ./dev-db/start.sh 20 20 + ./scripts/db-start.sh 21 21 ``` 22 22 23 23 to start the docker container for the database. 24 24 25 25 - To restore a backup from production to your local database, you will need an AWS access key and secret. Put these in your `private.env`: 25 25 + To restore a backup from production to your local database, you will need an AWS access key and secret. Put these in your `.env`: 26 26 27 27 ```bash 28 28 export AWS_ACCESS_KEY_ID=... 29 29 export AWS_SECRET_ACCESS_KEY=... 30 30 ``` 31 31 32 32 - and run `./dev-db/backup.sh` from the project root. 32 32 + and run `./scripts/db-restore-aws.sh` from the project root. 33 33 34 34 #### Auth 35 35 36 36 - In order to log in via Twitch and/or Discord, you will need to obtain client secrets from the team. Add them to your `private.env`: 36 36 + In order to log in via Twitch and/or Discord, you will need to obtain client secrets from the team. Add them to your `.env`: 37 37 38 38 ```bash 39 39 export DISCORD_CLIENT_SECRET=... ··· 41 41 ``` 42 42 43 43 #### The App 44 44 - Once the database is running, source the environment and run the app with `go`: 44 44 + Once the database is running, run the app with `./scripts/app.sh`: 45 45 46 46 ```bash 47 47 - $ source .env 48 48 - $ go run -v main.go 47 47 + $ ./scripts/app.sh 49 48 ``` 50 49 51 50 Once you have your server running, you can point your frontend to the http url configured in your env - by default, `http://localhost:6001`. 52 51 53 52 #### Connecting Ryujinx 54 53 55 55 - In order to connect Ryujinx to your local server, you must build a version yourself with SSL verification patched out. (TODO [jneen] get everything running behind caddy by default). The patch can be found in `ryujinx/ryujinx_disable_ssl_verify_for_local_testing.patch`. 54 54 + In order to connect Ryujinx to your local server, you must build a version yourself with SSL verification patched out. This is available on [our Ryujinx fork](https://github.com/mm2srv/Ryujinx), which we try to keep reasonably up to date. Alternatively, the patch can be found in this repository at `ryujinx/ryujinx_disable_ssl_verify_for_local_testing.patch`. 56 55 57 56 ## Game Client Mod 58 57 59 58 [client-mod](https://github.com/mm2srv/client-mod) and [releases](https://github.com/mm2srv/client-mod/releases) 60 59 61 61 - ## Support 60 60 + ## Running With Docker (recommended for Windows) 62 61 63 63 - At this time there is no support to get your own server up and running. 62 62 + ### Setup 63 63 + Before we get running with docker, you will need to: 64 64 + 65 65 + * Install Docker. 66 66 + * Create a file in the project called `.env` - you can place config overrides here. 67 67 + * Open a terminal and `cd` or `dir` into the project root. 68 68 + 69 69 + ### Running 70 70 + 71 71 + * Build the containers with `docker compose build`. 72 72 + * Run `docker compose up` and all the services will start. 73 73 + * To restart, quit the server using Ctrl-C and run both the above commands again. 74 74 + * For faster restarts, it's recommended to run the db in a separate terminal with `docker compose up db`. You can then run the app with `docker compose build app` and `docker compose run app`. Run both commands to restart when you've made code changes. 75 75 + 76 76 + ### Scripts 77 77 + * To run scripts, use `docker compose exec app bash`, or `docker compose exec app ./scripts/some-script.sh`. 78 78 + - For example, to run a database console, use `docker compose exec app ./scripts/db-console.sh` 79 79 + - To print `ocw-config.json` for the dev user, use `docker compose exec app ./scripts/ocw-api.sh ocw-config.json` 80 80 + - You can do similar for other scripts documented above. 81 81 + * If the app isn't running, use `docker compose run --rm` in place of `docker compose exec`. This will create a temporary container for running scripts in.

+24 -2

api/router.go

reviewed

··· 44 44 return user, result.Error 45 45 } 46 46 47 47 + const devMessage = ` 48 48 + some test commands you can run: 49 49 + %s stats 50 50 + %s import?maker_code=231723D7G 51 51 + %s import?maker_code=NGDSJG21H 52 52 + %s import?maker_code=KGHK00BWF 53 53 + %s ocw-config.json > ocw-config.json 54 54 + 55 55 + ` 56 56 + 57 57 + const varsTemplate = ` 58 58 + OCW_TOKEN='%s' 59 59 + OCW_ADDRESS='%s' 60 60 + ` 61 61 + 47 62 func NewRouter(_cfg *config.Config, _view view.DataPresenter) http.Handler { 48 63 cfg = _cfg 49 64 jwtSecret = []byte(cfg.JwtSecret) ··· 83 98 token, _ := createToken(user.ID) 84 99 fmt.Println("\n", user) 85 100 86 86 - fmt.Printf("\nsome test commands you can run:\n./bin/ocw-api stats\n./bin/ocw-api import?maker_code=231723D7G\n./bin/ocw-api import?maker_code=NGDSJG21H\n./bin/ocw-api import?maker_code=KGHK00BWF\n./bin/ocw-api ocw-config.json > ocw-config.json\n\n") 101 101 + var script string 102 102 + if isDocker := os.Getenv("IS_DOCKER"); isDocker != "" { 103 103 + script = "docker compose exec app ./scripts/ocw-api.sh" 104 104 + } else { 105 105 + script = "./scripts/ocw-api.sh" 106 106 + } 107 107 + 108 108 + fmt.Printf(devMessage, script, script, script, script, script) 87 109 88 88 - vars := fmt.Sprintf("export OCW_TOKEN='%s'\nexport OCW_ADDRESS='%s'\n", token, cfg.DnsAddressHttps) 110 110 + vars := fmt.Sprintf(varsTemplate, token, cfg.DnsAddressHttps) 89 111 os.MkdirAll("tmp/dev-user", 0755) 90 112 ioutil.WriteFile("tmp/dev-user/vars.sh", []byte(vars), 0644) 91 113 }

-41

bin/ocw-api

reviewed

··· 1 1 - #!/usr/bin/env bash 2 2 - 3 3 - stderr() { echo "$@" >&2 ;} 4 4 - execd() { stderr "$@"; "$@" ;} 5 5 - fail() { 6 6 - local status="$1"; shift 7 7 - stderr "$@" && exit "$status" 8 8 - } 9 9 - 10 10 - [[ -s "tmp/dev-user/vars.sh" ]] || fail 1 "server is not running with dev user enabled" 11 11 - 12 12 - . tmp/dev-user/vars.sh 13 13 - 14 14 - get() { 15 15 - local path="$1"; shift 16 16 - 17 17 - execd curl -k \ 18 18 - -H "Accept: application/json" \ 19 19 - -H "Authorization: Bearer $OCW_TOKEN" \ 20 20 - https://"$OCW_ADDRESS"/api/"$path" 21 21 - } 22 22 - 23 23 - websocket() { 24 24 - wscat -w 6000 -x "$OCW_TOKEN" -n -c wss://"$OCW_ADDRESS"/api/live 25 25 - } 26 26 - 27 27 - usage() { 28 28 - cat <<DOC >&2 29 29 - usage: 30 30 - $0 get some/path 31 31 - $0 websocket 32 32 - DOC 33 33 - } 34 34 - 35 35 - cmd="$1"; shift 36 36 - 37 37 - case cmd in 38 38 - get) get "$@" ;; 39 39 - websocket|ws) websocket "$@" ;; 40 40 - *) get "$cmd" "$@" 41 41 - esac

config/initdb/min_capacity_remaining.sql

reviewed

··· 1 1 + set cluster setting kv.bulk_io_write.min_capacity_remaining_fraction to 0;

-9

config/initdb/timeseries.sh

reviewed

··· 1 1 - #!/bin/bash 2 2 - 3 3 - # set to 24 hours 4 4 - #run_sql_query -e "SET CLUSTER SETTING timeseries.storage.resolution_10s.ttl = '24h0m0s';" 5 5 - 6 6 - # disable completely 7 7 - run_sql_query -e "SET CLUSTER SETTING timeseries.storage.enabled = false;" 8 8 - run_sql_query -e "SET CLUSTER SETTING timeseries.storage.resolution_10s.ttl = '0s';" 9 9 - run_sql_query -e "SET CLUSTER SETTING timeseries.storage.resolution_30m.ttl = '0s';"

config/initdb/timeseries.sql

reviewed

··· 1 1 + -- disable completely 2 2 + SET CLUSTER SETTING timeseries.storage.enabled = false; 3 3 + SET CLUSTER SETTING timeseries.storage.resolution_10s.ttl = '0s'; 4 4 + SET CLUSTER SETTING timeseries.storage.resolution_30m.ttl = '0s';

data/.gitkeep

reviewed

··· 1 1 + keep

data/certs/.gitkeep

reviewed

··· 1 1 + keep

data/dumps/.gitkeep

reviewed

··· 1 1 + keep

data/initdb/.gitkeep

reviewed

··· 1 1 + keep

+9 -2

db/dyna/dyna.go

reviewed

··· 21 21 "github.com/jackc/pgx/v5/pgtype" 22 22 "github.com/mm2srv/smm2_parsing" 23 23 24 24 + "github.com/avast/retry-go/v4" 25 25 + 24 26 "github.com/golang-migrate/migrate/v4" 25 27 _ "github.com/golang-migrate/migrate/v4/database/cockroachdb" 26 28 _ "github.com/golang-migrate/migrate/v4/database/postgres" ··· 54 56 } 55 57 56 58 var err error 57 57 - s.Conn, err = InitDyna(config) 59 59 + s.Conn, err = retry.DoWithData(func() (*pgxpool.Pool, error) { 60 60 + return InitDyna(config) 61 61 + }, retry.Attempts(10)) 62 62 + 58 63 if err != nil { 59 64 return nil, err 60 65 } ··· 125 130 126 131 fmt.Println("Connecting to dyna database...") 127 132 conn, err := pgxpool.ConnectConfig(context.Background(), config) 128 128 - fmt.Println("Connected to dyna database!") 129 133 if err != nil { 130 134 return nil, fmt.Errorf("could not connect to postgresql: %s", err) 131 135 } 136 136 + fmt.Println("Connected to dyna database!") 132 137 133 138 var m *migrate.Migrate 134 139 if cfg.DynaIsPostgres { ··· 137 142 m, err = migrate.New("file://db/migrations", strings.ReplaceAll(cfg.DynaUrl, "postgres://", "cockroachdb://")) 138 143 } 139 144 if err != nil { 145 145 + fmt.Printf("error loading migrations: %v\n", err) 140 146 return conn, err 141 147 } 142 148 143 149 if err := m.Up(); err != nil { 150 150 + fmt.Printf("error running migrations: %v\n", err) 144 151 if err != migrate.ErrNoChange { 145 152 return conn, err 146 153 }

-19

dev-db/backup.sh

reviewed

··· 1 1 - #!/usr/bin/env bash 2 2 - 3 3 - . .env 4 4 - 5 5 - fail() { 6 6 - echo "$@" >&2 7 7 - exit 1 8 8 - } 9 9 - 10 10 - [[ -z "$AWS_SECRET_ACCESS_KEY" ]] && fail 'please set AWS_SECRET_ACCESS_KEY.' 11 11 - [[ -z "$AWS_ACCESS_KEY_ID" ]] && fail 'please set AWS_ACCESS_KEY_ID.' 12 12 - 13 13 - sql() { 14 14 - cockroach sql --host localhost --insecure --port "$DB_PORT" --execute "$@" 15 15 - } 16 16 - 17 17 - echo "$DB_PORT" 18 18 - sql 'set cluster setting kv.bulk_io_write.min_capacity_remaining_fraction to 0' 19 19 - sql "restore database opencourseworld from latest in 's3://ocw-db/database-backups/opencourseworld?AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID&AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY'"

-2

dev-db/console.sh

reviewed

··· 1 1 - #!/usr/bin/env bash 2 2 - exec cockroach sql --host localhost --insecure --port "$DB_PORT" -d opencourseworld "$@"

-19

dev-db/docker-compose.yml

reviewed

··· 1 1 - services: 2 2 - db: 3 3 - image: cockroachdb/cockroach:v22.2.6 4 4 - network_mode: bridge 5 5 - command: 6 6 - - start-single-node 7 7 - - --insecure 8 8 - - --sql-addr=:${DB_PORT} 9 9 - - --listen-addr=:26257 10 10 - - --locality=region=wizulus 11 11 - - --max-disk-temp-storage 12 12 - - '0' 13 13 - volumes: 14 14 - - "./data/cockroach-data:/cockroach/cockroach-data" 15 15 - - "./data/db:/docker-entrypoint-initdb.d" 16 16 - ports: 17 17 - - 26257:26257 18 18 - - ${DB_PORT}:${DB_PORT} 19 19 - - 26280:8080

-5

dev-db/start.sh

reviewed

··· 1 1 - #!/usr/bin/env bash 2 2 - 3 3 - . .env 4 4 - 5 5 - cd dev-db && docker compose up db

+83

docker-compose.yml

reviewed

··· 1 1 + # This file is for local development. The production service description 2 2 + # is stored in portainer. 3 3 + # 4 4 + # Note that there is some duplication with defaults.env - this is because env_file 5 5 + # directives don't get interpolated into the docker-compose.yml. 6 6 + # see https://github.com/docker/compose/issues/3435 7 7 + # 8 8 + # Specifically, DB_PORT, CERTS_CACHE, etc variables have defaults here that 9 9 + # must match defaults.env. 10 10 + services: 11 11 + db: 12 12 + container_name: ocw-db 13 13 + image: cockroachdb/cockroach:v22.2.6 14 14 + environment: 15 15 + - COCKROACH_DATABASE=opencourseworld 16 16 + - COCKROACH_USER=root 17 17 + command: 18 18 + - start-single-node 19 19 + - --insecure 20 20 + - --sql-addr=:${DB_PORT:-5432} 21 21 + - --listen-addr=:26257 22 22 + - --locality=region=wizulus 23 23 + ports: 24 24 + - ${DB_PORT:-5432}:${DB_PORT:-5432} 25 25 + volumes: 26 26 + - './data/db:/cockroach/cockroach-data:rw' 27 27 + - "./config/initdb:/docker-entrypoint-initdb.d:ro" 28 28 + 29 29 + app: 30 30 + ports: 31 31 + - ${HTTP_PORT:-6001}:${HTTP_PORT:-6001} 32 32 + - ${HTTPS_PORT:-6000}:${HTTPS_PORT:-6000} 33 33 + volumes: 34 34 + - './data/dumps:/app/data/dumps:rw' 35 35 + - './data/course_import_cache:/app/data/course_import_cache:rw' 36 36 + - './db/migrations:/app/db/migrations:rw' 37 37 + 38 38 + # overwrite the scripts from Dockerfile so all the dev scripts are present 39 39 + - './scripts:/app/scripts:ro' 40 40 + build: 41 41 + dockerfile: Dockerfile 42 42 + 43 43 + command: 44 44 + - bash 45 45 + - '-c' 46 46 + - 'source ./scripts/defaults.sh && exec /bin/server' 47 47 + environment: 48 48 + - DB_HOST=ocw-db 49 49 + - LISTEN_ADDRESS_HTTP=:${HTTP_PORT:-6001} 50 50 + - LISTEN_ADDRESS=:${HTTPS_PORT:-6000} 51 51 + 52 52 + # inherit from .env and the os env 53 53 + - APP_PORT 54 54 + - DB_PORT 55 55 + - HTTP_PORT 56 56 + - HTTPS_PORT 57 57 + - LISTEN_HOST 58 58 + - DB_USER 59 59 + - DB_NAME 60 60 + - ENABLE_API_DEV_USER 61 61 + - FORCE_HTTP 62 62 + - DEBUG_LEVEL 63 63 + - COURSE_IMPORT_REQUIRE_VERIFY 64 64 + - COURSE_IMPORT_ENABLE_CACHE 65 65 + - USE_DUMP_FOR_SEARCHES 66 66 + - MIXED_DIFFICULTY 67 67 + - DATA_VIEW_TYPE 68 68 + - DUMP_FILENAME 69 69 + - DUMP_THUMBS_FILENAME 70 70 + - JWT_SECRET 71 71 + - TWITCH_CLIENT_ID 72 72 + - TWITCH_CLIENT_SECRET 73 73 + - DISCORD_CLIENT_ID 74 74 + - DISCORD_CLIENT_SECRET 75 75 + - ENABLE_TAGS 76 76 + - DB_SCHEMA_NAME 77 77 + - AWS_ACCESS_KEY_ID 78 78 + - AWS_SECRET_ACCESS_KEY 79 79 + - APP_URLS 80 80 + - API_URLS 81 81 + - REDIRECT_URLS 82 82 + - DYNA_URL 83 83 + - DNS_WHITELIST

go.mod

reviewed

··· 6 6 7 7 require ( 8 8 github.com/aead/cmac v0.0.0-20160719120800-7af84192f0b1 9 9 + github.com/avast/retry-go/v4 v4.6.0 9 10 github.com/buger/jsonparser v1.1.1 10 11 github.com/dgrijalva/jwt-go v3.2.0+incompatible 11 12 github.com/fogleman/gg v1.3.1-0.20210928143535-8febc0f526ad

go.sum

reviewed

··· 127 127 github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= 128 128 github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= 129 129 github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= 130 130 + github.com/avast/retry-go/v4 v4.6.0 h1:K9xNA+KeB8HHc2aWFuLb25Offp+0iVRXEvFx8IinRJA= 131 131 + github.com/avast/retry-go/v4 v4.6.0/go.mod h1:gvWlPhBVsvBbLkVGDg/KwvBv0bEkCOLRRSHKIr2PyOE= 130 132 github.com/aws/aws-sdk-go v1.15.11/go.mod h1:mFuSZ37Z9YOHbQEwBWztmVzqXrEkub65tZoCYDt7FT0= 131 133 github.com/aws/aws-sdk-go v1.17.7/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= 132 134 github.com/aws/aws-sdk-go-v2 v1.8.0/go.mod h1:xEFuWz+3TYdlPRuo+CqATbeDWIWyaT5uAPwPaWtgse0= ··· 1088 1090 github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= 1089 1091 github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= 1090 1092 github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= 1093 1093 + github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= 1091 1094 github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= 1092 1095 github.com/syndtr/gocapability v0.0.0-20170704070218-db04d3cc01c8/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww= 1093 1096 github.com/syndtr/gocapability v0.0.0-20180916011248-d98352740cb2/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww=

+10 -4

orm/orm.go

reviewed

··· 6 6 "gorm.io/driver/postgres" 7 7 "gorm.io/gorm" 8 8 "gorm.io/gorm/logger" 9 9 + 10 10 + "github.com/avast/retry-go/v4" 9 11 ) 10 12 11 13 func New(cfg *config.Config) *gorm.DB { 12 12 - pg := postgres.Open(cfg.DynaUrl) 13 13 - db, err := gorm.Open(pg, &gorm.Config{ 14 14 - Logger: logger.Default.LogMode(logger.Info), 15 15 - }) 14 14 + db, err := retry.DoWithData(func() (*gorm.DB, error) { 15 15 + pg := postgres.Open(cfg.DynaUrl) 16 16 + 17 17 + return gorm.Open(pg, &gorm.Config{ 18 18 + Logger: logger.Default.LogMode(logger.Info), 19 19 + }) 20 20 + }, retry.Attempts(10)) 21 21 + 16 22 if err != nil { 17 23 panic(err) 18 24 }

+10

scripts/app.sh

reviewed

··· 1 1 + #!/usr/bin/env bash 2 2 + 3 3 + . ./scripts/util.sh 4 4 + 5 5 + [[ -n "$IS_DOCKER" ]] && \ 6 6 + fail 'no need to use this script inside docker, use /bin/server instead' 7 7 + 8 8 + check-exe go 'you must install golang to run the server: https://go.dev' 9 9 + 10 10 + go run -v main.go

scripts/db-console.sh

reviewed

··· 1 1 + #!/usr/bin/env bash 2 2 + . ./scripts/util.sh 3 3 + 4 4 + dbconsole "$@"

scripts/db-drop.sh

reviewed

··· 1 1 + #!/usr/bin/env bash 2 2 + . ./scripts/util.sh 3 3 + 4 4 + sql 'drop database opencourseworld'

scripts/db-restore-aws.sh

reviewed

··· 1 1 + #!/usr/bin/env bash 2 2 + 3 3 + . ./scripts/util.sh 4 4 + 5 5 + [[ -z "$AWS_SECRET_ACCESS_KEY" ]] && fail 'please set AWS_SECRET_ACCESS_KEY.' 6 6 + [[ -z "$AWS_ACCESS_KEY_ID" ]] && fail 'please set AWS_ACCESS_KEY_ID.' 7 7 + 8 8 + sql "drop database if exists opencourseworld" 9 9 + sql "restore database opencourseworld from latest in 's3://ocw-db/single_backups/opencourseworld?AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID&AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY'"

scripts/db-start.sh

reviewed

··· 1 1 + #!/usr/bin/env bash 2 2 + 3 3 + . ./scripts/util.sh 4 4 + 5 5 + [[ -n "$IS_DOCKER" ]] && fail 'no need to run scripts/db-start.sh inside docker' 6 6 + 7 7 + execd docker compose up db

+80

scripts/defaults.sh

reviewed

··· 1 1 + #!/usr/bin/env bash 2 2 + 3 3 + # Default configs for the app. override these in .env or in your OS environment. 4 4 + # 5 5 + # DO NOT EDIT THIS FILE, you should override these variables in .env instead 6 6 + # 7 7 + # If you are changing or adding defaults *for everyone*, you can do it here, 8 8 + # but please make sure they are in sync with the defaults in docker-compose.yml. 9 9 + 10 10 + default() { 11 11 + local __var="$1"; shift 12 12 + local __val="$1"; shift 13 13 + 14 14 + [[ -z "${!__var}" ]] && export "$__var"="$__val" 15 15 + } 16 16 + 17 17 + # Specify the port for ocw-web. 18 18 + default APP_PORT 9000 19 19 + 20 20 + # Specify the port for the database. 21 21 + default DB_PORT 5432 22 22 + default HTTP_PORT 6001 23 23 + default HTTPS_PORT 6000 24 24 + 25 25 + # note: for local dev, must be localhost for login to work 26 26 + default LISTEN_HOST localhost 27 27 + 28 28 + # variables for accessing cockroachdb. 29 29 + # Note: these are provided for overriding convenience only. Scripts and the app 30 30 + # should only rely on the value of $DYNA_URL. 31 31 + default DB_HOST localhost 32 32 + default DB_USER root 33 33 + default DB_NAME opencourseworld 34 34 + 35 35 + # General app configs 36 36 + default ENABLE_API_DEV_USER true 37 37 + default FORCE_HTTP false 38 38 + default DEBUG_LEVEL light 39 39 + default COURSE_IMPORT_REQUIRE_VERIFY false 40 40 + default COURSE_IMPORT_ENABLE_CACHE true 41 41 + default USE_DUMP_FOR_SEARCHES false 42 42 + default MIXED_DIFFICULTY false 43 43 + default DATA_VIEW_TYPE combined_dyna 44 44 + 45 45 + default DATA_DIR ./data 46 46 + 47 47 + # Location of dumps on your system, if in use. Note: if using docker, 48 48 + # these must live in ./data, and you should use a symbolic link if you 49 49 + # need them elsewhere. 50 50 + default DUMP_FILENAME ./data/dumps/dump.db 51 51 + default DUMP_THUMBS_FILENAME ./data/dumps/dump_thumbs.db 52 52 + 53 53 + # A secret to use for logins. Does not have to be secure in local dev. 54 54 + default JWT_SECRET localdev 55 55 + 56 56 + # can be overridden to cockroach or psql, depending on which interface 57 57 + # you prefer. by default we try to detect which is installed. 58 58 + default DB_CONSOLE_TYPE auto 59 59 + 60 60 + # can be overridden to cockroach or psql, depending on which interface 61 61 + # you prefer. by default we try to detect which is installed. 62 62 + default DB_CONSOLE_TYPE auto 63 63 + 64 64 + default TWITCH_CLIENT_ID ufm5q62qafbnkp3leizjli8z89xo9w 65 65 + default TWITCH_CLIENT_SECRET 'please set TWITCH_CLIENT_SECRET in .env' 66 66 + default DISCORD_CLIENT_ID 1080395755581349908 67 67 + default DISCORD_CLIENT_SECRET 'please set DISCORD_CLIENT_SECRET in .env' 68 68 + default ENABLE_TAGS false 69 69 + default DB_SCHEMA_NAME smm2srv 70 70 + 71 71 + default AWS_ACCESS_KEY_ID 'please set AWS_ACCESS_KEY_ID in .env' 72 72 + default AWS_SECRET_ACCESS_KEY 'please set AWS_SECRET_ACCESS_KEY in .env' 73 73 + 74 74 + default APP_URLS "http://$LISTEN_HOST:$APP_PORT" 75 75 + default API_URLS "http://$LISTEN_HOST:$HTTP_PORT,https://$LISTEN_HOST:$HTTPS_PORT" 76 76 + default REDIRECT_URLS "http://$LISTEN_HOST:$HTTP_PORT/api/connect/discord/callback" 77 77 + default DYNA_URL "postgres://$DB_HOST:$DB_PORT/$DB_NAME?sslmode=disable&user=$DB_USER&options=-c%20search_path%3D$DB_SCHEMA_NAME,public" 78 78 + default LISTEN_ADDRESS "$LISTEN_HOST:$HTTPS_PORT" 79 79 + default LISTEN_ADDRESS_HTTP "$LISTEN_HOST:$HTTP_PORT" 80 80 + default DNS_WHITELIST "$LISTEN_HOST:$HTTPS_PORT,$LISTEN_HOST:$HTTP_PORT"

scripts/env.sh

reviewed

··· 1 1 + #!/usr/bin/env bash 2 2 + 3 3 + . ./scripts/util.sh 4 4 + 5 5 + env

scripts/migrate-create.sh

reviewed

··· 1 1 + #!/usr/bin/env bash 2 2 + 3 3 + . ./scripts/util.sh 4 4 + 5 5 + db-migrate create -ext=sql -dir=db/migrations -seq -digits=6 "$@"

scripts/migrate.sh

reviewed

··· 1 1 + #!/usr/bin/env bash 2 2 + 3 3 + . scripts/util.sh 4 4 + 5 5 + db-migrate "$@"

+46

scripts/ocw-api.sh

reviewed

··· 1 1 + #!/usr/bin/env bash 2 2 + 3 3 + . ./scripts/util.sh 4 4 + 5 5 + [[ -s "tmp/dev-user/vars.sh" ]] || { 6 6 + if [[ -n "$IS_DOCKER" ]]; then 7 7 + fail "can't find a running server on this image. use \`docker compose up app\` and then \`docker compose exec app $0\`" 8 8 + else 9 9 + fail "server is not running with dev user enabled" 10 10 + fi 11 11 + } 12 12 + 13 13 + set -a 14 14 + . ./tmp/dev-user/vars.sh 15 15 + set +a 16 16 + 17 17 + check-exe curl "curl not found" 18 18 + 19 19 + get() { 20 20 + local path="$1"; shift 21 21 + 22 22 + execd curl -k \ 23 23 + -H "Accept: application/json" \ 24 24 + -H "Authorization: Bearer $OCW_TOKEN" \ 25 25 + https://"$OCW_ADDRESS"/api/"$path" 26 26 + } 27 27 + 28 28 + websocket() { 29 29 + wscat -w 6000 -x "$OCW_TOKEN" -n -c wss://"$OCW_ADDRESS"/api/live 30 30 + } 31 31 + 32 32 + usage() { 33 33 + cat <<DOC >&2 34 34 + usage: 35 35 + $0 get some/path 36 36 + $0 websocket 37 37 + DOC 38 38 + } 39 39 + 40 40 + cmd="$1"; shift 41 41 + 42 42 + case cmd in 43 43 + get) get "$@" ;; 44 44 + websocket|ws) websocket "$@" ;; 45 45 + *) get "$cmd" "$@" 46 46 + esac

+110

scripts/util.sh

reviewed

··· 1 1 + #!/usr/bin/env bash 2 2 + 3 3 + stderr() { 4 4 + echo "$@" >&2 5 5 + } 6 6 + 7 7 + fail() { 8 8 + stderr "$@" 9 9 + exit 1 10 10 + } 11 11 + 12 12 + execd() { 13 13 + case "$@" in 14 14 + *SECRET*) ;; 15 15 + *) stderr "$@" 16 16 + esac 17 17 + 18 18 + "$@" 19 19 + } 20 20 + 21 21 + putd() { 22 22 + local __var="$1"; shift 23 23 + stderr "$__var=[${!__var}]" 24 24 + } 25 25 + 26 26 + check-exe() { 27 27 + local exe="$1"; shift 28 28 + local msg="$1"; shift 29 29 + 30 30 + type "$exe" >&2 || fail "$msg" 31 31 + } 32 32 + 33 33 + db-migrate() { 34 34 + check-exe migrate "install golang-migrate first:\nhttps://github.com/golang-migrate/migrate/tree/master/cmd/migrate" 35 35 + local crdb_url="${DYNA_URL/postgres:/cockroachdb:}" 36 36 + execd migrate -path=db/migrations -database="$crdb_url" "$@" 37 37 + } 38 38 + 39 39 + [[ -s ./scripts/defaults.sh ]] \ 40 40 + || fail "please run scripts from the project root, with ./scripts/thing.sh" 41 41 + 42 42 + # load the dotenv files in export mode 43 43 + # loading defaults.sh later because it won't override previously-set variables, 44 44 + # and needs overrides set for derived vars. 45 45 + set -a 46 46 + [[ -s .env ]] && source ./.env 47 47 + [[ -s ./scripts/defaults.sh ]] && source ./scripts/defaults.sh 48 48 + set +a 49 49 + 50 50 + dbconsole() { 51 51 + init-dbconsole 52 52 + 53 53 + dbconsole "$@" 54 54 + } 55 55 + 56 56 + sql() { 57 57 + init-dbconsole 58 58 + 59 59 + sql "$@" 60 60 + } 61 61 + 62 62 + init-dbconsole() { 63 63 + [[ "$DB_CONSOLE_TYPE" = auto ]] && type cockroach >&2 && { 64 64 + export DB_CONSOLE_TYPE=cockroach 65 65 + cockroach-sql() { cockroach sql "$@" ;} 66 66 + } 67 67 + 68 68 + [[ "$DB_CONSOLE_TYPE" = auto ]] && type cockroach-sql >&2 && export DB_CONSOLE_TYPE=cockroach 69 69 + 70 70 + [[ "$DB_CONSOLE_TYPE" = auto ]] && type psql >&2 && export DB_CONSOLE_TYPE=psql 71 71 + 72 72 + case "$DB_CONSOLE_TYPE" in 73 73 + cockroach) 74 74 + dbconsole() { 75 75 + execd cockroach sql --url="${DYNA_URL}" "$@" 76 76 + } 77 77 + 78 78 + sql() { 79 79 + dbconsole --execute "$@" 80 80 + } 81 81 + ;; 82 82 + cockroach-sql) 83 83 + dbconsole() { 84 84 + execd cockroach-sql --url="${DYNA_URL}" "$@" 85 85 + } 86 86 + 87 87 + sql() { 88 88 + dbconsole --execute "$@" 89 89 + } 90 90 + ;; 91 91 + psql) 92 92 + dbconsole() { 93 93 + PGCLIENTENCODING='utf-8' execd psql "$DYNA_URL" 94 94 + } 95 95 + 96 96 + sql() { 97 97 + dbconsole --command "$@;" 98 98 + } 99 99 + ;; 100 100 + *) 101 101 + dbconsole() { 102 102 + fail 'you must install either cockroachdb or psql to use db scripts.' 103 103 + } 104 104 + 105 105 + sql() { 106 106 + dbconsole 107 107 + } 108 108 + ;; 109 109 + esac 110 110 + }