Monorepo for Tangled tangled.org

*/db: set sqlite options across all connections in the pool

This does result in the removal of some options that aren't supported by
the SQLite driver [0], but IMO we can stick with the defaults for these.

[0]: https://github.com/mattn/go-sqlite3#connection-string

Signed-off-by: Winter <winter@winter.cafe>

Changed files
+59 -40
appview
db
knotserver
db
spindle
db
+31 -22
appview/db/db.go
··· 27 27 } 28 28 29 29 func Make(dbPath string) (*DB, error) { 30 - db, err := sql.Open("sqlite3", dbPath+"?_foreign_keys=1") 30 + // https://github.com/mattn/go-sqlite3#connection-string 31 + opts := []string{ 32 + "_foreign_keys=1", 33 + "_journal_mode=WAL", 34 + "_synchronous=NORMAL", 35 + "_auto_vacuum=incremental", 36 + } 37 + 38 + db, err := sql.Open("sqlite3", dbPath+"?"+strings.Join(opts, "&")) 39 + if err != nil { 40 + return nil, err 41 + } 42 + 43 + ctx := context.Background() 44 + 45 + conn, err := db.Conn(ctx) 31 46 if err != nil { 32 47 return nil, err 33 48 } 34 - _, err = db.Exec(` 35 - pragma journal_mode = WAL; 36 - pragma synchronous = normal; 37 - pragma temp_store = memory; 38 - pragma mmap_size = 30000000000; 39 - pragma page_size = 32768; 40 - pragma auto_vacuum = incremental; 41 - pragma busy_timeout = 5000; 49 + defer conn.Close() 42 50 51 + _, err = conn.ExecContext(ctx, ` 43 52 create table if not exists registrations ( 44 53 id integer primary key autoincrement, 45 54 domain text not null unique, ··· 467 476 } 468 477 469 478 // run migrations 470 - runMigration(db, "add-description-to-repos", func(tx *sql.Tx) error { 479 + runMigration(conn, "add-description-to-repos", func(tx *sql.Tx) error { 471 480 tx.Exec(` 472 481 alter table repos add column description text check (length(description) <= 200); 473 482 `) 474 483 return nil 475 484 }) 476 485 477 - runMigration(db, "add-rkey-to-pubkeys", func(tx *sql.Tx) error { 486 + runMigration(conn, "add-rkey-to-pubkeys", func(tx *sql.Tx) error { 478 487 // add unconstrained column 479 488 _, err := tx.Exec(` 480 489 alter table public_keys ··· 497 506 return nil 498 507 }) 499 508 500 - runMigration(db, "add-rkey-to-comments", func(tx *sql.Tx) error { 509 + runMigration(conn, "add-rkey-to-comments", func(tx *sql.Tx) error { 501 510 _, err := tx.Exec(` 502 511 alter table comments drop column comment_at; 503 512 alter table comments add column rkey text; ··· 505 514 return err 506 515 }) 507 516 508 - runMigration(db, "add-deleted-and-edited-to-issue-comments", func(tx *sql.Tx) error { 517 + runMigration(conn, "add-deleted-and-edited-to-issue-comments", func(tx *sql.Tx) error { 509 518 _, err := tx.Exec(` 510 519 alter table comments add column deleted text; -- timestamp 511 520 alter table comments add column edited text; -- timestamp ··· 513 522 return err 514 523 }) 515 524 516 - runMigration(db, "add-source-info-to-pulls-and-submissions", func(tx *sql.Tx) error { 525 + runMigration(conn, "add-source-info-to-pulls-and-submissions", func(tx *sql.Tx) error { 517 526 _, err := tx.Exec(` 518 527 alter table pulls add column source_branch text; 519 528 alter table pulls add column source_repo_at text; ··· 522 531 return err 523 532 }) 524 533 525 - runMigration(db, "add-source-to-repos", func(tx *sql.Tx) error { 534 + runMigration(conn, "add-source-to-repos", func(tx *sql.Tx) error { 526 535 _, err := tx.Exec(` 527 536 alter table repos add column source text; 528 537 `) ··· 533 542 // NOTE: this cannot be done in a transaction, so it is run outside [0] 534 543 // 535 544 // [0]: https://sqlite.org/pragma.html#pragma_foreign_keys 536 - db.Exec("pragma foreign_keys = off;") 537 - runMigration(db, "recreate-pulls-column-for-stacking-support", func(tx *sql.Tx) error { 545 + conn.ExecContext(ctx, "pragma foreign_keys = off;") 546 + runMigration(conn, "recreate-pulls-column-for-stacking-support", func(tx *sql.Tx) error { 538 547 _, err := tx.Exec(` 539 548 create table pulls_new ( 540 549 -- identifiers ··· 589 598 `) 590 599 return err 591 600 }) 592 - db.Exec("pragma foreign_keys = on;") 601 + conn.ExecContext(ctx, "pragma foreign_keys = on;") 593 602 594 603 // run migrations 595 - runMigration(db, "add-spindle-to-repos", func(tx *sql.Tx) error { 604 + runMigration(conn, "add-spindle-to-repos", func(tx *sql.Tx) error { 596 605 tx.Exec(` 597 606 alter table repos add column spindle text; 598 607 `) ··· 600 609 }) 601 610 602 611 // recreate and add rkey + created columns with default constraint 603 - runMigration(db, "rework-collaborators-table", func(tx *sql.Tx) error { 612 + runMigration(conn, "rework-collaborators-table", func(tx *sql.Tx) error { 604 613 // create new table 605 614 // - repo_at instead of repo integer 606 615 // - rkey field ··· 659 668 660 669 type migrationFn = func(*sql.Tx) error 661 670 662 - func runMigration(d *sql.DB, name string, migrationFn migrationFn) error { 663 - tx, err := d.Begin() 671 + func runMigration(c *sql.Conn, name string, migrationFn migrationFn) error { 672 + tx, err := c.BeginTx(context.Background(), nil) 664 673 if err != nil { 665 674 return err 666 675 }
+14 -9
knotserver/db/init.go
··· 2 2 3 3 import ( 4 4 "database/sql" 5 + "strings" 5 6 6 7 _ "github.com/mattn/go-sqlite3" 7 8 ) ··· 11 12 } 12 13 13 14 func Setup(dbPath string) (*DB, error) { 14 - db, err := sql.Open("sqlite3", dbPath+"?_foreign_keys=1") 15 + // https://github.com/mattn/go-sqlite3#connection-string 16 + opts := []string{ 17 + "_foreign_keys=1", 18 + "_journal_mode=WAL", 19 + "_synchronous=NORMAL", 20 + "_auto_vacuum=incremental", 21 + } 22 + 23 + db, err := sql.Open("sqlite3", dbPath+"?"+strings.Join(opts, "&")) 15 24 if err != nil { 16 25 return nil, err 17 26 } 18 27 28 + // NOTE: If any other migration is added here, you MUST 29 + // copy the pattern in appview: use a single sql.Conn 30 + // for every migration. 31 + 19 32 _, err = db.Exec(` 20 - pragma journal_mode = WAL; 21 - pragma synchronous = normal; 22 - pragma temp_store = memory; 23 - pragma mmap_size = 30000000000; 24 - pragma page_size = 32768; 25 - pragma auto_vacuum = incremental; 26 - pragma busy_timeout = 5000; 27 - 28 33 create table if not exists known_dids ( 29 34 did text primary key 30 35 );
+14 -9
spindle/db/db.go
··· 2 2 3 3 import ( 4 4 "database/sql" 5 + "strings" 5 6 6 7 _ "github.com/mattn/go-sqlite3" 7 8 ) ··· 11 12 } 12 13 13 14 func Make(dbPath string) (*DB, error) { 14 - db, err := sql.Open("sqlite3", dbPath+"?_foreign_keys=1") 15 + // https://github.com/mattn/go-sqlite3#connection-string 16 + opts := []string{ 17 + "_foreign_keys=1", 18 + "_journal_mode=WAL", 19 + "_synchronous=NORMAL", 20 + "_auto_vacuum=incremental", 21 + } 22 + 23 + db, err := sql.Open("sqlite3", dbPath+"?"+strings.Join(opts, "&")) 15 24 if err != nil { 16 25 return nil, err 17 26 } 18 27 28 + // NOTE: If any other migration is added here, you MUST 29 + // copy the pattern in appview: use a single sql.Conn 30 + // for every migration. 31 + 19 32 _, err = db.Exec(` 20 - pragma journal_mode = WAL; 21 - pragma synchronous = normal; 22 - pragma temp_store = memory; 23 - pragma mmap_size = 30000000000; 24 - pragma page_size = 32768; 25 - pragma auto_vacuum = incremental; 26 - pragma busy_timeout = 5000; 27 - 28 33 create table if not exists _jetstream ( 29 34 id integer primary key autoincrement, 30 35 last_time_us integer not null