loading up the forgejo repo on tangled to test page performance

feat: use XORM EngineGroup instead of single Engine connection (#7212)

Resolves #7207

Add new configuration to make XORM work with a main and replicas database instances. The follow configuration parameters were added:

- `HOST_PRIMARY`
- `HOST_REPLICAS`
- `LOAD_BALANCE_POLICY`. Options:
- `"WeightRandom"` -> `xorm.WeightRandomPolicy`
- `"WeightRoundRobin` -> `WeightRoundRobinPolicy`
- `"LeastCon"` -> `LeastConnPolicy`
- `"RoundRobin"` -> `xorm.RoundRobinPolicy()`
- default: `xorm.RandomPolicy()`
- `LOAD_BALANCE_WEIGHTS`

Co-authored-by: pat-s <patrick.schratz@gmail.com@>
Reviewed-on: https://codeberg.org/forgejo/forgejo/pulls/7212
Reviewed-by: Gusted <gusted@noreply.codeberg.org>
Co-authored-by: pat-s <patrick.schratz@gmail.com>
Co-committed-by: pat-s <patrick.schratz@gmail.com>

authored by pat-s pat-s pat-s and committed by Gusted 63a80bf2 a23d0453

Changed files
+462 -128
cmd
models
modules
routers
common
install
services
tests
integration
+10 -5
cmd/doctor.go
··· 20 20 "forgejo.org/services/doctor" 21 21 22 22 "github.com/urfave/cli/v2" 23 - "xorm.io/xorm" 24 23 ) 25 24 26 25 // CmdDoctor represents the available doctor sub-command. ··· 120 119 121 120 args := ctx.Args() 122 121 names := make([]string, 0, ctx.NArg()) 123 - for i := 0; i < ctx.NArg(); i++ { 122 + for i := range ctx.NArg() { 124 123 names = append(names, args.Get(i)) 125 124 } 126 125 ··· 130 129 } 131 130 recreateTables := migrate_base.RecreateTables(beans...) 132 131 133 - return db.InitEngineWithMigration(stdCtx, func(x *xorm.Engine) error { 134 - if err := migrations.EnsureUpToDate(x); err != nil { 132 + return db.InitEngineWithMigration(stdCtx, func(x db.Engine) error { 133 + engine, err := db.GetMasterEngine(x) 134 + if err != nil { 135 135 return err 136 136 } 137 - return recreateTables(x) 137 + 138 + if err := migrations.EnsureUpToDate(engine); err != nil { 139 + return err 140 + } 141 + 142 + return recreateTables(engine) 138 143 }) 139 144 } 140 145
+7 -1
cmd/migrate.go
··· 36 36 log.Info("Log path: %s", setting.Log.RootPath) 37 37 log.Info("Configuration file: %s", setting.CustomConf) 38 38 39 - if err := db.InitEngineWithMigration(context.Background(), migrations.Migrate); err != nil { 39 + if err := db.InitEngineWithMigration(context.Background(), func(dbEngine db.Engine) error { 40 + masterEngine, err := db.GetMasterEngine(dbEngine) 41 + if err != nil { 42 + return err 43 + } 44 + return migrations.Migrate(masterEngine) 45 + }); err != nil { 40 46 log.Fatal("Failed to initialize ORM engine: %v", err) 41 47 return err 42 48 }
+4 -1
cmd/migrate_storage.go
··· 23 23 "forgejo.org/modules/storage" 24 24 25 25 "github.com/urfave/cli/v2" 26 + "xorm.io/xorm" 26 27 ) 27 28 28 29 // CmdMigrateStorage represents the available migrate storage sub-command. ··· 195 196 log.Info("Log path: %s", setting.Log.RootPath) 196 197 log.Info("Configuration file: %s", setting.CustomConf) 197 198 198 - if err := db.InitEngineWithMigration(context.Background(), migrations.Migrate); err != nil { 199 + if err := db.InitEngineWithMigration(context.Background(), func(e db.Engine) error { 200 + return migrations.Migrate(e.(*xorm.Engine)) 201 + }); err != nil { 199 202 log.Fatal("Failed to initialize ORM engine: %v", err) 200 203 return err 201 204 }
+123 -59
models/db/engine.go
··· 95 95 } 96 96 } 97 97 98 - // newXORMEngine returns a new XORM engine from the configuration 99 - func newXORMEngine() (*xorm.Engine, error) { 100 - connStr, err := setting.DBConnStr() 98 + // newXORMEngineGroup creates an xorm.EngineGroup (with one master and one or more slaves). 99 + // It assumes you have separate master and slave DSNs defined via the settings package. 100 + func newXORMEngineGroup() (Engine, error) { 101 + // Retrieve master DSN from settings. 102 + masterConnStr, err := setting.DBMasterConnStr() 101 103 if err != nil { 102 - return nil, err 104 + return nil, fmt.Errorf("failed to determine master DSN: %w", err) 103 105 } 104 106 105 - var engine *xorm.Engine 106 - 107 + var masterEngine *xorm.Engine 108 + // For PostgreSQL: if a schema is provided, we use the special "postgresschema" driver. 107 109 if setting.Database.Type.IsPostgreSQL() && len(setting.Database.Schema) > 0 { 108 - // OK whilst we sort out our schema issues - create a schema aware postgres 109 110 registerPostgresSchemaDriver() 110 - engine, err = xorm.NewEngine("postgresschema", connStr) 111 + masterEngine, err = xorm.NewEngine("postgresschema", masterConnStr) 111 112 } else { 112 - engine, err = xorm.NewEngine(setting.Database.Type.String(), connStr) 113 + masterEngine, err = xorm.NewEngine(setting.Database.Type.String(), masterConnStr) 114 + } 115 + if err != nil { 116 + return nil, fmt.Errorf("failed to create master engine: %w", err) 113 117 } 118 + if setting.Database.Type.IsMySQL() { 119 + masterEngine.Dialect().SetParams(map[string]string{"rowFormat": "DYNAMIC"}) 120 + } 121 + masterEngine.SetSchema(setting.Database.Schema) 114 122 123 + slaveConnStrs, err := setting.DBSlaveConnStrs() 115 124 if err != nil { 116 - return nil, err 125 + return nil, fmt.Errorf("failed to load slave DSNs: %w", err) 126 + } 127 + 128 + var slaveEngines []*xorm.Engine 129 + // Iterate over all slave DSNs and create engines 130 + for _, dsn := range slaveConnStrs { 131 + slaveEngine, err := xorm.NewEngine(setting.Database.Type.String(), dsn) 132 + if err != nil { 133 + return nil, fmt.Errorf("failed to create slave engine for dsn %q: %w", dsn, err) 134 + } 135 + if setting.Database.Type.IsMySQL() { 136 + slaveEngine.Dialect().SetParams(map[string]string{"rowFormat": "DYNAMIC"}) 137 + } 138 + slaveEngine.SetSchema(setting.Database.Schema) 139 + slaveEngines = append(slaveEngines, slaveEngine) 117 140 } 118 - if setting.Database.Type.IsMySQL() { 119 - engine.Dialect().SetParams(map[string]string{"rowFormat": "DYNAMIC"}) 141 + 142 + policy := setting.BuildLoadBalancePolicy(&setting.Database, slaveEngines) 143 + 144 + // Create the EngineGroup using the selected policy 145 + group, err := xorm.NewEngineGroup(masterEngine, slaveEngines, policy) 146 + if err != nil { 147 + return nil, fmt.Errorf("failed to create engine group: %w", err) 120 148 } 121 - engine.SetSchema(setting.Database.Schema) 122 - return engine, nil 149 + return engineGroupWrapper{group}, nil 150 + } 151 + 152 + type engineGroupWrapper struct { 153 + *xorm.EngineGroup 154 + } 155 + 156 + func (w engineGroupWrapper) AddHook(hook contexts.Hook) bool { 157 + w.EngineGroup.AddHook(hook) 158 + return true 123 159 } 124 160 125 - // SyncAllTables sync the schemas of all tables, is required by unit test code 161 + // SyncAllTables sync the schemas of all tables 126 162 func SyncAllTables() error { 127 163 _, err := x.StoreEngine("InnoDB").SyncWithOptions(xorm.SyncOptions{ 128 164 WarnIfDatabaseColumnMissed: true, ··· 130 166 return err 131 167 } 132 168 133 - // InitEngine initializes the xorm.Engine and sets it as db.DefaultContext 169 + // InitEngine initializes the xorm EngineGroup and sets it as db.DefaultContext 134 170 func InitEngine(ctx context.Context) error { 135 - xormEngine, err := newXORMEngine() 171 + xormEngine, err := newXORMEngineGroup() 136 172 if err != nil { 137 173 return fmt.Errorf("failed to connect to database: %w", err) 138 174 } 139 - 140 - xormEngine.SetMapper(names.GonicMapper{}) 141 - // WARNING: for serv command, MUST remove the output to os.stdout, 142 - // so use log file to instead print to stdout. 143 - xormEngine.SetLogger(NewXORMLogger(setting.Database.LogSQL)) 144 - xormEngine.ShowSQL(setting.Database.LogSQL) 145 - xormEngine.SetMaxOpenConns(setting.Database.MaxOpenConns) 146 - xormEngine.SetMaxIdleConns(setting.Database.MaxIdleConns) 147 - xormEngine.SetConnMaxLifetime(setting.Database.ConnMaxLifetime) 148 - xormEngine.SetConnMaxIdleTime(setting.Database.ConnMaxIdleTime) 149 - xormEngine.SetDefaultContext(ctx) 175 + // Try to cast to the concrete type to access diagnostic methods 176 + if eng, ok := xormEngine.(engineGroupWrapper); ok { 177 + eng.SetMapper(names.GonicMapper{}) 178 + // WARNING: for serv command, MUST remove the output to os.Stdout, 179 + // so use a log file instead of printing to stdout. 180 + eng.SetLogger(NewXORMLogger(setting.Database.LogSQL)) 181 + eng.ShowSQL(setting.Database.LogSQL) 182 + eng.SetMaxOpenConns(setting.Database.MaxOpenConns) 183 + eng.SetMaxIdleConns(setting.Database.MaxIdleConns) 184 + eng.SetConnMaxLifetime(setting.Database.ConnMaxLifetime) 185 + eng.SetConnMaxIdleTime(setting.Database.ConnMaxIdleTime) 186 + eng.SetDefaultContext(ctx) 150 187 151 - if setting.Database.SlowQueryThreshold > 0 { 152 - xormEngine.AddHook(&SlowQueryHook{ 153 - Treshold: setting.Database.SlowQueryThreshold, 154 - Logger: log.GetLogger("xorm"), 155 - }) 156 - } 188 + if setting.Database.SlowQueryThreshold > 0 { 189 + eng.AddHook(&SlowQueryHook{ 190 + Treshold: setting.Database.SlowQueryThreshold, 191 + Logger: log.GetLogger("xorm"), 192 + }) 193 + } 157 194 158 - errorLogger := log.GetLogger("xorm") 159 - if setting.IsInTesting { 160 - errorLogger = log.GetLogger(log.DEFAULT) 161 - } 195 + errorLogger := log.GetLogger("xorm") 196 + if setting.IsInTesting { 197 + errorLogger = log.GetLogger(log.DEFAULT) 198 + } 162 199 163 - xormEngine.AddHook(&ErrorQueryHook{ 164 - Logger: errorLogger, 165 - }) 200 + eng.AddHook(&ErrorQueryHook{ 201 + Logger: errorLogger, 202 + }) 166 203 167 - xormEngine.AddHook(&TracingHook{}) 204 + eng.AddHook(&TracingHook{}) 168 205 169 - SetDefaultEngine(ctx, xormEngine) 206 + SetDefaultEngine(ctx, eng) 207 + } else { 208 + // Fallback: if type assertion fails, set default engine without extended diagnostics 209 + SetDefaultEngine(ctx, xormEngine) 210 + } 170 211 return nil 171 212 } 172 213 173 - // SetDefaultEngine sets the default engine for db 174 - func SetDefaultEngine(ctx context.Context, eng *xorm.Engine) { 175 - x = eng 214 + // SetDefaultEngine sets the default engine for db. 215 + func SetDefaultEngine(ctx context.Context, eng Engine) { 216 + masterEngine, err := GetMasterEngine(eng) 217 + if err == nil { 218 + x = masterEngine 219 + } 220 + 176 221 DefaultContext = &Context{ 177 222 Context: ctx, 178 - e: x, 223 + e: eng, 179 224 } 180 225 } 181 226 ··· 191 236 DefaultContext = nil 192 237 } 193 238 194 - // InitEngineWithMigration initializes a new xorm.Engine and sets it as the db.DefaultContext 239 + // InitEngineWithMigration initializes a new xorm EngineGroup, runs migrations, and sets it as db.DefaultContext 195 240 // This function must never call .Sync() if the provided migration function fails. 196 241 // When called from the "doctor" command, the migration function is a version check 197 242 // that prevents the doctor from fixing anything in the database if the migration level 198 243 // is different from the expected value. 199 - func InitEngineWithMigration(ctx context.Context, migrateFunc func(*xorm.Engine) error) (err error) { 244 + func InitEngineWithMigration(ctx context.Context, migrateFunc func(Engine) error) (err error) { 200 245 if err = InitEngine(ctx); err != nil { 201 246 return err 202 247 } ··· 230 275 return nil 231 276 } 232 277 233 - // NamesToBean return a list of beans or an error 278 + // NamesToBean returns a list of beans given names 234 279 func NamesToBean(names ...string) ([]any, error) { 235 280 beans := []any{} 236 281 if len(names) == 0 { 237 282 beans = append(beans, tables...) 238 283 return beans, nil 239 284 } 240 - // Need to map provided names to beans... 285 + // Map provided names to beans 241 286 beanMap := make(map[string]any) 242 287 for _, bean := range tables { 243 288 beanMap[strings.ToLower(reflect.Indirect(reflect.ValueOf(bean)).Type().Name())] = bean ··· 259 304 return beans, nil 260 305 } 261 306 262 - // DumpDatabase dumps all data from database according the special database SQL syntax to file system. 307 + // DumpDatabase dumps all data from database using special SQL syntax to the file system. 263 308 func DumpDatabase(filePath, dbType string) error { 264 309 var tbs []*schemas.Table 265 310 for _, t := range tables { ··· 295 340 return 999 / len(t.ColumnsSeq()) 296 341 } 297 342 298 - // IsTableNotEmpty returns true if table has at least one record 343 + // IsTableNotEmpty returns true if the table has at least one record 299 344 func IsTableNotEmpty(beanOrTableName any) (bool, error) { 300 345 return x.Table(beanOrTableName).Exist() 301 346 } 302 347 303 - // DeleteAllRecords will delete all the records of this table 348 + // DeleteAllRecords deletes all records in the given table. 304 349 func DeleteAllRecords(tableName string) error { 305 350 _, err := x.Exec(fmt.Sprintf("DELETE FROM %s", tableName)) 306 351 return err 307 352 } 308 353 309 - // GetMaxID will return max id of the table 354 + // GetMaxID returns the maximum id in the table 310 355 func GetMaxID(beanOrTableName any) (maxID int64, err error) { 311 356 _, err = x.Select("MAX(id)").Table(beanOrTableName).Get(&maxID) 312 357 return maxID, err 313 358 } 314 359 315 360 func SetLogSQL(ctx context.Context, on bool) { 316 - e := GetEngine(ctx) 317 - if x, ok := e.(*xorm.Engine); ok { 318 - x.ShowSQL(on) 319 - } else if sess, ok := e.(*xorm.Session); ok { 361 + ctxEngine := GetEngine(ctx) 362 + 363 + if sess, ok := ctxEngine.(*xorm.Session); ok { 320 364 sess.Engine().ShowSQL(on) 365 + } else if wrapper, ok := ctxEngine.(engineGroupWrapper); ok { 366 + // Handle engineGroupWrapper directly 367 + wrapper.ShowSQL(on) 368 + } else if masterEngine, err := GetMasterEngine(ctxEngine); err == nil { 369 + masterEngine.ShowSQL(on) 321 370 } 322 371 } 323 372 ··· 374 423 } 375 424 return nil 376 425 } 426 + 427 + // GetMasterEngine extracts the master xorm.Engine from the provided xorm.Engine. 428 + // This handles both direct xorm.Engine cases and engines that implement a Master() method. 429 + func GetMasterEngine(x Engine) (*xorm.Engine, error) { 430 + if getter, ok := x.(interface{ Master() *xorm.Engine }); ok { 431 + return getter.Master(), nil 432 + } 433 + 434 + engine, ok := x.(*xorm.Engine) 435 + if !ok { 436 + return nil, fmt.Errorf("unsupported engine type: %T", x) 437 + } 438 + 439 + return engine, nil 440 + }
+5 -3
models/db/index_test.go
··· 33 33 34 34 func TestSyncMaxResourceIndex(t *testing.T) { 35 35 require.NoError(t, unittest.PrepareTestDatabase()) 36 - xe := unittest.GetXORMEngine() 36 + xe, err := unittest.GetXORMEngine() 37 + require.NoError(t, err) 37 38 require.NoError(t, xe.Sync(&TestIndex{})) 38 39 39 - err := db.SyncMaxResourceIndex(db.DefaultContext, "test_index", 10, 51) 40 + err = db.SyncMaxResourceIndex(db.DefaultContext, "test_index", 10, 51) 40 41 require.NoError(t, err) 41 42 42 43 // sync new max index ··· 88 89 89 90 func TestGetNextResourceIndex(t *testing.T) { 90 91 require.NoError(t, unittest.PrepareTestDatabase()) 91 - xe := unittest.GetXORMEngine() 92 + xe, err := unittest.GetXORMEngine() 93 + require.NoError(t, err) 92 94 require.NoError(t, xe.Sync(&TestIndex{})) 93 95 94 96 // create a new record
+2 -1
models/db/iterate_test.go
··· 17 17 18 18 func TestIterate(t *testing.T) { 19 19 require.NoError(t, unittest.PrepareTestDatabase()) 20 - xe := unittest.GetXORMEngine() 20 + xe, err := unittest.GetXORMEngine() 21 + require.NoError(t, err) 21 22 require.NoError(t, xe.Sync(&repo_model.RepoUnit{})) 22 23 23 24 cnt, err := db.GetEngine(db.DefaultContext).Count(&repo_model.RepoUnit{})
+3 -2
models/db/list_test.go
··· 29 29 30 30 func TestFind(t *testing.T) { 31 31 require.NoError(t, unittest.PrepareTestDatabase()) 32 - xe := unittest.GetXORMEngine() 32 + xe, err := unittest.GetXORMEngine() 33 + require.NoError(t, err) 33 34 require.NoError(t, xe.Sync(&repo_model.RepoUnit{})) 34 35 35 36 var repoUnitCount int 36 - _, err := db.GetEngine(db.DefaultContext).SQL("SELECT COUNT(*) FROM repo_unit").Get(&repoUnitCount) 37 + _, err = db.GetEngine(db.DefaultContext).SQL("SELECT COUNT(*) FROM repo_unit").Get(&repoUnitCount) 37 38 require.NoError(t, err) 38 39 assert.NotEmpty(t, repoUnitCount) 39 40
+10
models/migrations/migrations.go
··· 8 8 "context" 9 9 "fmt" 10 10 11 + "forgejo.org/models/db" 11 12 "forgejo.org/models/forgejo_migrations" 12 13 "forgejo.org/models/migrations/v1_10" 13 14 "forgejo.org/models/migrations/v1_11" ··· 510 511 // Execute Forgejo specific migrations. 511 512 return forgejo_migrations.Migrate(x) 512 513 } 514 + 515 + // WrapperMigrate is a wrapper for Migrate to be called in diagnostics 516 + func WrapperMigrate(e db.Engine) error { 517 + engine, err := db.GetMasterEngine(e) 518 + if err != nil { 519 + return err 520 + } 521 + return Migrate(engine) 522 + }
+4 -1
models/migrations/test/tests.go
··· 175 175 if err := db.InitEngine(context.Background()); err != nil { 176 176 return nil, err 177 177 } 178 - x := unittest.GetXORMEngine() 178 + x, err := unittest.GetXORMEngine() 179 + if err != nil { 180 + return nil, err 181 + } 179 182 return x, nil 180 183 } 181 184
+12 -7
models/unittest/fixtures.go
··· 22 22 var fixturesLoader *testfixtures.Loader 23 23 24 24 // GetXORMEngine gets the XORM engine 25 - func GetXORMEngine(engine ...*xorm.Engine) (x *xorm.Engine) { 25 + func GetXORMEngine(engine ...*xorm.Engine) (x *xorm.Engine, err error) { 26 26 if len(engine) == 1 { 27 - return engine[0] 27 + return engine[0], nil 28 28 } 29 - return db.DefaultContext.(*db.Context).Engine().(*xorm.Engine) 29 + return db.GetMasterEngine(db.DefaultContext.(*db.Context).Engine()) 30 30 } 31 31 32 32 func OverrideFixtures(opts FixturesOptions, engine ...*xorm.Engine) func() { ··· 41 41 42 42 // InitFixtures initialize test fixtures for a test database 43 43 func InitFixtures(opts FixturesOptions, engine ...*xorm.Engine) (err error) { 44 - e := GetXORMEngine(engine...) 44 + e, err := GetXORMEngine(engine...) 45 + if err != nil { 46 + return err 47 + } 45 48 var fixtureOptionFiles func(*testfixtures.Loader) error 46 49 if opts.Dir != "" { 47 50 fixtureOptionFiles = testfixtures.Directory(opts.Dir) ··· 93 96 94 97 // LoadFixtures load fixtures for a test database 95 98 func LoadFixtures(engine ...*xorm.Engine) error { 96 - e := GetXORMEngine(engine...) 97 - var err error 99 + e, err := GetXORMEngine(engine...) 100 + if err != nil { 101 + return err 102 + } 98 103 // (doubt) database transaction conflicts could occur and result in ROLLBACK? just try for a few times. 99 - for i := 0; i < 5; i++ { 104 + for range 5 { 100 105 if err = fixturesLoader.Load(); err == nil { 101 106 break 102 107 }
+133 -32
modules/setting/database.go
··· 10 10 "net/url" 11 11 "os" 12 12 "path/filepath" 13 + "strconv" 13 14 "strings" 14 15 "time" 16 + 17 + "forgejo.org/modules/log" 18 + 19 + "xorm.io/xorm" 15 20 ) 16 21 17 22 var ( ··· 24 29 EnableSQLite3 bool 25 30 26 31 // Database holds the database settings 27 - Database = struct { 28 - Type DatabaseType 29 - Host string 30 - Name string 31 - User string 32 - Passwd string 33 - Schema string 34 - SSLMode string 35 - Path string 36 - LogSQL bool 37 - MysqlCharset string 38 - CharsetCollation string 39 - Timeout int // seconds 40 - SQLiteJournalMode string 41 - DBConnectRetries int 42 - DBConnectBackoff time.Duration 43 - MaxIdleConns int 44 - MaxOpenConns int 45 - ConnMaxIdleTime time.Duration 46 - ConnMaxLifetime time.Duration 47 - IterateBufferSize int 48 - AutoMigration bool 49 - SlowQueryThreshold time.Duration 50 - }{ 32 + Database = DatabaseSettings{ 51 33 Timeout: 500, 52 34 IterateBufferSize: 50, 53 35 } 54 36 ) 55 37 38 + type DatabaseSettings struct { 39 + Type DatabaseType 40 + Host string 41 + HostPrimary string 42 + HostReplica string 43 + LoadBalancePolicy string 44 + LoadBalanceWeights string 45 + Name string 46 + User string 47 + Passwd string 48 + Schema string 49 + SSLMode string 50 + Path string 51 + LogSQL bool 52 + MysqlCharset string 53 + CharsetCollation string 54 + Timeout int // seconds 55 + SQLiteJournalMode string 56 + DBConnectRetries int 57 + DBConnectBackoff time.Duration 58 + MaxIdleConns int 59 + MaxOpenConns int 60 + ConnMaxIdleTime time.Duration 61 + ConnMaxLifetime time.Duration 62 + IterateBufferSize int 63 + AutoMigration bool 64 + SlowQueryThreshold time.Duration 65 + } 66 + 56 67 // LoadDBSetting loads the database settings 57 68 func LoadDBSetting() { 58 69 loadDBSetting(CfgProvider) ··· 63 74 Database.Type = DatabaseType(sec.Key("DB_TYPE").String()) 64 75 65 76 Database.Host = sec.Key("HOST").String() 77 + Database.HostPrimary = sec.Key("HOST_PRIMARY").String() 78 + Database.HostReplica = sec.Key("HOST_REPLICA").String() 79 + Database.LoadBalancePolicy = sec.Key("LOAD_BALANCE_POLICY").String() 80 + Database.LoadBalanceWeights = sec.Key("LOAD_BALANCE_WEIGHTS").String() 66 81 Database.Name = sec.Key("NAME").String() 67 82 Database.User = sec.Key("USER").String() 68 83 if len(Database.Passwd) == 0 { ··· 99 114 } 100 115 } 101 116 102 - // DBConnStr returns database connection string 103 - func DBConnStr() (string, error) { 117 + // DBMasterConnStr returns the connection string for the master (primary) database. 118 + // If a primary host is defined in the configuration, it is used; 119 + // otherwise, it falls back to Database.Host. 120 + // Returns an error if no master host is provided but a slave is defined. 121 + func DBMasterConnStr() (string, error) { 122 + var host string 123 + if Database.HostPrimary != "" { 124 + host = Database.HostPrimary 125 + } else { 126 + host = Database.Host 127 + } 128 + if host == "" && Database.HostReplica != "" { 129 + return "", errors.New("master host is not defined while slave is defined; cannot proceed") 130 + } 131 + 132 + // For SQLite, no host is needed 133 + if host == "" && !Database.Type.IsSQLite3() { 134 + return "", errors.New("no database host defined") 135 + } 136 + 137 + return dbConnStrWithHost(host) 138 + } 139 + 140 + // DBSlaveConnStrs returns one or more connection strings for the replica databases. 141 + // If a replica host is defined (possibly as a comma-separated list) then those DSNs are returned. 142 + // Otherwise, this function falls back to the master DSN (with a warning log). 143 + func DBSlaveConnStrs() ([]string, error) { 144 + var dsns []string 145 + if Database.HostReplica != "" { 146 + // support multiple replica hosts separated by commas 147 + replicas := strings.SplitSeq(Database.HostReplica, ",") 148 + for r := range replicas { 149 + trimmed := strings.TrimSpace(r) 150 + if trimmed == "" { 151 + continue 152 + } 153 + dsn, err := dbConnStrWithHost(trimmed) 154 + if err != nil { 155 + return nil, err 156 + } 157 + dsns = append(dsns, dsn) 158 + } 159 + } 160 + // Fall back to master if no slave DSN was provided. 161 + if len(dsns) == 0 { 162 + master, err := DBMasterConnStr() 163 + if err != nil { 164 + return nil, err 165 + } 166 + log.Debug("Database: No dedicated replica host defined; falling back to primary DSN for replica connections") 167 + dsns = append(dsns, master) 168 + } 169 + return dsns, nil 170 + } 171 + 172 + func BuildLoadBalancePolicy(settings *DatabaseSettings, slaveEngines []*xorm.Engine) xorm.GroupPolicy { 173 + var policy xorm.GroupPolicy 174 + switch settings.LoadBalancePolicy { // Use the settings parameter directly 175 + case "WeightRandom": 176 + var weights []int 177 + if settings.LoadBalanceWeights != "" { // Use the settings parameter directly 178 + for part := range strings.SplitSeq(settings.LoadBalanceWeights, ",") { 179 + w, err := strconv.Atoi(strings.TrimSpace(part)) 180 + if err != nil { 181 + w = 1 // use a default weight if conversion fails 182 + } 183 + weights = append(weights, w) 184 + } 185 + } 186 + // If no valid weights were provided, default each slave to weight 1 187 + if len(weights) == 0 { 188 + weights = make([]int, len(slaveEngines)) 189 + for i := range weights { 190 + weights[i] = 1 191 + } 192 + } 193 + policy = xorm.WeightRandomPolicy(weights) 194 + case "RoundRobin": 195 + policy = xorm.RoundRobinPolicy() 196 + default: 197 + policy = xorm.RandomPolicy() 198 + } 199 + return policy 200 + } 201 + 202 + // dbConnStrWithHost constructs the connection string, given a host value. 203 + func dbConnStrWithHost(host string) (string, error) { 104 204 var connStr string 105 205 paramSep := "?" 106 206 if strings.Contains(Database.Name, paramSep) { ··· 109 209 switch Database.Type { 110 210 case "mysql": 111 211 connType := "tcp" 112 - if len(Database.Host) > 0 && Database.Host[0] == '/' { // looks like a unix socket 212 + // if the host starts with '/' it is assumed to be a unix socket path 213 + if len(host) > 0 && host[0] == '/' { 113 214 connType = "unix" 114 215 } 115 216 tls := Database.SSLMode 116 - if tls == "disable" { // allow (Postgres-inspired) default value to work in MySQL 217 + // allow the "disable" value (borrowed from Postgres defaults) to behave as false 218 + if tls == "disable" { 117 219 tls = "false" 118 220 } 119 221 connStr = fmt.Sprintf("%s:%s@%s(%s)/%s%sparseTime=true&tls=%s", 120 - Database.User, Database.Passwd, connType, Database.Host, Database.Name, paramSep, tls) 222 + Database.User, Database.Passwd, connType, host, Database.Name, paramSep, tls) 121 223 case "postgres": 122 - connStr = getPostgreSQLConnectionString(Database.Host, Database.User, Database.Passwd, Database.Name, Database.SSLMode) 224 + connStr = getPostgreSQLConnectionString(host, Database.User, Database.Passwd, Database.Name, Database.SSLMode) 123 225 case "sqlite3": 124 226 if !EnableSQLite3 { 125 227 return "", errors.New("this Gitea binary was not built with SQLite3 support") 126 228 } 127 229 if err := os.MkdirAll(filepath.Dir(Database.Path), os.ModePerm); err != nil { 128 - return "", fmt.Errorf("Failed to create directories: %w", err) 230 + return "", fmt.Errorf("failed to create directories: %w", err) 129 231 } 130 232 journalMode := "" 131 233 if Database.SQLiteJournalMode != "" { ··· 136 238 default: 137 239 return "", fmt.Errorf("unknown database type: %s", Database.Type) 138 240 } 139 - 140 241 return connStr, nil 141 242 } 142 243
+102
modules/setting/database_test.go
··· 4 4 package setting 5 5 6 6 import ( 7 + "strings" 7 8 "testing" 8 9 9 10 "github.com/stretchr/testify/assert" ··· 107 108 assert.Equal(t, test.Output, connStr) 108 109 } 109 110 } 111 + 112 + func getPostgreSQLEngineGroupConnectionStrings(primaryHost, replicaHosts, user, passwd, name, sslmode string) (string, []string) { 113 + // Determine the primary connection string. 114 + primary := primaryHost 115 + if strings.TrimSpace(primary) == "" { 116 + primary = "127.0.0.1:5432" 117 + } 118 + primaryConn := getPostgreSQLConnectionString(primary, user, passwd, name, sslmode) 119 + 120 + // Build the replica connection strings. 121 + replicaConns := []string{} 122 + if strings.TrimSpace(replicaHosts) != "" { 123 + // Split comma-separated replica host values. 124 + hosts := strings.Split(replicaHosts, ",") 125 + for _, h := range hosts { 126 + trimmed := strings.TrimSpace(h) 127 + if trimmed != "" { 128 + replicaConns = append(replicaConns, 129 + getPostgreSQLConnectionString(trimmed, user, passwd, name, sslmode)) 130 + } 131 + } 132 + } 133 + 134 + return primaryConn, replicaConns 135 + } 136 + 137 + func Test_getPostgreSQLEngineGroupConnectionStrings(t *testing.T) { 138 + tests := []struct { 139 + primaryHost string // primary host setting (e.g. "localhost" or "[::1]:1234") 140 + replicaHosts string // comma-separated replica hosts (e.g. "replica1,replica2:2345") 141 + user string 142 + passwd string 143 + name string 144 + sslmode string 145 + outputPrimary string 146 + outputReplicas []string 147 + }{ 148 + { 149 + // No primary override (empty => default) and no replicas. 150 + primaryHost: "", 151 + replicaHosts: "", 152 + user: "", 153 + passwd: "", 154 + name: "", 155 + sslmode: "", 156 + outputPrimary: "postgres://:@127.0.0.1:5432?sslmode=", 157 + outputReplicas: []string{}, 158 + }, 159 + { 160 + // Primary set and one replica. 161 + primaryHost: "localhost", 162 + replicaHosts: "replicahost", 163 + user: "user", 164 + passwd: "pass", 165 + name: "gitea", 166 + sslmode: "disable", 167 + outputPrimary: "postgres://user:pass@localhost:5432/gitea?sslmode=disable", 168 + outputReplicas: []string{"postgres://user:pass@replicahost:5432/gitea?sslmode=disable"}, 169 + }, 170 + { 171 + // Primary with explicit port; multiple replicas (one without and one with an explicit port). 172 + primaryHost: "localhost:5433", 173 + replicaHosts: "replica1,replica2:5434", 174 + user: "test", 175 + passwd: "secret", 176 + name: "db", 177 + sslmode: "require", 178 + outputPrimary: "postgres://test:secret@localhost:5433/db?sslmode=require", 179 + outputReplicas: []string{ 180 + "postgres://test:secret@replica1:5432/db?sslmode=require", 181 + "postgres://test:secret@replica2:5434/db?sslmode=require", 182 + }, 183 + }, 184 + { 185 + // IPv6 addresses for primary and replica. 186 + primaryHost: "[::1]:1234", 187 + replicaHosts: "[::2]:2345", 188 + user: "ipv6", 189 + passwd: "ipv6pass", 190 + name: "ipv6db", 191 + sslmode: "disable", 192 + outputPrimary: "postgres://ipv6:ipv6pass@[::1]:1234/ipv6db?sslmode=disable", 193 + outputReplicas: []string{ 194 + "postgres://ipv6:ipv6pass@[::2]:2345/ipv6db?sslmode=disable", 195 + }, 196 + }, 197 + } 198 + 199 + for _, test := range tests { 200 + primary, replicas := getPostgreSQLEngineGroupConnectionStrings( 201 + test.primaryHost, 202 + test.replicaHosts, 203 + test.user, 204 + test.passwd, 205 + test.name, 206 + test.sslmode, 207 + ) 208 + assert.Equal(t, test.outputPrimary, primary) 209 + assert.Equal(t, test.outputReplicas, replicas) 210 + } 211 + }
+3
modules/testlogger/testlogger.go
··· 364 364 // TestDatabaseCollation 365 365 `[E] [Error SQL Query] INSERT INTO test_collation_tbl (txt) VALUES ('main') []`, 366 366 367 + // Test_CmdForgejo_Actions 368 + `DB: No dedicated replica host defined; falling back to primary DSN for replica connections`, 369 + 367 370 // TestDevtestErrorpages 368 371 `ErrorPage() [E] Example error: Example error`, 369 372 }
+1 -1
routers/common/db.go
··· 28 28 default: 29 29 } 30 30 log.Info("ORM engine initialization attempt #%d/%d...", i+1, setting.Database.DBConnectRetries) 31 - if err = db.InitEngineWithMigration(ctx, migrateWithSetting); err == nil { 31 + if err = db.InitEngineWithMigration(ctx, func(eng db.Engine) error { return migrateWithSetting(eng.(*xorm.Engine)) }); err == nil { 32 32 break 33 33 } else if i == setting.Database.DBConnectRetries-1 { 34 34 return err
+3 -2
routers/install/install.go
··· 361 361 } 362 362 363 363 // Init the engine with migration 364 - if err = db.InitEngineWithMigration(ctx, migrations.Migrate); err != nil { 364 + // Wrap migrations.Migrate into a function of type func(db.Engine) error to fix diagnostics. 365 + if err = db.InitEngineWithMigration(ctx, migrations.WrapperMigrate); err != nil { 365 366 db.UnsetDefaultEngine() 366 367 ctx.Data["Err_DbSetting"] = true 367 368 ctx.RenderWithErr(ctx.Tr("install.invalid_db_setting", err), tplInstall, &form) ··· 587 588 588 589 go func() { 589 590 // Sleep for a while to make sure the user's browser has loaded the post-install page and its assets (images, css, js) 590 - // What if this duration is not long enough? That's impossible -- if the user can't load the simple page in time, how could they install or use Gitea in the future .... 591 + // What if this duration is not long enough? That's impossible -- if the user can't load the simple page in time, how could they install or use Forgejo in the future .... 591 592 time.Sleep(3 * time.Second) 592 593 593 594 // Now get the http.Server from this request and shut it down
+8 -1
services/doctor/dbconsistency.go
··· 78 78 79 79 func checkDBConsistency(ctx context.Context, logger log.Logger, autofix bool) error { 80 80 // make sure DB version is up-to-date 81 - if err := db.InitEngineWithMigration(ctx, migrations.EnsureUpToDate); err != nil { 81 + ensureUpToDateWrapper := func(e db.Engine) error { 82 + engine, err := db.GetMasterEngine(e) 83 + if err != nil { 84 + return err 85 + } 86 + return migrations.EnsureUpToDate(engine) 87 + } 88 + if err := db.InitEngineWithMigration(ctx, ensureUpToDateWrapper); err != nil { 82 89 logger.Critical("Model version on the database does not match the current Gitea version. Model consistency will not be checked until the database is upgraded") 83 90 return err 84 91 }
+8 -2
services/doctor/dbversion.go
··· 9 9 "forgejo.org/models/db" 10 10 "forgejo.org/models/migrations" 11 11 "forgejo.org/modules/log" 12 + 13 + "xorm.io/xorm" 12 14 ) 13 15 14 16 func checkDBVersion(ctx context.Context, logger log.Logger, autofix bool) error { 15 17 logger.Info("Expected database version: %d", migrations.ExpectedDBVersion()) 16 - if err := db.InitEngineWithMigration(ctx, migrations.EnsureUpToDate); err != nil { 18 + if err := db.InitEngineWithMigration(ctx, func(eng db.Engine) error { 19 + return migrations.EnsureUpToDate(eng.(*xorm.Engine)) 20 + }); err != nil { 17 21 if !autofix { 18 22 logger.Critical("Error: %v during ensure up to date", err) 19 23 return err ··· 21 25 logger.Warn("Got Error: %v during ensure up to date", err) 22 26 logger.Warn("Attempting to migrate to the latest DB version to fix this.") 23 27 24 - err = db.InitEngineWithMigration(ctx, migrations.Migrate) 28 + err = db.InitEngineWithMigration(ctx, func(eng db.Engine) error { 29 + return migrations.Migrate(eng.(*xorm.Engine)) 30 + }) 25 31 if err != nil { 26 32 logger.Critical("Error: %v during migration", err) 27 33 }
+4 -3
tests/integration/db_collation_test.go
··· 16 16 17 17 "github.com/stretchr/testify/assert" 18 18 "github.com/stretchr/testify/require" 19 - "xorm.io/xorm" 20 19 ) 21 20 22 21 type TestCollationTbl struct { ··· 48 47 } 49 48 50 49 func TestDatabaseCollation(t *testing.T) { 51 - x := db.GetEngine(db.DefaultContext).(*xorm.Engine) 50 + engine, err := db.GetMasterEngine(db.GetEngine(db.DefaultContext)) 51 + require.NoError(t, err) 52 + x := engine 52 53 53 54 // all created tables should use case-sensitive collation by default 54 55 _, _ = x.Exec("DROP TABLE IF EXISTS test_collation_tbl") 55 - err := x.Sync(&TestCollationTbl{}) 56 + err = x.Sync(&TestCollationTbl{}) 56 57 require.NoError(t, err) 57 58 _, _ = x.Exec("INSERT INTO test_collation_tbl (txt) VALUES ('main')") 58 59 _, _ = x.Exec("INSERT INTO test_collation_tbl (txt) VALUES ('Main')") // case-sensitive, so it inserts a new row
+20 -7
tests/integration/migration-test/migration_test.go
··· 278 278 279 279 setting.InitSQLLoggersForCli(log.INFO) 280 280 281 - err := db.InitEngineWithMigration(t.Context(), wrappedMigrate) 281 + err := db.InitEngineWithMigration(t.Context(), func(e db.Engine) error { 282 + engine, err := db.GetMasterEngine(e) 283 + if err != nil { 284 + return err 285 + } 286 + currentEngine = engine 287 + return wrappedMigrate(engine) 288 + }) 282 289 require.NoError(t, err) 283 290 currentEngine.Close() 284 291 285 292 beans, _ := db.NamesToBean() 286 293 287 - err = db.InitEngineWithMigration(t.Context(), func(x *xorm.Engine) error { 288 - currentEngine = x 289 - return migrate_base.RecreateTables(beans...)(x) 294 + err = db.InitEngineWithMigration(t.Context(), func(e db.Engine) error { 295 + currentEngine, err = db.GetMasterEngine(e) 296 + if err != nil { 297 + return err 298 + } 299 + return migrate_base.RecreateTables(beans...)(currentEngine) 290 300 }) 291 301 require.NoError(t, err) 292 302 currentEngine.Close() 293 303 294 304 // We do this a second time to ensure that there is not a problem with retained indices 295 - err = db.InitEngineWithMigration(t.Context(), func(x *xorm.Engine) error { 296 - currentEngine = x 297 - return migrate_base.RecreateTables(beans...)(x) 305 + err = db.InitEngineWithMigration(t.Context(), func(e db.Engine) error { 306 + currentEngine, err = db.GetMasterEngine(e) 307 + if err != nil { 308 + return err 309 + } 310 + return migrate_base.RecreateTables(beans...)(currentEngine) 298 311 }) 299 312 require.NoError(t, err) 300 313