From dd136858f1ea40ad3c94191d647487fa4f31926c Mon Sep 17 00:00:00 2001 From: Daniel Baumann Date: Fri, 18 Oct 2024 20:33:49 +0200 Subject: Adding upstream version 9.0.0. Signed-off-by: Daniel Baumann --- models/migrations/base/db.go | 436 ++++++++++++++++++++++++++++++++++++ models/migrations/base/db_test.go | 98 ++++++++ models/migrations/base/hash.go | 16 ++ models/migrations/base/main_test.go | 14 ++ 4 files changed, 564 insertions(+) create mode 100644 models/migrations/base/db.go create mode 100644 models/migrations/base/db_test.go create mode 100644 models/migrations/base/hash.go create mode 100644 models/migrations/base/main_test.go (limited to 'models/migrations/base') diff --git a/models/migrations/base/db.go b/models/migrations/base/db.go new file mode 100644 index 0000000..333fa31 --- /dev/null +++ b/models/migrations/base/db.go @@ -0,0 +1,436 @@ +// Copyright 2022 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package base + +import ( + "errors" + "fmt" + "reflect" + "regexp" + "strings" + + "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/modules/setting" + + "xorm.io/xorm" + "xorm.io/xorm/schemas" +) + +// RecreateTables will recreate the tables for the provided beans using the newly provided bean definition and move all data to that new table +// WARNING: YOU MUST PROVIDE THE FULL BEAN DEFINITION +func RecreateTables(beans ...any) func(*xorm.Engine) error { + return func(x *xorm.Engine) error { + sess := x.NewSession() + defer sess.Close() + if err := sess.Begin(); err != nil { + return err + } + sess = sess.StoreEngine("InnoDB") + for _, bean := range beans { + log.Info("Recreating Table: %s for Bean: %s", x.TableName(bean), reflect.Indirect(reflect.ValueOf(bean)).Type().Name()) + if err := RecreateTable(sess, bean); err != nil { + return err + } + } + return sess.Commit() + } +} + +// RecreateTable will recreate the table using the newly provided bean definition and move all data to that new table +// WARNING: YOU MUST PROVIDE THE FULL BEAN DEFINITION +// WARNING: YOU MUST COMMIT THE SESSION AT THE END +func RecreateTable(sess *xorm.Session, bean any) error { + // TODO: This will not work if there are foreign keys + + tableName := sess.Engine().TableName(bean) + tempTableName := fmt.Sprintf("tmp_recreate__%s", tableName) + + // We need to move the old table away and create a new one with the correct columns + // We will need to do this in stages to prevent data loss + // + // First create the temporary table + if err := sess.Table(tempTableName).CreateTable(bean); err != nil { + log.Error("Unable to create table %s. Error: %v", tempTableName, err) + return err + } + + if err := sess.Table(tempTableName).CreateUniques(bean); err != nil { + log.Error("Unable to create uniques for table %s. Error: %v", tempTableName, err) + return err + } + + if err := sess.Table(tempTableName).CreateIndexes(bean); err != nil { + log.Error("Unable to create indexes for table %s. Error: %v", tempTableName, err) + return err + } + + // Work out the column names from the bean - these are the columns to select from the old table and install into the new table + table, err := sess.Engine().TableInfo(bean) + if err != nil { + log.Error("Unable to get table info. Error: %v", err) + + return err + } + newTableColumns := table.Columns() + if len(newTableColumns) == 0 { + return fmt.Errorf("no columns in new table") + } + hasID := false + for _, column := range newTableColumns { + hasID = hasID || (column.IsPrimaryKey && column.IsAutoIncrement) + } + + sqlStringBuilder := &strings.Builder{} + _, _ = sqlStringBuilder.WriteString("INSERT INTO `") + _, _ = sqlStringBuilder.WriteString(tempTableName) + _, _ = sqlStringBuilder.WriteString("` (`") + _, _ = sqlStringBuilder.WriteString(newTableColumns[0].Name) + _, _ = sqlStringBuilder.WriteString("`") + for _, column := range newTableColumns[1:] { + _, _ = sqlStringBuilder.WriteString(", `") + _, _ = sqlStringBuilder.WriteString(column.Name) + _, _ = sqlStringBuilder.WriteString("`") + } + _, _ = sqlStringBuilder.WriteString(")") + _, _ = sqlStringBuilder.WriteString(" SELECT ") + if newTableColumns[0].Default != "" { + _, _ = sqlStringBuilder.WriteString("COALESCE(`") + _, _ = sqlStringBuilder.WriteString(newTableColumns[0].Name) + _, _ = sqlStringBuilder.WriteString("`, ") + _, _ = sqlStringBuilder.WriteString(newTableColumns[0].Default) + _, _ = sqlStringBuilder.WriteString(")") + } else { + _, _ = sqlStringBuilder.WriteString("`") + _, _ = sqlStringBuilder.WriteString(newTableColumns[0].Name) + _, _ = sqlStringBuilder.WriteString("`") + } + + for _, column := range newTableColumns[1:] { + if column.Default != "" { + _, _ = sqlStringBuilder.WriteString(", COALESCE(`") + _, _ = sqlStringBuilder.WriteString(column.Name) + _, _ = sqlStringBuilder.WriteString("`, ") + _, _ = sqlStringBuilder.WriteString(column.Default) + _, _ = sqlStringBuilder.WriteString(")") + } else { + _, _ = sqlStringBuilder.WriteString(", `") + _, _ = sqlStringBuilder.WriteString(column.Name) + _, _ = sqlStringBuilder.WriteString("`") + } + } + _, _ = sqlStringBuilder.WriteString(" FROM `") + _, _ = sqlStringBuilder.WriteString(tableName) + _, _ = sqlStringBuilder.WriteString("`") + + if _, err := sess.Exec(sqlStringBuilder.String()); err != nil { + log.Error("Unable to set copy data in to temp table %s. Error: %v", tempTableName, err) + return err + } + + switch { + case setting.Database.Type.IsSQLite3(): + // SQLite will drop all the constraints on the old table + if _, err := sess.Exec(fmt.Sprintf("DROP TABLE `%s`", tableName)); err != nil { + log.Error("Unable to drop old table %s. Error: %v", tableName, err) + return err + } + + if err := sess.Table(tempTableName).DropIndexes(bean); err != nil { + log.Error("Unable to drop indexes on temporary table %s. Error: %v", tempTableName, err) + return err + } + + if _, err := sess.Exec(fmt.Sprintf("ALTER TABLE `%s` RENAME TO `%s`", tempTableName, tableName)); err != nil { + log.Error("Unable to rename %s to %s. Error: %v", tempTableName, tableName, err) + return err + } + + if err := sess.Table(tableName).CreateIndexes(bean); err != nil { + log.Error("Unable to recreate indexes on table %s. Error: %v", tableName, err) + return err + } + + if err := sess.Table(tableName).CreateUniques(bean); err != nil { + log.Error("Unable to recreate uniques on table %s. Error: %v", tableName, err) + return err + } + + case setting.Database.Type.IsMySQL(): + // MySQL will drop all the constraints on the old table + if _, err := sess.Exec(fmt.Sprintf("DROP TABLE `%s`", tableName)); err != nil { + log.Error("Unable to drop old table %s. Error: %v", tableName, err) + return err + } + + if err := sess.Table(tempTableName).DropIndexes(bean); err != nil { + log.Error("Unable to drop indexes on temporary table %s. Error: %v", tempTableName, err) + return err + } + + // SQLite and MySQL will move all the constraints from the temporary table to the new table + if _, err := sess.Exec(fmt.Sprintf("ALTER TABLE `%s` RENAME TO `%s`", tempTableName, tableName)); err != nil { + log.Error("Unable to rename %s to %s. Error: %v", tempTableName, tableName, err) + return err + } + + if err := sess.Table(tableName).CreateIndexes(bean); err != nil { + log.Error("Unable to recreate indexes on table %s. Error: %v", tableName, err) + return err + } + + if err := sess.Table(tableName).CreateUniques(bean); err != nil { + log.Error("Unable to recreate uniques on table %s. Error: %v", tableName, err) + return err + } + case setting.Database.Type.IsPostgreSQL(): + var originalSequences []string + type sequenceData struct { + LastValue int `xorm:"'last_value'"` + IsCalled bool `xorm:"'is_called'"` + } + sequenceMap := map[string]sequenceData{} + + schema := sess.Engine().Dialect().URI().Schema + sess.Engine().SetSchema("") + if err := sess.Table("information_schema.sequences").Cols("sequence_name").Where("sequence_name LIKE ? || '_%' AND sequence_catalog = ?", tableName, setting.Database.Name).Find(&originalSequences); err != nil { + log.Error("Unable to rename %s to %s. Error: %v", tempTableName, tableName, err) + return err + } + sess.Engine().SetSchema(schema) + + for _, sequence := range originalSequences { + sequenceData := sequenceData{} + if _, err := sess.Table(sequence).Cols("last_value", "is_called").Get(&sequenceData); err != nil { + log.Error("Unable to get last_value and is_called from %s. Error: %v", sequence, err) + return err + } + sequenceMap[sequence] = sequenceData + } + + // CASCADE causes postgres to drop all the constraints on the old table + if _, err := sess.Exec(fmt.Sprintf("DROP TABLE `%s` CASCADE", tableName)); err != nil { + log.Error("Unable to drop old table %s. Error: %v", tableName, err) + return err + } + + // CASCADE causes postgres to move all the constraints from the temporary table to the new table + if _, err := sess.Exec(fmt.Sprintf("ALTER TABLE `%s` RENAME TO `%s`", tempTableName, tableName)); err != nil { + log.Error("Unable to rename %s to %s. Error: %v", tempTableName, tableName, err) + return err + } + + var indices []string + sess.Engine().SetSchema("") + if err := sess.Table("pg_indexes").Cols("indexname").Where("tablename = ? ", tableName).Find(&indices); err != nil { + log.Error("Unable to rename %s to %s. Error: %v", tempTableName, tableName, err) + return err + } + sess.Engine().SetSchema(schema) + + for _, index := range indices { + newIndexName := strings.Replace(index, "tmp_recreate__", "", 1) + if _, err := sess.Exec(fmt.Sprintf("ALTER INDEX `%s` RENAME TO `%s`", index, newIndexName)); err != nil { + log.Error("Unable to rename %s to %s. Error: %v", index, newIndexName, err) + return err + } + } + + var sequences []string + sess.Engine().SetSchema("") + if err := sess.Table("information_schema.sequences").Cols("sequence_name").Where("sequence_name LIKE 'tmp_recreate__' || ? || '_%' AND sequence_catalog = ?", tableName, setting.Database.Name).Find(&sequences); err != nil { + log.Error("Unable to rename %s to %s. Error: %v", tempTableName, tableName, err) + return err + } + sess.Engine().SetSchema(schema) + + for _, sequence := range sequences { + newSequenceName := strings.Replace(sequence, "tmp_recreate__", "", 1) + if _, err := sess.Exec(fmt.Sprintf("ALTER SEQUENCE `%s` RENAME TO `%s`", sequence, newSequenceName)); err != nil { + log.Error("Unable to rename %s sequence to %s. Error: %v", sequence, newSequenceName, err) + return err + } + val, ok := sequenceMap[newSequenceName] + if newSequenceName == tableName+"_id_seq" { + if ok && val.LastValue != 0 { + if _, err := sess.Exec(fmt.Sprintf("SELECT setval('%s', %d, %t)", newSequenceName, val.LastValue, val.IsCalled)); err != nil { + log.Error("Unable to reset %s to %d. Error: %v", newSequenceName, val, err) + return err + } + } else { + // We're going to try to guess this + if _, err := sess.Exec(fmt.Sprintf("SELECT setval('%s', COALESCE((SELECT MAX(id)+1 FROM `%s`), 1), false)", newSequenceName, tableName)); err != nil { + log.Error("Unable to reset %s. Error: %v", newSequenceName, err) + return err + } + } + } else if ok { + if _, err := sess.Exec(fmt.Sprintf("SELECT setval('%s', %d, %t)", newSequenceName, val.LastValue, val.IsCalled)); err != nil { + log.Error("Unable to reset %s to %d. Error: %v", newSequenceName, val, err) + return err + } + } + } + + default: + log.Fatal("Unrecognized DB") + } + return nil +} + +// WARNING: YOU MUST COMMIT THE SESSION AT THE END +func DropTableColumns(sess *xorm.Session, tableName string, columnNames ...string) (err error) { + if tableName == "" || len(columnNames) == 0 { + return nil + } + // TODO: This will not work if there are foreign keys + + switch { + case setting.Database.Type.IsSQLite3(): + // First drop the indexes on the columns + res, errIndex := sess.Query(fmt.Sprintf("PRAGMA index_list(`%s`)", tableName)) + if errIndex != nil { + return errIndex + } + for _, row := range res { + indexName := row["name"] + indexRes, err := sess.Query(fmt.Sprintf("PRAGMA index_info(`%s`)", indexName)) + if err != nil { + return err + } + if len(indexRes) != 1 { + continue + } + indexColumn := string(indexRes[0]["name"]) + for _, name := range columnNames { + if name == indexColumn { + _, err := sess.Exec(fmt.Sprintf("DROP INDEX `%s`", indexName)) + if err != nil { + return err + } + } + } + } + + // Here we need to get the columns from the original table + sql := fmt.Sprintf("SELECT sql FROM sqlite_master WHERE tbl_name='%s' and type='table'", tableName) + res, err := sess.Query(sql) + if err != nil { + return err + } + tableSQL := string(res[0]["sql"]) + + // Get the string offset for column definitions: `CREATE TABLE ( column-definitions... )` + columnDefinitionsIndex := strings.Index(tableSQL, "(") + if columnDefinitionsIndex < 0 { + return errors.New("couldn't find column definitions") + } + + // Separate out the column definitions + tableSQL = tableSQL[columnDefinitionsIndex:] + + // Remove the required columnNames + for _, name := range columnNames { + tableSQL = regexp.MustCompile(regexp.QuoteMeta("`"+name+"`")+"[^`,)]*?[,)]").ReplaceAllString(tableSQL, "") + } + + // Ensure the query is ended properly + tableSQL = strings.TrimSpace(tableSQL) + if tableSQL[len(tableSQL)-1] != ')' { + if tableSQL[len(tableSQL)-1] == ',' { + tableSQL = tableSQL[:len(tableSQL)-1] + } + tableSQL += ")" + } + + // Find all the columns in the table + columns := regexp.MustCompile("`([^`]*)`").FindAllString(tableSQL, -1) + + tableSQL = fmt.Sprintf("CREATE TABLE `new_%s_new` ", tableName) + tableSQL + if _, err := sess.Exec(tableSQL); err != nil { + return err + } + + // Now restore the data + columnsSeparated := strings.Join(columns, ",") + insertSQL := fmt.Sprintf("INSERT INTO `new_%s_new` (%s) SELECT %s FROM %s", tableName, columnsSeparated, columnsSeparated, tableName) + if _, err := sess.Exec(insertSQL); err != nil { + return err + } + + // Now drop the old table + if _, err := sess.Exec(fmt.Sprintf("DROP TABLE `%s`", tableName)); err != nil { + return err + } + + // Rename the table + if _, err := sess.Exec(fmt.Sprintf("ALTER TABLE `new_%s_new` RENAME TO `%s`", tableName, tableName)); err != nil { + return err + } + + case setting.Database.Type.IsPostgreSQL(): + cols := "" + for _, col := range columnNames { + if cols != "" { + cols += ", " + } + cols += "DROP COLUMN `" + col + "` CASCADE" + } + if _, err := sess.Exec(fmt.Sprintf("ALTER TABLE `%s` %s", tableName, cols)); err != nil { + return fmt.Errorf("Drop table `%s` columns %v: %v", tableName, columnNames, err) + } + case setting.Database.Type.IsMySQL(): + // Drop indexes on columns first + sql := fmt.Sprintf("SHOW INDEX FROM %s WHERE column_name IN ('%s')", tableName, strings.Join(columnNames, "','")) + res, err := sess.Query(sql) + if err != nil { + return err + } + for _, index := range res { + indexName := index["column_name"] + if len(indexName) > 0 { + _, err := sess.Exec(fmt.Sprintf("DROP INDEX `%s` ON `%s`", indexName, tableName)) + if err != nil { + return err + } + } + } + + // Now drop the columns + cols := "" + for _, col := range columnNames { + if cols != "" { + cols += ", " + } + cols += "DROP COLUMN `" + col + "`" + } + if _, err := sess.Exec(fmt.Sprintf("ALTER TABLE `%s` %s", tableName, cols)); err != nil { + return fmt.Errorf("Drop table `%s` columns %v: %v", tableName, columnNames, err) + } + default: + log.Fatal("Unrecognized DB") + } + + return nil +} + +// ModifyColumn will modify column's type or other property. SQLITE is not supported +func ModifyColumn(x *xorm.Engine, tableName string, col *schemas.Column) error { + var indexes map[string]*schemas.Index + var err error + + defer func() { + for _, index := range indexes { + _, err = x.Exec(x.Dialect().CreateIndexSQL(tableName, index)) + if err != nil { + log.Error("Create index %s on table %s failed: %v", index.Name, tableName, err) + } + } + }() + + alterSQL := x.Dialect().ModifyColumnSQL(tableName, col) + if _, err := x.Exec(alterSQL); err != nil { + return err + } + return nil +} diff --git a/models/migrations/base/db_test.go b/models/migrations/base/db_test.go new file mode 100644 index 0000000..4010a14 --- /dev/null +++ b/models/migrations/base/db_test.go @@ -0,0 +1,98 @@ +// Copyright 2022 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package base + +import ( + "testing" + + migrations_tests "code.gitea.io/gitea/models/migrations/test" + "code.gitea.io/gitea/modules/timeutil" + + "xorm.io/xorm/names" +) + +func Test_DropTableColumns(t *testing.T) { + x, deferable := migrations_tests.PrepareTestEnv(t, 0) + if x == nil || t.Failed() { + defer deferable() + return + } + defer deferable() + + type DropTest struct { + ID int64 `xorm:"pk autoincr"` + FirstColumn string + ToDropColumn string `xorm:"unique"` + AnotherColumn int64 + CreatedUnix timeutil.TimeStamp `xorm:"INDEX created"` + UpdatedUnix timeutil.TimeStamp `xorm:"INDEX updated"` + } + + columns := []string{ + "first_column", + "to_drop_column", + "another_column", + "created_unix", + "updated_unix", + } + + x.SetMapper(names.GonicMapper{}) + + for i := range columns { + if err := x.Sync(new(DropTest)); err != nil { + t.Errorf("unable to create DropTest table: %v", err) + return + } + + sess := x.NewSession() + if err := sess.Begin(); err != nil { + sess.Close() + t.Errorf("unable to begin transaction: %v", err) + return + } + if err := DropTableColumns(sess, "drop_test", columns[i:]...); err != nil { + sess.Close() + t.Errorf("Unable to drop columns[%d:]: %s from drop_test: %v", i, columns[i:], err) + return + } + if err := sess.Commit(); err != nil { + sess.Close() + t.Errorf("unable to commit transaction: %v", err) + return + } + sess.Close() + if err := x.DropTables(new(DropTest)); err != nil { + t.Errorf("unable to drop table: %v", err) + return + } + for j := range columns[i+1:] { + if err := x.Sync(new(DropTest)); err != nil { + t.Errorf("unable to create DropTest table: %v", err) + return + } + dropcols := append([]string{columns[i]}, columns[j+i+1:]...) + sess := x.NewSession() + if err := sess.Begin(); err != nil { + sess.Close() + t.Errorf("unable to begin transaction: %v", err) + return + } + if err := DropTableColumns(sess, "drop_test", dropcols...); err != nil { + sess.Close() + t.Errorf("Unable to drop columns: %s from drop_test: %v", dropcols, err) + return + } + if err := sess.Commit(); err != nil { + sess.Close() + t.Errorf("unable to commit transaction: %v", err) + return + } + sess.Close() + if err := x.DropTables(new(DropTest)); err != nil { + t.Errorf("unable to drop table: %v", err) + return + } + } + } +} diff --git a/models/migrations/base/hash.go b/models/migrations/base/hash.go new file mode 100644 index 0000000..00fd1ef --- /dev/null +++ b/models/migrations/base/hash.go @@ -0,0 +1,16 @@ +// Copyright 2022 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package base + +import ( + "crypto/sha256" + "encoding/hex" + + "golang.org/x/crypto/pbkdf2" +) + +func HashToken(token, salt string) string { + tempHash := pbkdf2.Key([]byte(token), []byte(salt), 10000, 50, sha256.New) + return hex.EncodeToString(tempHash) +} diff --git a/models/migrations/base/main_test.go b/models/migrations/base/main_test.go new file mode 100644 index 0000000..c625ef0 --- /dev/null +++ b/models/migrations/base/main_test.go @@ -0,0 +1,14 @@ +// Copyright 2021 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package base + +import ( + "testing" + + migrations_tests "code.gitea.io/gitea/models/migrations/test" +) + +func TestMain(m *testing.M) { + migrations_tests.MainTest(m) +} -- cgit v1.2.3