From b53770932df2a4e084127b844ebb314976e08e90 Mon Sep 17 00:00:00 2001 From: Ben Visness Date: Sun, 5 Sep 2021 19:43:49 -0500 Subject: [PATCH] No longer require a migration in seedfile We don't need to do this any more, since our database backups now include the migration table. The seedfile command also no longer requires the user to type a password. This also prevents the Discord history bot from running when it has no credentials. --- .gitignore | 1 + local/backups/.gitkeep | 0 local/download_database.sh | 15 ++++++++ local/resetdb.sh | 22 +++++++++++ server/download_database.sh | 2 +- server/serversetup.sh | 3 +- src/discord/history.go | 8 +++- src/migration/migration.go | 38 ++++--------------- .../2021-08-28T121228Z_ChangeHMNColors.go | 2 +- 9 files changed, 55 insertions(+), 36 deletions(-) create mode 100644 local/backups/.gitkeep create mode 100644 local/download_database.sh create mode 100644 local/resetdb.sh diff --git a/.gitignore b/.gitignore index c6214fb..383ab59 100644 --- a/.gitignore +++ b/.gitignore @@ -11,3 +11,4 @@ annotations/ hmn.conf adminmailer/config.go adminmailer/adminmailer +local/backups \ No newline at end of file diff --git a/local/backups/.gitkeep b/local/backups/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/local/download_database.sh b/local/download_database.sh new file mode 100644 index 0000000..ff1731d --- /dev/null +++ b/local/download_database.sh @@ -0,0 +1,15 @@ +#!/bin/bash + +set -euo pipefail + +s3cmd ls s3://hmn-backup/db/ + +echo "" +echo "Above is a list of all the available database backups." +echo "Enter the name of the one you would like to download (e.g. \"hmn_pg_dump_live_2021-09-01\"):" +read filename + +s3cmd get --force s3://hmn-backup/db/$filename ./local/backups/$filename + +echo "" +echo "Downloaded $filename to local/backups." diff --git a/local/resetdb.sh b/local/resetdb.sh new file mode 100644 index 0000000..f8cf387 --- /dev/null +++ b/local/resetdb.sh @@ -0,0 +1,22 @@ +#!/bin/bash + +set -eou pipefail + +# This script is for use in local development only. It wipes the existing db, +# creates a new empty one, runs the initial migration to create the schema, +# and then imports actual db content on top of that. + +# TODO(opensource): We should adapt Asaf's seedfile command and then delete this. + +THIS_PATH=$(pwd) +BETA_PATH='/mnt/c/Users/bvisn/Developer/handmade/handmade-beta' +# BETA_PATH='/Users/benvisness/Developer/handmade/handmade-beta' + +pushd $BETA_PATH + docker-compose down -v + docker-compose up -d postgres s3 + sleep 3 + + docker-compose exec postgres bash -c "psql -U postgres -c \"CREATE ROLE hmn CREATEDB LOGIN PASSWORD 'password';\"" +popd +go run src/main.go seedfile local/backups/hmn_pg_dump_live_2021-09-06 diff --git a/server/download_database.sh b/server/download_database.sh index d27f126..235c112 100755 --- a/server/download_database.sh +++ b/server/download_database.sh @@ -13,7 +13,7 @@ echo "Above is a list of all the available database backups." echo "Enter the name of the one you would like to download (e.g. \"hmn_pg_dump_live_2021-09-01\"):" read filename -s3cmd --config /home/hmn/.s3cfg get s3://hmn-backup/db/$filename $filename +s3cmd --config /home/hmn/.s3cfg get --force s3://hmn-backup/db/$filename $filename echo "" echo "Downloaded $filename to $(pwd)." diff --git a/server/serversetup.sh b/server/serversetup.sh index da69051..a200fab 100755 --- a/server/serversetup.sh +++ b/server/serversetup.sh @@ -377,8 +377,7 @@ ${BLUE_BOLD}Download and restore a database backup${RESET} su hmn cd ~ - hmn migrate --list - hmn seedfile + hmn seedfile hmn migrate ${BLUE_BOLD}Restore static files${RESET} diff --git a/src/discord/history.go b/src/discord/history.go index 6b71313..828332d 100644 --- a/src/discord/history.go +++ b/src/discord/history.go @@ -17,8 +17,14 @@ func RunHistoryWatcher(ctx context.Context, dbConn *pgxpool.Pool) <-chan struct{ log := logging.ExtractLogger(ctx).With().Str("discord goroutine", "history watcher").Logger() ctx = logging.AttachLoggerToContext(&log, ctx) - done := make(chan struct{}) + if config.Config.Discord.BotToken == "" { + log.Warn().Msg("No Discord bot token was provided, so the Discord history bot cannot run.") + done := make(chan struct{}, 1) + done <- struct{}{} + return done + } + done := make(chan struct{}) go func() { defer func() { log.Debug().Msg("shut down Discord history watcher") diff --git a/src/migration/migration.go b/src/migration/migration.go index 4a34b5b..d6e3fdb 100644 --- a/src/migration/migration.go +++ b/src/migration/migration.go @@ -65,24 +65,16 @@ func init() { } seedFromFileCommand := &cobra.Command{ - Use: "seedfile ", - Short: "Resets the db, runs migrations up to and including , and runs the seed file.", + Use: "seedfile ", + Short: "Resets the db and runs the seed file.", Run: func(cmd *cobra.Command, args []string) { - if len(args) < 2 { - fmt.Printf("You must provide a seed file and migration id.\n\n") + if len(args) < 1 { + fmt.Printf("You must provide a seed file.\n\n") cmd.Usage() os.Exit(1) } - seedFile := args[0] - - afterMigration, err := time.Parse(time.RFC3339, args[1]) - if err != nil { - fmt.Printf("ERROR: bad version string: %v", err) - os.Exit(1) - } - - SeedFromFile(seedFile, types.MigrationVersion(afterMigration)) + SeedFromFile(args[0]) }, } @@ -301,19 +293,13 @@ func MakeMigration(name, description string) { // Applies a cloned db to the local db. // Applies the seed after the migration specified in `afterMigration`. // NOTE(asaf): The db role specified in the config must have the CREATEDB attribute! `ALTER ROLE hmn WITH CREATEDB;` -func SeedFromFile(seedFile string, afterMigration types.MigrationVersion) { +func SeedFromFile(seedFile string) { file, err := os.Open(seedFile) if err != nil { panic(fmt.Errorf("couldn't open seed file %s: %w", seedFile, err)) } file.Close() - migration := migrations.All[afterMigration] - - if migration == nil { - panic(fmt.Errorf("could not find migration: %s", afterMigration)) - } - fmt.Println("Resetting database...") { ctx := context.Background() @@ -348,20 +334,10 @@ func SeedFromFile(seedFile string, afterMigration types.MigrationVersion) { } } - fmt.Println("Running migrations...") - Migrate(afterMigration) - fmt.Println("Executing seed...") cmd := exec.Command("pg_restore", "--single-transaction", - "--dbname", - config.Config.Postgres.DbName, - "--host", - config.Config.Postgres.Hostname, - "--username", - config.Config.Postgres.User, - "--password", - "--data-only", + "--dbname", config.Config.Postgres.DSN(), seedFile, ) fmt.Println("Running command:", cmd) diff --git a/src/migration/migrations/2021-08-28T121228Z_ChangeHMNColors.go b/src/migration/migrations/2021-08-28T121228Z_ChangeHMNColors.go index f3d54f6..c047836 100644 --- a/src/migration/migrations/2021-08-28T121228Z_ChangeHMNColors.go +++ b/src/migration/migrations/2021-08-28T121228Z_ChangeHMNColors.go @@ -40,7 +40,7 @@ func (m ChangeHMNColors) Up(ctx context.Context, tx pgx.Tx) error { return oops.New(err, "failed to update HMN colors") } - if tag.RowsAffected() != 1 { + if tag.RowsAffected() > 1 { return oops.New(nil, "was supposed to update only HMN, but updated %d projects instead", tag.RowsAffected()) }