From a644ec1caac7c678bb567210b6fdc2fa11497849 Mon Sep 17 00:00:00 2001 From: Ben Visness Date: Fri, 16 Apr 2021 00:08:47 -0500 Subject: [PATCH] Successfully remove the member and memberextended tables! --- resetdb.sh | 4 + .../2021-04-09T000000Z_DeleteOrphanedData.go | 111 +++++++++++++++ .../2021-04-09T000000Z_DeleteOrphanedLinks.go | 55 -------- ...21-04-10T013044Z_CopyMemberExtendedData.go | 107 --------------- ...1-04-10T015339Z_DropMemberExtendedTable.go | 43 ------ ...1-04-11T195747Z_RemoveMemberAndExtended.go | 128 ++++++++++++++++-- ...-04-11T210958Z_RemoveMemberAndExtended2.go | 48 +++++++ 7 files changed, 279 insertions(+), 217 deletions(-) create mode 100644 src/migration/migrations/2021-04-09T000000Z_DeleteOrphanedData.go delete mode 100644 src/migration/migrations/2021-04-09T000000Z_DeleteOrphanedLinks.go delete mode 100644 src/migration/migrations/2021-04-10T013044Z_CopyMemberExtendedData.go delete mode 100644 src/migration/migrations/2021-04-10T015339Z_DropMemberExtendedTable.go diff --git a/resetdb.sh b/resetdb.sh index e628e08..af33896 100644 --- a/resetdb.sh +++ b/resetdb.sh @@ -1,5 +1,9 @@ #!/bin/bash +# This script is for use in local development only. It wipes the existing db, +# creates a new empty one, runs the initial migration to create the schema, +# and then imports actual db content on top of that. + THIS_PATH=$(pwd) BETA_PATH='/mnt/c/Users/bvisn/Developer/handmade/handmade-beta' diff --git a/src/migration/migrations/2021-04-09T000000Z_DeleteOrphanedData.go b/src/migration/migrations/2021-04-09T000000Z_DeleteOrphanedData.go new file mode 100644 index 0000000..3d95c64 --- /dev/null +++ b/src/migration/migrations/2021-04-09T000000Z_DeleteOrphanedData.go @@ -0,0 +1,111 @@ +package migrations + +import ( + "context" + "fmt" + "time" + + "git.handmade.network/hmn/hmn/src/migration/types" + "git.handmade.network/hmn/hmn/src/oops" + "github.com/jackc/pgx/v4" +) + +func init() { + registerMigration(DeleteOrphanedData{}) +} + +type DeleteOrphanedData struct{} + +func (m DeleteOrphanedData) Version() types.MigrationVersion { + return types.MigrationVersion(time.Date(2021, 4, 9, 0, 0, 0, 0, time.UTC)) +} + +func (m DeleteOrphanedData) Name() string { + return "DeleteOrphanedData" +} + +func (m DeleteOrphanedData) Description() string { + return "Delete data that doesn't have other important associated records" +} + +func (m DeleteOrphanedData) Up(tx pgx.Tx) error { + // Delete orphaned users (no member) + res, err := tx.Exec(context.Background(), ` + DELETE FROM auth_user + WHERE + id IN ( + SELECT auth_user.id + FROM + auth_user + LEFT JOIN handmade_member ON auth_user.id = handmade_member.user_id + WHERE + handmade_member.user_id IS NULL + ) + `) + if err != nil { + return oops.New(err, "failed to delete users without members") + } + fmt.Printf("Deleted %v users\n", res.RowsAffected()) + + orphanedMemberExtendedIdsQuery := ` + SELECT mext.id + FROM + handmade_memberextended AS mext + LEFT JOIN handmade_member AS member ON mext.id = member.extended_id + WHERE + member.user_id IS NULL + ` + + // Delete memberextended<->links joins for memberextendeds that are about to die + // (my kingdom for ON DELETE CASCADE, I mean come on) + res, err = tx.Exec(context.Background(), ` + DELETE FROM handmade_memberextended_links + WHERE + memberextended_id IN ( + `+orphanedMemberExtendedIdsQuery+` + ) + `) + if err != nil { + return oops.New(err, "failed to delete memberextendeds without members") + } + fmt.Printf("Deleted %v memberextended<->links joins\n", res.RowsAffected()) + + // Delete orphaned memberextendeds (no member) + res, err = tx.Exec(context.Background(), ` + DELETE FROM handmade_memberextended + WHERE + id IN ( + `+orphanedMemberExtendedIdsQuery+` + ) + `) + if err != nil { + return oops.New(err, "failed to delete memberextendeds without members") + } + fmt.Printf("Deleted %v memberextendeds\n", res.RowsAffected()) + + // Delete orphaned links (no member or project) + res, err = tx.Exec(context.Background(), ` + DELETE FROM handmade_links + WHERE + id IN ( + SELECT links.id + FROM + handmade_links AS links + LEFT JOIN handmade_memberextended_links AS mlinks ON mlinks.links_id = links.id + LEFT JOIN handmade_project_links AS plinks ON plinks.links_id = links.id + WHERE + mlinks.id IS NULL + AND plinks.id IS NULL + ) + `) + if err != nil { + return oops.New(err, "failed to delete orphaned links") + } + fmt.Printf("Deleted %v links\n", res.RowsAffected()) + + return nil +} + +func (m DeleteOrphanedData) Down(tx pgx.Tx) error { + panic("Implement me") +} diff --git a/src/migration/migrations/2021-04-09T000000Z_DeleteOrphanedLinks.go b/src/migration/migrations/2021-04-09T000000Z_DeleteOrphanedLinks.go deleted file mode 100644 index ec58bd4..0000000 --- a/src/migration/migrations/2021-04-09T000000Z_DeleteOrphanedLinks.go +++ /dev/null @@ -1,55 +0,0 @@ -package migrations - -import ( - "context" - "time" - - "git.handmade.network/hmn/hmn/src/migration/types" - "git.handmade.network/hmn/hmn/src/oops" - "github.com/jackc/pgx/v4" -) - -func init() { - registerMigration(DeleteOrphanedLinks{}) -} - -type DeleteOrphanedLinks struct{} - -func (m DeleteOrphanedLinks) Version() types.MigrationVersion { - return types.MigrationVersion(time.Date(2021, 4, 9, 0, 0, 0, 0, time.UTC)) -} - -func (m DeleteOrphanedLinks) Name() string { - return "DeleteOrphanedLinks" -} - -func (m DeleteOrphanedLinks) Description() string { - return "Delete links with no member or project" -} - -func (m DeleteOrphanedLinks) Up(tx pgx.Tx) error { - // Delete orphaned links (no member or project) - _, err := tx.Exec(context.Background(), ` - DELETE FROM handmade_links - WHERE - id IN ( - SELECT links.id - FROM - handmade_links AS links - LEFT JOIN handmade_memberextended_links AS mlinks ON mlinks.links_id = links.id - LEFT JOIN handmade_project_links AS plinks ON plinks.links_id = links.id - WHERE - mlinks.id IS NULL - AND plinks.id IS NULL - ) - `) - if err != nil { - return oops.New(err, "failed to delete orphaned links") - } - - return nil -} - -func (m DeleteOrphanedLinks) Down(tx pgx.Tx) error { - panic("Implement me") -} diff --git a/src/migration/migrations/2021-04-10T013044Z_CopyMemberExtendedData.go b/src/migration/migrations/2021-04-10T013044Z_CopyMemberExtendedData.go deleted file mode 100644 index 45b5f23..0000000 --- a/src/migration/migrations/2021-04-10T013044Z_CopyMemberExtendedData.go +++ /dev/null @@ -1,107 +0,0 @@ -package migrations - -import ( - "context" - "time" - - "git.handmade.network/hmn/hmn/src/migration/types" - "git.handmade.network/hmn/hmn/src/oops" - "github.com/jackc/pgx/v4" -) - -func init() { - // registerMigration(CopyMemberExtendedData{}) -} - -type CopyMemberExtendedData struct{} - -func (m CopyMemberExtendedData) Version() types.MigrationVersion { - return types.MigrationVersion(time.Date(2021, 4, 10, 1, 30, 44, 0, time.UTC)) -} - -func (m CopyMemberExtendedData) Name() string { - return "CopyMemberExtendedData" -} - -func (m CopyMemberExtendedData) Description() string { - return "Copy MemberExtended data into Member" -} - -func (m CopyMemberExtendedData) Up(tx pgx.Tx) error { - // Add columns to member table - _, err := tx.Exec(context.Background(), ` - ALTER TABLE handmade_member - ADD COLUMN bio TEXT NOT NULL DEFAULT '', - ADD COLUMN showemail BOOLEAN NOT NULL DEFAULT FALSE - `) - if err != nil { - return oops.New(err, "failed to add columns to member table") - } - - // Copy data to members from memberextended - _, err = tx.Exec(context.Background(), ` - UPDATE handmade_member - SET (bio, showemail) = ( - SELECT COALESCE(bio, ''), showemail - FROM handmade_memberextended - WHERE handmade_memberextended.id = handmade_member.extended_id - ); - `) - if err != nil { - return oops.New(err, "failed to copy data from the memberextended table") - } - - // Directly associate links with members - _, err = tx.Exec(context.Background(), ` - ALTER TABLE handmade_links - ADD COLUMN member_id INTEGER REFERENCES handmade_member, - ADD COLUMN project_id INTEGER REFERENCES handmade_project, - ADD CONSTRAINT exactly_one_foreign_key CHECK ( - ( - CASE WHEN member_id IS NULL THEN 0 ELSE 1 END - + CASE WHEN project_id IS NULL THEN 0 ELSE 1 END - ) = 1 - ); - - UPDATE handmade_links - SET (member_id) = ( - SELECT mem.user_id - FROM - handmade_memberextended_links AS mlinks - JOIN handmade_memberextended AS memext ON memext.id = mlinks.memberextended_id - JOIN handmade_member AS mem ON mem.extended_id = memext.id - WHERE - mlinks.links_id = handmade_links.id - ); - - UPDATE handmade_links - SET (project_id) = ( - SELECT proj.id - FROM - handmade_project_links AS plinks - JOIN handmade_project AS proj ON proj.id = plinks.project_id - WHERE - plinks.links_id = handmade_links.id - ); - `) - if err != nil { - return oops.New(err, "failed to associate links with members") - } - - return nil -} - -func (m CopyMemberExtendedData) Down(tx pgx.Tx) error { - // _, err := tx.Exec(context.Background(), ` - // ALTER TABLE handmade_member - // DROP COLUMN bio, - // DROP COLUMN showemail - // `) - // if err != nil { - // return oops.New(err, "failed to drop columns from member table") - // } - // - // return nil - - panic("you do not want to do this") -} diff --git a/src/migration/migrations/2021-04-10T015339Z_DropMemberExtendedTable.go b/src/migration/migrations/2021-04-10T015339Z_DropMemberExtendedTable.go deleted file mode 100644 index 83ef016..0000000 --- a/src/migration/migrations/2021-04-10T015339Z_DropMemberExtendedTable.go +++ /dev/null @@ -1,43 +0,0 @@ -package migrations - -import ( - "context" - "time" - - "git.handmade.network/hmn/hmn/src/migration/types" - "github.com/jackc/pgx/v4" -) - -func init() { - // TODO: Delete this migration - // registerMigration(DropMemberExtendedTable{}) -} - -type DropMemberExtendedTable struct{} - -func (m DropMemberExtendedTable) Version() types.MigrationVersion { - return types.MigrationVersion(time.Date(2021, 4, 10, 1, 53, 39, 0, time.UTC)) -} - -func (m DropMemberExtendedTable) Name() string { - return "DropMemberExtendedTable" -} - -func (m DropMemberExtendedTable) Description() string { - return "Remove the MemberExtended record outright" -} - -func (m DropMemberExtendedTable) Up(tx pgx.Tx) error { - _, err := tx.Exec(context.Background(), ` - ALTER TABLE handmade_member - DROP COLUMN extended_id; - - DROP TABLE handmade_memberextended_links; - DROP TABLE handmade_memberextended; - `) - return err -} - -func (m DropMemberExtendedTable) Down(tx pgx.Tx) error { - panic("you do not want to do this") -} diff --git a/src/migration/migrations/2021-04-11T195747Z_RemoveMemberAndExtended.go b/src/migration/migrations/2021-04-11T195747Z_RemoveMemberAndExtended.go index 928172f..066ae0d 100644 --- a/src/migration/migrations/2021-04-11T195747Z_RemoveMemberAndExtended.go +++ b/src/migration/migrations/2021-04-11T195747Z_RemoveMemberAndExtended.go @@ -50,18 +50,7 @@ func (m RemoveMemberAndExtended) Up(tx pgx.Tx) error { } } - /* - Models referencing handmade_member: - - CommunicationChoice - - CommunicationSubCategory - - CommunicationSubThread - - Discord - - CategoryLastReadInfo - - ThreadLastReadInfo - - PostTextVersion - - Post - - handmade_member_projects - */ + // Migrate a lot of simple foreign keys createUserColumn(context.Background(), tx, "handmade_communicationchoice", "member_id", true) createUserColumn(context.Background(), tx, "handmade_communicationsubcategory", "member_id", true) createUserColumn(context.Background(), tx, "handmade_communicationsubthread", "member_id", true) @@ -70,6 +59,121 @@ func (m RemoveMemberAndExtended) Up(tx pgx.Tx) error { createUserColumn(context.Background(), tx, "handmade_threadlastreadinfo", "member_id", true) createUserColumn(context.Background(), tx, "handmade_posttextversion", "editor_id", false) createUserColumn(context.Background(), tx, "handmade_post", "author_id", false) + createUserColumn(context.Background(), tx, "handmade_member_projects", "member_id", true) + + // Directly associate links with members + _, err := tx.Exec(context.Background(), ` + ALTER TABLE handmade_links + ADD COLUMN user_id INTEGER DEFAULT 99999, + ADD COLUMN project_id INTEGER DEFAULT 99999; + + UPDATE handmade_links + SET (user_id) = ( + SELECT mem.user_id + FROM + handmade_memberextended_links AS mlinks + JOIN handmade_memberextended AS memext ON memext.id = mlinks.memberextended_id + JOIN handmade_member AS mem ON mem.extended_id = memext.id + WHERE + mlinks.links_id = handmade_links.id + ); + + UPDATE handmade_links + SET (project_id) = ( + SELECT proj.id + FROM + handmade_project_links AS plinks + JOIN handmade_project AS proj ON proj.id = plinks.project_id + WHERE + plinks.links_id = handmade_links.id + ); + `) + if err != nil { + return oops.New(err, "failed to associate links with members and projects") + } + + _, err = tx.Exec(context.Background(), ` + ALTER TABLE auth_user + -- From handmade_member -- + ADD blurb VARCHAR(140) NOT NULL DEFAULT '', + ADD name VARCHAR(255) NOT NULL DEFAULT '', + ADD signature TEXT NOT NULL DEFAULT '', + ADD avatar VARCHAR(100), + ADD location VARCHAR(100) NOT NULL DEFAULT '', + -- ordering is dropped + -- posts is dropped + -- profileviews is dropped + -- thanked is dropped + ADD timezone VARCHAR(255), + ADD color_1 VARCHAR(6), + ADD color_2 VARCHAR(6), + ADD darktheme BOOLEAN NOT NULL DEFAULT FALSE, + -- extended_id is dropped + -- project_count_all is dropped + -- project_count_public is dropped + -- matrix_username is dropped + -- set_matrix_display_name is dropped + ADD edit_library BOOLEAN NOT NULL DEFAULT FALSE, + ADD discord_delete_snippet_on_message_delete BOOLEAN NOT NULL DEFAULT TRUE, + ADD discord_save_showcase BOOLEAN NOT NULL DEFAULT TRUE, + + -- From handmade_memberextended -- + ADD bio TEXT NOT NULL DEFAULT '', + ADD showemail BOOLEAN NOT NULL DEFAULT FALSE; + -- sendemail is dropped + -- joomlaid is dropped + -- lastResetTime is dropped + -- resetCount is dropped + -- requireReset is dropped + -- birthdate is dropped + + UPDATE auth_user + SET ( + blurb, + name, + signature, + avatar, + location, + timezone, + color_1, + color_2, + darktheme, + edit_library, + discord_delete_snippet_on_message_delete, + discord_save_showcase, + bio, + showemail + ) = ( + SELECT + COALESCE(blurb, ''), + COALESCE(name, ''), + COALESCE(signature, ''), + avatar, + COALESCE(location, ''), + timezone, + color_1, + color_2, + darktheme, + edit_library, + discord_delete_snippet_on_message_delete, + discord_save_showcase, + COALESCE(bio, ''), + showemail + FROM + handmade_member AS mem + JOIN handmade_memberextended AS mext ON mem.extended_id = mext.id + WHERE + mem.user_id = auth_user.id + ); + + ALTER TABLE auth_user + ALTER timezone SET NOT NULL, + ALTER color_1 SET NOT NULL, + ALTER color_2 SET NOT NULL; + `) + if err != nil { + return oops.New(err, "failed to copy fields to auth_user") + } return nil } diff --git a/src/migration/migrations/2021-04-11T210958Z_RemoveMemberAndExtended2.go b/src/migration/migrations/2021-04-11T210958Z_RemoveMemberAndExtended2.go index 9ee7755..d1ae5dd 100644 --- a/src/migration/migrations/2021-04-11T210958Z_RemoveMemberAndExtended2.go +++ b/src/migration/migrations/2021-04-11T210958Z_RemoveMemberAndExtended2.go @@ -56,6 +56,54 @@ func (m RemoveMemberAndExtended2) Up(tx pgx.Tx) error { dropOldColumn(context.Background(), tx, "handmade_threadlastreadinfo", "member_id", "user_id", "CASCADE") dropOldColumn(context.Background(), tx, "handmade_posttextversion", "editor_id", "editor_id", "SET NULL") dropOldColumn(context.Background(), tx, "handmade_post", "author_id", "author_id", "SET NULL") + dropOldColumn(context.Background(), tx, "handmade_member_projects", "member_id", "user_id", "SET NULL") + + _, err := tx.Exec(context.Background(), ` + ALTER TABLE handmade_member_projects + RENAME TO handmade_user_projects; + `) + if err != nil { + return oops.New(err, "failed to rename member projects table") + } + + _, err = tx.Exec(context.Background(), ` + ALTER TABLE handmade_links + ADD FOREIGN KEY (user_id) REFERENCES auth_user ON DELETE CASCADE, + ALTER user_id DROP DEFAULT, + ADD FOREIGN KEY (project_id) REFERENCES handmade_project ON DELETE CASCADE, + ALTER project_id DROP DEFAULT, + ADD CONSTRAINT exactly_one_foreign_key CHECK ( + ( + CASE WHEN user_id IS NOT NULL THEN 1 ELSE 0 END + + CASE WHEN project_id IS NOT NULL THEN 1 ELSE 0 END + ) = 1 + ); + + DROP TABLE handmade_memberextended_links; + DROP TABLE handmade_project_links; + `) + if err != nil { + return oops.New(err, "failed to add constraints to new handmade_links columns") + } + + // And now, the moment you've all been waiting for: + _, err = tx.Exec(context.Background(), ` + DROP TABLE handmade_member; + DROP TABLE handmade_memberextended; + `) + if err != nil { + return oops.New(err, "failed to delete those damn tables") + } + + // And finally, a little cleanup. + _, err = tx.Exec(context.Background(), ` + ALTER INDEX handmade_member_projects_b098ad43 RENAME TO user_projects_btree; + ALTER SEQUENCE handmade_member_projects_id_seq RENAME TO user_projects_id_seq; + ALTER INDEX handmade_member_projects_pkey RENAME TO user_projects_pkey; + `) + if err != nil { + return oops.New(err, "failed to rename some indexes and stuff") + } return nil }