Make tons of fixes to server setup

This commit is contained in:
Ben Visness 2021-08-29 18:11:41 -05:00
parent 6056f6c1cc
commit 4204081563
4 changed files with 159 additions and 73 deletions

View File

@ -1,21 +1,30 @@
#!/bin/bash #!/bin/bash
# This script should be called with the name of the branch to deploy. ($1 will set -eo pipefail
# be the branch name.)
set -euo pipefail branch=$1
if [ -z "$branch" ]; then
echo "Type the name of the branch you would like to deploy: "
read branch
fi
sudo -u hmn bash -s <<SCRIPT echo "Deploying branch $branch..."
do_as() {
sudo -u $1 --preserve-env=PATH bash -s
}
do_as hmn <<SCRIPT
set -euo pipefail set -euo pipefail
pushd /home/hmn/hmn cd /home/hmn/hmn
git fetch --all git fetch --all
git reset --hard $1 git reset --hard origin/$branch
go build -o /home/hmn/bin/hmn src/main.go go build -o /home/hmn/bin/hmn src/main.go
popd
SCRIPT SCRIPT
echo "Running migrations..."
systemctl stop hmn systemctl stop hmn
sudo -u hmn bash -s <<'SCRIPT' do_as hmn <<'SCRIPT'
set -euo pipefail set -euo pipefail
/home/hmn/bin/hmn migrate /home/hmn/bin/hmn migrate
SCRIPT SCRIPT

28
server/root.Makefile Normal file
View File

@ -0,0 +1,28 @@
# from https://marmelab.com/blog/2016/02/29/auto-documented-makefile.html
help: ## Print this help.
@# wrap text here -> |
@# |
@echo 'This Makefile contains tasks for the root user only. Most website management'
@echo 'commands must be run as a different user account, e.g. hmn or caddy. To become'
@echo 'another user, run "su <username>" and then "cd ~".'
@echo ''
@grep -E '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | sort \
| sed 's/^.*\/\(.*\)/\1/' \
| awk 'BEGIN {FS = ":[^:]*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}'
deploy:
/home/hmn/hmn/server/deploy.sh $1
edit-config:
vim /home/hmn/hmn/src/config/config.go
@echo 'Now that you have edited the config, you probably want to re-deploy the site:'
@echo ''
@echo ' make deploy'
@echo ''
logs-hmn: ## View logs for the website
journalctl -u hmn.service -f
logs-caddy: ## View logs for Caddy
journalctl -u caddy.service -f

View File

@ -1,6 +1,6 @@
#!/bin/bash #!/bin/bash
set -euxo pipefail set -exo pipefail
BLUE_BOLD=$'\e[1;34m' BLUE_BOLD=$'\e[1;34m'
RESET=$'\e[0m' RESET=$'\e[0m'
@ -11,11 +11,13 @@ savecheckpoint() {
echo $1 > ./hmn_setup_checkpoint echo $1 > ./hmn_setup_checkpoint
} }
do_as() {
sudo -u $1 --preserve-env=PATH bash -s
}
# Add swap space # Add swap space
# https://www.digitalocean.com/community/tutorials/how-to-add-swap-space-on-ubuntu-20-04 # https://www.digitalocean.com/community/tutorials/how-to-add-swap-space-on-ubuntu-20-04
if [ $checkpoint -lt 10 ]; then if [ $checkpoint -lt 10 ]; then
savecheckpoint 10
fallocate -l 1G /swapfile fallocate -l 1G /swapfile
chmod 600 /swapfile chmod 600 /swapfile
mkswap /swapfile mkswap /swapfile
@ -26,12 +28,12 @@ if [ $checkpoint -lt 10 ]; then
sysctl vm.vfs_cache_pressure=50 sysctl vm.vfs_cache_pressure=50
echo 'vm.swappiness=10' >> /etc/sysctl.conf echo 'vm.swappiness=10' >> /etc/sysctl.conf
echo 'vm.vfs_cache_pressure=50' >> /etc/sysctl.conf echo 'vm.vfs_cache_pressure=50' >> /etc/sysctl.conf
savecheckpoint 10
fi fi
# Configure Linux users # Configure Linux users
if [ $checkpoint -lt 20 ]; then if [ $checkpoint -lt 20 ]; then
savecheckpoint 20
groupadd --system caddy groupadd --system caddy
useradd --system \ useradd --system \
--gid caddy \ --gid caddy \
@ -50,33 +52,43 @@ if [ $checkpoint -lt 20 ]; then
--shell /bin/bash \ --shell /bin/bash \
--create-home --home-dir /home/annotations \ --create-home --home-dir /home/annotations \
annotations annotations
savecheckpoint 20
fi fi
# Install important stuff # Install important stuff
if [ $checkpoint -lt 30 ]; then if [ $checkpoint -lt 30 ]; then
savecheckpoint 30
apt update apt update
apt install -y \ apt install -y \
build-essential \ build-essential \
libcurl4-openssl-dev byacc flex libcurl4-openssl-dev byacc flex
savecheckpoint 30
fi fi
# Install Go # Install Go
if [ $checkpoint -lt 40 ]; then if [ $checkpoint -lt 40 ]; then
savecheckpoint 40
wget https://golang.org/dl/go1.17.linux-amd64.tar.gz wget https://golang.org/dl/go1.17.linux-amd64.tar.gz
tar -C /usr/local -xzf go1.17.linux-amd64.tar.gz tar -C /usr/local -xzf go1.17.linux-amd64.tar.gz
echo 'PATH=$PATH:/usr/local/go/bin:/root/go/bin' >> ~/.bash_profile
source ~/.bash_profile export PATH=$PATH:/usr/local/go/bin:/root/go/bin
echo 'export PATH=$PATH:/usr/local/go/bin:/root/go/bin' >> ~/.bashrc
go version
do_as hmn <<'SCRIPT'
set -euxo pipefail
echo 'export PATH=$PATH:/usr/local/go/bin:/home/hmn/go/bin' >> ~/.bashrc
go version
SCRIPT
savecheckpoint 40
fi fi
export PATH=$PATH:/usr/local/go/bin:/root/go/bin
# Install Caddy # Install Caddy
# https://www.digitalocean.com/community/tutorials/how-to-host-a-website-with-caddy-on-ubuntu-18-04 # https://www.digitalocean.com/community/tutorials/how-to-host-a-website-with-caddy-on-ubuntu-18-04
if [ $checkpoint -lt 50 ]; then if [ $checkpoint -lt 50 ]; then
savecheckpoint 50
go install github.com/caddyserver/xcaddy/cmd/xcaddy@v0.1.9 go install github.com/caddyserver/xcaddy/cmd/xcaddy@v0.1.9
xcaddy build \ xcaddy build \
--with github.com/caddy-dns/cloudflare \ --with github.com/caddy-dns/cloudflare \
@ -84,41 +96,47 @@ if [ $checkpoint -lt 50 ]; then
mv caddy /usr/bin mv caddy /usr/bin
chown root:root /usr/bin/caddy chown root:root /usr/bin/caddy
chmod 755 /usr/bin/caddy chmod 755 /usr/bin/caddy
savecheckpoint 50
fi fi
# Install Postgres # Install Postgres
# (instructions at https://www.postgresql.org/download/linux/ubuntu/) # (instructions at https://www.postgresql.org/download/linux/ubuntu/)
if [ $checkpoint -lt 60 ]; then if [ $checkpoint -lt 60 ]; then
savecheckpoint 60
sudo sh -c 'echo "deb http://apt.postgresql.org/pub/repos/apt $(lsb_release -cs)-pgdg main" > /etc/apt/sources.list.d/pgdg.list' sudo sh -c 'echo "deb http://apt.postgresql.org/pub/repos/apt $(lsb_release -cs)-pgdg main" > /etc/apt/sources.list.d/pgdg.list'
wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | sudo apt-key add - wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | sudo apt-key add -
sudo apt-get update sudo apt-get update
sudo apt-get -y install postgresql sudo apt-get -y install postgresql
savecheckpoint 60
fi fi
# Configure Postgres # Configure Postgres
# TODO: This was supposed to create a user without a password - why didn't it?
# ...or was it?
if [ $checkpoint -lt 70 ]; then if [ $checkpoint -lt 70 ]; then
savecheckpoint 70 echo "Enter the password for the HMN postgres user:"
sudo -u postgres createuser --createdb --login --pwprompt hmn sudo -u postgres createuser --createdb --login --pwprompt hmn
savecheckpoint 70
fi fi
# Set up the folder structure, clone the repo # Set up the folder structure
if [ $checkpoint -lt 80 ]; then if [ $checkpoint -lt 80 ]; then
savecheckpoint 80 set +x
sudo -u hmn bash -s <<'SCRIPT'
set -euxo pipefail
do_as hmn <<'SCRIPT'
cd ~ cd ~
mkdir log mkdir log
mkdir bin mkdir bin
echo 'PATH=$PATH:/usr/local/go/bin:/home/hmn/bin' >> ~/.bash_profile echo 'export PATH=$PATH:/home/hmn/bin' >> ~/.bashrc
source ~/.bash_profile SCRIPT
savecheckpoint 80
fi
# Set up SSH
if [ $checkpoint -lt 81]
do_as hmn <<'SCRIPT'
ssh-keygen -t ed25519 -C "beta-server" -N "" -f ~/.ssh/gitlab ssh-keygen -t ed25519 -C "beta-server" -N "" -f ~/.ssh/gitlab
git config --global core.sshCommand "ssh -i ~/.ssh/gitlab" git config --global core.sshCommand "ssh -i ~/.ssh/gitlab"
echo "" echo ""
@ -127,35 +145,52 @@ if [ $checkpoint -lt 80 ]; then
echo "" echo ""
cat ~/.ssh/gitlab.pub cat ~/.ssh/gitlab.pub
echo "" echo ""
echo "Press enter to continue when you're done." echo "Run this script again when you're done - it will continue where it left off."
read exit 0
while true ; do
ssh -T git@gitssh.handmade.network && break || true
echo "Failed to connect to GitLab. Fix the issue and then try again. (Press enter when you're done.)"
read
done
SCRIPT SCRIPT
echo 'PATH=$PATH:/home/hmn/bin' >> ~/.bash_profile savecheckpoint 81
source ~/.bash_profile
# This is a special case, where we want to halt the script and allow the
# user to perform an action before moving on.
exit 0
fi fi
if [ $checkpoint -lt 90 ]; then # Test SSH
savecheckpoint 90 if [ $checkpoint -lt 82 ]; then
do_as hmn <<'SCRIPT'
set -euxo pipefail
sudo -u hmn bash -s <<'SCRIPT' if ! ssh -T -i ~/.ssh/gitlab git@gitssh.handmade.network; then
set +x
echo "Failed to connect to GitLab. Fix the issue and then run this script again."
echo ""
echo "Copy the following key and add it as a Deploy Key in the project in GitLab (https://git.handmade.network/hmn/hmn/-/settings/ci_cd#js-deploy-keys-settings):"
echo ""
cat ~/.ssh/gitlab.pub
echo ""
exit 1
fi
SCRIPT
savecheckpoint 82
fi
# Clone the repo
if [ $checkpoint -lt 90 ]; then
do_as hmn <<'SCRIPT'
set -euxo pipefail set -euxo pipefail
cd ~ cd ~
git clone git@gitssh.handmade.network:hmn/hmn.git git clone git@gitssh.handmade.network:hmn/hmn.git
SCRIPT SCRIPT
savecheckpoint 90
fi fi
# Copy config files to the right places # Copy config files to the right places
if [ $checkpoint -lt 100 ]; then if [ $checkpoint -lt 100 ]; then
savecheckpoint 100
cp /home/hmn/hmn/server/Caddyfile /home/caddy/Caddyfile cp /home/hmn/hmn/server/Caddyfile /home/caddy/Caddyfile
cp /home/hmn/hmn/server/caddy.service /etc/systemd/system/caddy.service cp /home/hmn/hmn/server/caddy.service /etc/systemd/system/caddy.service
@ -170,21 +205,31 @@ if [ $checkpoint -lt 100 ]; then
cp /home/hmn/hmn/src/config/config.go.example /home/hmn/hmn/src/config/config.go cp /home/hmn/hmn/src/config/config.go.example /home/hmn/hmn/src/config/config.go
cp /home/hmn/hmn/server/deploy.conf.example /home/hmn/hmn/server/deploy.conf cp /home/hmn/hmn/server/deploy.conf.example /home/hmn/hmn/server/deploy.conf
cp /home/hmn/hmn/cinera/cinera.conf.sample /home/hmn/hmn/cinera/cinera.conf cp /home/hmn/hmn/cinera/cinera.conf.sample /home/hmn/hmn/cinera/cinera.conf
chown hmn:hmn /home/hmn/hmn/src/config/config.go
chown hmn:hmn /home/hmn/hmn/server/deploy.conf
chown hmn:hmn /home/hmn/hmn/cinera/cinera.conf
cp /home/hmn/hmn/server/root.Makefile /root/Makefile
systemctl daemon-reload systemctl daemon-reload
savecheckpoint 100
fi fi
# Build the site for the first time (despite bad config) # Build the site for the first time (despite bad config)
if [ $checkpoint -lt 110 ]; then if [ $checkpoint -lt 110 ]; then
savecheckpoint 110 do_as hmn <<'SCRIPT'
sudo -u hmn bash -s <<'SCRIPT'
set -euxo pipefail set -euxo pipefail
cd /home/hmn/hmn cd /home/hmn/hmn
echo "Building the site for the first time. This may take a while..." echo "Building the site for the first time. This may take a while..."
go build -o /home/hmn/bin/hmn src/main.go go build -v -o /home/hmn/bin/hmn src/main.go
SCRIPT SCRIPT
echo 'PATH=$PATH:/home/hmn/bin' >> ~/.bashrc
source ~/.bashrc
savecheckpoint 110
fi fi
cat <<HELP cat <<HELP
@ -206,7 +251,7 @@ ${BLUE_BOLD}Caddy${RESET}: /home/caddy/Caddyfile
Also, in the CGI config, add the name of the Git branch you would like to Also, in the CGI config, add the name of the Git branch you would like to
use when deploying. For example, a deployment of the beta site should use use when deploying. For example, a deployment of the beta site should use
the `beta` branch. the 'beta' branch.
${BLUE_BOLD}Monit${RESET}: ~/.monitrc ${BLUE_BOLD}Monit${RESET}: ~/.monitrc
@ -239,18 +284,22 @@ ${BLUE_BOLD}Cinera${RESET}: /home/hmn/hmn/cinera/cinera.conf
${BLUE_BOLD}===== Next steps =====${RESET} ${BLUE_BOLD}===== Next steps =====${RESET}
Make sure you have everything on your path:
source ~/.bashrc
Restore a database backup: Restore a database backup:
su hmn su hmn
cd ~ cd ~
/home/hmn/bin/hmn seedfile <I dunno man figure it out> hmn seedfile <I dunno man figure it out>
Start up Caddy: Start up Caddy:
systemctl start caddy systemctl start caddy
Then run the deploy script: Then deploy the site:
/home/hmn/hmn/server/deploy.sh make deploy
HELP HELP

View File

@ -10,37 +10,37 @@ var Config = HMNConfig{
Addr: ":9001", Addr: ":9001",
PrivateAddr: ":9002", PrivateAddr: ":9002",
BaseUrl: "http://handmade.local:9001", BaseUrl: "http://handmade.local:9001",
LogLevel: zerolog.TraceLevel, LogLevel: zerolog.TraceLevel, // InfoLevel is recommended for production
Postgres: PostgresConfig{ Postgres: PostgresConfig{
User: "hmn", User: "hmn",
Password: "password", Password: "password",
Hostname: "handmade.local", Hostname: "handmade.local",
Port: 5432, Port: 5432,
DbName: "hmn", DbName: "hmn",
LogLevel: pgx.LogLevelTrace, LogLevel: pgx.LogLevelTrace, // LogLevelWarn is recommended for production
MinConn: 2, // Keep these low for dev, high for production MinConn: 2, // Keep these low for dev, high for production
MaxConn: 2, MaxConn: 10,
}, },
Auth: AuthConfig{ Auth: AuthConfig{
CookieDomain: ".handmade.local", CookieDomain: ".handmade.local",
CookieSecure: false, CookieSecure: false,
}, },
Email: EmailConfig{ Email: EmailConfig{
ServerAddress: "smtp.example.com", ServerAddress: "smtp.example.com",
ServerPort: 587, ServerPort: 587,
FromAddress: "noreply@example.com", FromAddress: "noreply@example.com",
FromAddressPassword: "", FromAddressPassword: "",
FromName: "Handmade Network Team", FromName: "Handmade Network Team",
OverrideRecipientEmail: "override@handmade.network", // NOTE(asaf): If this is not empty, all emails will be redirected to this address. OverrideRecipientEmail: "override@handmade.network", // NOTE(asaf): If this is not empty, all emails will be redirected to this address.
}, },
DigitalOcean: DigitalOceanConfig{ DigitalOcean: DigitalOceanConfig{
AssetsSpacesKey: "", AssetsSpacesKey: "",
AssetsSpacesSecret: "", AssetsSpacesSecret: "",
AssetsSpacesRegion: "", AssetsSpacesRegion: "",
AssetsSpacesEndpoint: "", AssetsSpacesEndpoint: "",
AssetsSpacesBucket: "", AssetsSpacesBucket: "",
AssetsPathPrefix: "", AssetsPathPrefix: "",
AssetsPublicUrlRoot: "", AssetsPublicUrlRoot: "",
}, },
EpisodeGuide: EpisodeGuide{ EpisodeGuide: EpisodeGuide{
CineraOutputPath: "./annotations/", CineraOutputPath: "./annotations/",