Save Discord messages and attachments

This commit is contained in:
Ben Visness 2021-08-21 11:15:27 -05:00
parent 4f01e1fdcf
commit 76f9256e97
11 changed files with 617 additions and 53 deletions

5
go.mod
View File

@ -7,6 +7,11 @@ require (
github.com/Masterminds/semver v1.5.0 // indirect github.com/Masterminds/semver v1.5.0 // indirect
github.com/Masterminds/sprig v2.22.0+incompatible github.com/Masterminds/sprig v2.22.0+incompatible
github.com/alecthomas/chroma v0.9.2 github.com/alecthomas/chroma v0.9.2
github.com/aws/aws-sdk-go-v2 v1.8.1
github.com/aws/aws-sdk-go-v2/config v1.6.1
github.com/aws/aws-sdk-go-v2/credentials v1.3.3
github.com/aws/aws-sdk-go-v2/service/s3 v1.13.0
github.com/aws/smithy-go v1.7.0
github.com/frustra/bbcode v0.0.0-20201127003707-6ef347fbe1c8 github.com/frustra/bbcode v0.0.0-20201127003707-6ef347fbe1c8
github.com/go-stack/stack v1.8.0 github.com/go-stack/stack v1.8.0
github.com/google/uuid v1.2.0 github.com/google/uuid v1.2.0

30
go.sum
View File

@ -40,6 +40,30 @@ github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRF
github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o=
github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY=
github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8=
github.com/aws/aws-sdk-go-v2 v1.8.1 h1:GcFgQl7MsBygmeeqXyV1ivrTEmsVz/rdFJaTcltG9ag=
github.com/aws/aws-sdk-go-v2 v1.8.1/go.mod h1:xEFuWz+3TYdlPRuo+CqATbeDWIWyaT5uAPwPaWtgse0=
github.com/aws/aws-sdk-go-v2/config v1.6.1 h1:qrZINaORyr78syO1zfD4l7r4tZjy0Z1l0sy4jiysyOM=
github.com/aws/aws-sdk-go-v2/config v1.6.1/go.mod h1:t/y3UPu0XEDy0cEw6mvygaBQaPzWiYAxfP2SzgtvclA=
github.com/aws/aws-sdk-go-v2/credentials v1.3.3 h1:A13QPatmUl41SqUfnuT3V0E3XiNGL6qNTOINbE8cZL4=
github.com/aws/aws-sdk-go-v2/credentials v1.3.3/go.mod h1:oVieKMT3m9BSfqhOfuQ+E0j/yN84ZAJ7Qv8Sfume/ak=
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.4.1 h1:rc+fRGvlKbeSd9IFhFS1KWBs0XjTkq0CfK5xqyLgIp0=
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.4.1/go.mod h1:+GTydg3uHmVlQdkRoetz6VHKbOMEYof70m19IpMLifc=
github.com/aws/aws-sdk-go-v2/internal/ini v1.2.1 h1:IkqRRUZTKaS16P2vpX+FNc2jq3JWa3c478gykQp4ow4=
github.com/aws/aws-sdk-go-v2/internal/ini v1.2.1/go.mod h1:Pv3WenDjI0v2Jl7UaMFIIbPOBbhn33RmmAmGgkXDoqY=
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.2.2 h1:YcGVEqLQGHDa81776C3daai6ZkkRGf/8RAQ07hV0QcU=
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.2.2/go.mod h1:EASdTcM1lGhUe1/p4gkojHwlGJkeoRjjr1sRCzup3Is=
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.2.3 h1:VxFCgxsqWe7OThOwJ5IpFX3xrObtuIH9Hg/NW7oot1Y=
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.2.3/go.mod h1:7gcsONBmFoCcKrAqrm95trrMd2+C/ReYKP7Vfu8yHHA=
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.5.3 h1:7tPSbUWzuoMJ2woUKgOfIPuZS88hMdFHJBBB2vR0bHI=
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.5.3/go.mod h1:/ugW3qFkJe/h7sNtI6/zJnwRbvavs6GyOid69uI9eek=
github.com/aws/aws-sdk-go-v2/service/s3 v1.13.0 h1:2oMLrNpOSpkDTocIVv3Fut1XrmlbKPlgnnYMGYqFp0Y=
github.com/aws/aws-sdk-go-v2/service/s3 v1.13.0/go.mod h1:Tzxhu3GnCpj45WJqXyxcLF2gUHzTcmY7CzpQ9x9KVls=
github.com/aws/aws-sdk-go-v2/service/sso v1.3.3 h1:K2gCnGvAASpz+jqP9iyr+F/KNjmTYf8aWOtTQzhmZ5w=
github.com/aws/aws-sdk-go-v2/service/sso v1.3.3/go.mod h1:Jgw5O+SK7MZ2Yi9Yvzb4PggAPYaFSliiQuWR0hNjexk=
github.com/aws/aws-sdk-go-v2/service/sts v1.6.2 h1:l504GWCoQi1Pk68vSUFGLmDIEMzRfVGNgLakDK+Uj58=
github.com/aws/aws-sdk-go-v2/service/sts v1.6.2/go.mod h1:RBhoMJB8yFToaCnbe0jNq5Dcdy0jp6LhHqg55rjClkM=
github.com/aws/smithy-go v1.7.0 h1:+cLHMRrDZvQ4wk+KuQ9yH6eEg6KZEJ9RI2IkDqnygCg=
github.com/aws/smithy-go v1.7.0/go.mod h1:SObp3lf9smib00L/v3U2eAKG8FyQ7iLrJnQiAmR5n+E=
github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs=
@ -92,6 +116,9 @@ github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Z
github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.6 h1:BKbKCqvP6I+rmFHt06ZmyQtvB8xAkWdhFyr0ZUNZcxQ=
github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
@ -185,6 +212,8 @@ github.com/jackc/puddle v1.1.0/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dv
github.com/jackc/puddle v1.1.1/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= github.com/jackc/puddle v1.1.1/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk=
github.com/jackc/puddle v1.1.3 h1:JnPg/5Q9xVJGfjsO5CPUOjnJps1JaRUm8I9FXVCFK94= github.com/jackc/puddle v1.1.3 h1:JnPg/5Q9xVJGfjsO5CPUOjnJps1JaRUm8I9FXVCFK94=
github.com/jackc/puddle v1.1.3/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= github.com/jackc/puddle v1.1.3/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk=
github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo=
github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U=
github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo=
github.com/jpillora/backoff v1.0.0 h1:uvFg412JmmHBHw7iwprIxkPMI+sGQ4kzOWsMeHnm2EA= github.com/jpillora/backoff v1.0.0 h1:uvFg412JmmHBHw7iwprIxkPMI+sGQ4kzOWsMeHnm2EA=
github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4=
@ -484,6 +513,7 @@ gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bl
gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=

155
src/assets/assets.go Normal file
View File

@ -0,0 +1,155 @@
package assets
import (
"bytes"
"context"
"crypto/sha1"
"errors"
"fmt"
"regexp"
"git.handmade.network/hmn/hmn/src/config"
"git.handmade.network/hmn/hmn/src/db"
"git.handmade.network/hmn/hmn/src/models"
"git.handmade.network/hmn/hmn/src/oops"
"github.com/aws/aws-sdk-go-v2/aws"
awsconfig "github.com/aws/aws-sdk-go-v2/config"
"github.com/aws/aws-sdk-go-v2/credentials"
"github.com/aws/aws-sdk-go-v2/service/s3"
"github.com/aws/aws-sdk-go-v2/service/s3/types"
"github.com/aws/smithy-go"
"github.com/google/uuid"
)
var client *s3.Client
func init() {
cfg, err := awsconfig.LoadDefaultConfig(context.Background(),
awsconfig.WithCredentialsProvider(
credentials.NewStaticCredentialsProvider(
config.Config.DigitalOcean.AssetsSpacesKey,
config.Config.DigitalOcean.AssetsSpacesSecret,
"",
),
),
awsconfig.WithRegion(config.Config.DigitalOcean.AssetsSpacesRegion),
awsconfig.WithEndpointResolver(aws.EndpointResolverFunc(func(service, region string) (aws.Endpoint, error) {
return aws.Endpoint{
URL: config.Config.DigitalOcean.AssetsSpacesEndpoint,
}, nil
})),
)
if err != nil {
panic(err)
}
client = s3.NewFromConfig(cfg, func(o *s3.Options) {
o.UsePathStyle = true
})
}
type CreateInput struct {
Content []byte
Filename string
MimeType string
// Optional params
UploaderID *int // HMN user id
Width, Height int
}
var REIllegalFilenameChars = regexp.MustCompile(`[^\w \-.]`)
func SanitizeFilename(filename string) string {
if filename == "" {
return "unnamed"
}
return REIllegalFilenameChars.ReplaceAllString(filename, "")
}
func AssetKey(id, filename string) string {
return fmt.Sprintf("%s%s/%s", config.Config.DigitalOcean.AssetsPathPrefix, id, filename)
}
type InvalidAssetError error
func Create(ctx context.Context, dbConn db.ConnOrTx, in CreateInput) (*models.Asset, error) {
filename := SanitizeFilename(in.Filename)
if len(in.Content) == 0 {
return nil, InvalidAssetError(fmt.Errorf("could not upload asset '%s': no bytes of data were provided", filename))
}
if in.MimeType == "" {
return nil, InvalidAssetError(fmt.Errorf("could not upload asset '%s': no mime type provided", filename))
}
// Upload the asset to the DO space
id := uuid.New()
key := AssetKey(id.String(), filename)
checksum := fmt.Sprintf("%x", sha1.Sum(in.Content))
upload := func() error {
_, err := client.PutObject(ctx, &s3.PutObjectInput{
Bucket: &config.Config.DigitalOcean.AssetsSpacesBucket,
Key: &key,
Body: bytes.NewReader(in.Content),
ACL: types.ObjectCannedACLPublicRead,
})
return err
}
err := upload()
if err != nil {
var apiError smithy.APIError
if errors.As(err, &apiError) && apiError.ErrorCode() == "NoSuchBucket" {
_, err := client.CreateBucket(ctx, &s3.CreateBucketInput{
Bucket: &config.Config.DigitalOcean.AssetsSpacesBucket,
})
if err != nil {
return nil, oops.New(err, "failed to create assets bucket")
}
err = upload()
if err != nil {
return nil, oops.New(err, "failed to upload asset")
}
} else {
return nil, oops.New(err, "failed to upload asset")
}
}
// Save a record in our database
// TODO(db): Would be convient to use RETURNING here...
_, err = dbConn.Exec(ctx,
`
INSERT INTO handmade_asset (id, s3_key, filename, size, mime_type, sha1sum, width, height, uploader_id)
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9)
`,
id,
key,
filename,
len(in.Content),
in.MimeType,
checksum,
in.Width,
in.Height,
in.UploaderID,
)
if err != nil {
return nil, oops.New(err, "failed to save asset record")
}
// Fetch and return the new record
iasset, err := db.QueryOne(ctx, dbConn, models.Asset{},
`
SELECT $columns
FROM handmade_asset
WHERE id = $1
`,
id,
)
if err != nil {
return nil, oops.New(err, "failed to fetch newly-created asset")
}
return iasset.(*models.Asset), nil
}

13
src/assets/assets_test.go Normal file
View File

@ -0,0 +1,13 @@
package assets
import (
"testing"
"github.com/stretchr/testify/assert"
)
func TestSanitizeFilename(t *testing.T) {
assert.Equal(t, "cool filename.txt.wow", SanitizeFilename("cool filename.txt.wow"))
assert.Equal(t, " hi doggy ", SanitizeFilename("😎 hi doggy 🐶"))
assert.Equal(t, "newlinesaretotallylegal", SanitizeFilename("newlines\naretotallylegal"))
}

View File

@ -11,6 +11,7 @@ import (
"git.handmade.network/hmn/hmn/src/logging" "git.handmade.network/hmn/hmn/src/logging"
"git.handmade.network/hmn/hmn/src/oops" "git.handmade.network/hmn/hmn/src/oops"
"github.com/google/uuid" "github.com/google/uuid"
"github.com/jackc/pgconn"
"github.com/jackc/pgtype" "github.com/jackc/pgtype"
"github.com/jackc/pgx/v4" "github.com/jackc/pgx/v4"
"github.com/jackc/pgx/v4/log/zerologadapter" "github.com/jackc/pgx/v4/log/zerologadapter"
@ -57,6 +58,7 @@ func typeIsQueryable(t reflect.Type) bool {
// This interface should match both a direct pgx connection or a pgx transaction. // This interface should match both a direct pgx connection or a pgx transaction.
type ConnOrTx interface { type ConnOrTx interface {
Query(ctx context.Context, sql string, args ...interface{}) (pgx.Rows, error) Query(ctx context.Context, sql string, args ...interface{}) (pgx.Rows, error)
Exec(ctx context.Context, sql string, args ...interface{}) (pgconn.CommandTag, error)
} }
var connInfo = pgtype.NewConnInfo() var connInfo = pgtype.NewConnInfo()

View File

@ -563,7 +563,7 @@ func (bot *discordBotInstance) processEventMsg(ctx context.Context, msg *Gateway
} }
func (bot *discordBotInstance) messageCreateOrUpdate(ctx context.Context, msg *Message) error { func (bot *discordBotInstance) messageCreateOrUpdate(ctx context.Context, msg *Message) error {
if msg.Author != nil && msg.Author.ID == config.Config.Discord.BotUserID { if msg.OriginalHasFields("author") && msg.Author.ID == config.Config.Discord.BotUserID {
// Don't process your own messages // Don't process your own messages
return nil return nil
} }

45
src/discord/library.go Normal file
View File

@ -0,0 +1,45 @@
package discord
import (
"context"
"git.handmade.network/hmn/hmn/src/oops"
)
func (bot *discordBotInstance) processLibraryMsg(ctx context.Context, msg *Message) error {
switch msg.Type {
case MessageTypeDefault, MessageTypeReply, MessageTypeApplicationCommand:
default:
return nil
}
if !msg.OriginalHasFields("content") {
return nil
}
if !messageHasLinks(msg.Content) {
err := DeleteMessage(ctx, msg.ChannelID, msg.ID)
if err != nil {
return oops.New(err, "failed to delete message")
}
if !msg.Author.IsBot {
channel, err := CreateDM(ctx, msg.Author.ID)
if err != nil {
return oops.New(err, "failed to create DM channel")
}
err = SendMessages(ctx, bot.dbConn, MessageToSend{
ChannelID: channel.ID,
Req: CreateMessageRequest{
Content: "Posts in #the-library are required to have a link. Discuss library content in other relevant channels.",
},
})
if err != nil {
return oops.New(err, "failed to send showcase warning message")
}
}
}
return nil
}

View File

@ -2,6 +2,8 @@ package discord
import ( import (
"encoding/json" "encoding/json"
"fmt"
"time"
) )
type Opcode int type Opcode int
@ -170,19 +172,50 @@ const (
// https://discord.com/developers/docs/resources/channel#message-object // https://discord.com/developers/docs/resources/channel#message-object
type Message struct { type Message struct {
ID string `json:"id"` ID string `json:"id"`
ChannelID string `json:"channel_id"` ChannelID string `json:"channel_id"`
Content string `json:"content"` GuildID *string `json:"guild_id"`
Author *User `json:"author"` // note that this may not be an actual valid user (see the docs) Content string `json:"content"`
// TODO: Author info Author User `json:"author"` // note that this may not be an actual valid user (see the docs)
// TODO: Timestamp parsing, yay Timestamp string `json:"timestamp"`
Type MessageType `json:"type"` Type MessageType `json:"type"`
Attachments []Attachment `json:"attachments"` Attachments []Attachment `json:"attachments"`
// TODO: Embeds
originalMap map[string]interface{} originalMap map[string]interface{}
} }
func (m *Message) JumpURL() string {
guildStr := "@me"
if m.GuildID != nil {
guildStr = *m.GuildID
}
return fmt.Sprintf("https://discord.com/channels/%s/%s/%s", guildStr, m.ChannelID, m.ID)
}
func (m *Message) Time() time.Time {
t, err := time.Parse(time.RFC3339Nano, m.Timestamp)
if err != nil {
panic(err)
}
return t
}
func (m *Message) OriginalHasFields(fields ...string) bool {
if m.originalMap == nil {
return false
}
for _, field := range fields {
_, ok := m.originalMap[field]
if !ok {
return false
}
}
return true
}
func MessageFromMap(m interface{}) Message { func MessageFromMap(m interface{}) Message {
/* /*
Some gateway events, like MESSAGE_UPDATE, do not contain the Some gateway events, like MESSAGE_UPDATE, do not contain the
@ -194,15 +227,16 @@ func MessageFromMap(m interface{}) Message {
msg := Message{ msg := Message{
ID: mmap["id"].(string), ID: mmap["id"].(string),
ChannelID: mmap["channel_id"].(string), ChannelID: mmap["channel_id"].(string),
GuildID: maybeStringP(mmap, "guild_id"),
Content: maybeString(mmap, "content"), Content: maybeString(mmap, "content"),
Timestamp: maybeString(mmap, "timestamp"),
Type: MessageType(maybeInt(mmap, "type")), Type: MessageType(maybeInt(mmap, "type")),
originalMap: mmap, originalMap: mmap,
} }
if author, ok := mmap["author"]; ok { if author, ok := mmap["author"]; ok {
u := UserFromMap(author) msg.Author = UserFromMap(author)
msg.Author = &u
} }
if iattachments, ok := mmap["attachments"]; ok { if iattachments, ok := mmap["attachments"]; ok {
@ -212,6 +246,8 @@ func MessageFromMap(m interface{}) Message {
} }
} }
// TODO: Embeds
return msg return msg
} }
@ -276,6 +312,15 @@ func maybeString(m map[string]interface{}, k string) string {
return val.(string) return val.(string)
} }
func maybeStringP(m map[string]interface{}, k string) *string {
val, ok := m[k]
if !ok {
return nil
}
strval := val.(string)
return &strval
}
func maybeInt(m map[string]interface{}, k string) int { func maybeInt(m map[string]interface{}, k string) int {
val, ok := m[k] val, ok := m[k]
if !ok { if !ok {

View File

@ -2,15 +2,25 @@ package discord
import ( import (
"context" "context"
"errors"
"io"
"net/http"
"net/url" "net/url"
"regexp" "regexp"
"strings" "strings"
"git.handmade.network/hmn/hmn/src/assets"
"git.handmade.network/hmn/hmn/src/db"
"git.handmade.network/hmn/hmn/src/logging"
"git.handmade.network/hmn/hmn/src/models"
"git.handmade.network/hmn/hmn/src/oops" "git.handmade.network/hmn/hmn/src/oops"
"github.com/jackc/pgx/v4"
) )
var reDiscordMessageLink = regexp.MustCompile(`https?://.+?(\s|$)`) var reDiscordMessageLink = regexp.MustCompile(`https?://.+?(\s|$)`)
var errNotEnoughInfo = errors.New("Discord didn't send enough info in this event for us to do this")
func (bot *discordBotInstance) processShowcaseMsg(ctx context.Context, msg *Message) error { func (bot *discordBotInstance) processShowcaseMsg(ctx context.Context, msg *Message) error {
switch msg.Type { switch msg.Type {
case MessageTypeDefault, MessageTypeReply, MessageTypeApplicationCommand: case MessageTypeDefault, MessageTypeReply, MessageTypeApplicationCommand:
@ -18,26 +28,70 @@ func (bot *discordBotInstance) processShowcaseMsg(ctx context.Context, msg *Mess
return nil return nil
} }
didDelete, err := bot.maybeDeleteShowcaseMsg(ctx, msg)
if err != nil {
return err
}
if didDelete {
return nil
}
tx, err := bot.dbConn.Begin(ctx)
if err != nil {
panic(err)
}
defer tx.Rollback(ctx)
// save the message, maybe save its contents, and maybe make a snippet too
_, err = bot.saveMessageAndContents(ctx, tx, msg)
if errors.Is(err, errNotEnoughInfo) {
logging.ExtractLogger(ctx).Warn().
Interface("msg", msg).
Msg("didn't have enough info to process Discord message")
return nil
} else if err != nil {
return err
}
if doSnippet, err := bot.allowedToCreateMessageSnippet(ctx, msg); doSnippet && err == nil {
_, err := bot.createMessageSnippet(ctx, msg)
if err != nil {
return oops.New(err, "failed to create snippet in gateway")
}
} else if err != nil {
return oops.New(err, "failed to check snippet permissions in gateway")
}
err = tx.Commit(ctx)
if err != nil {
return oops.New(err, "failed to commit Discord message updates")
}
return nil
}
func (bot *discordBotInstance) maybeDeleteShowcaseMsg(ctx context.Context, msg *Message) (didDelete bool, err error) {
hasGoodContent := true hasGoodContent := true
if originalMessageHasField(msg, "content") && !messageHasLinks(msg.Content) { if msg.OriginalHasFields("content") && !messageHasLinks(msg.Content) {
hasGoodContent = false hasGoodContent = false
} }
hasGoodAttachments := true hasGoodAttachments := true
if originalMessageHasField(msg, "attachments") && len(msg.Attachments) == 0 { if msg.OriginalHasFields("attachments") && len(msg.Attachments) == 0 {
hasGoodAttachments = false hasGoodAttachments = false
} }
didDelete = false
if !hasGoodContent && !hasGoodAttachments { if !hasGoodContent && !hasGoodAttachments {
didDelete = true
err := DeleteMessage(ctx, msg.ChannelID, msg.ID) err := DeleteMessage(ctx, msg.ChannelID, msg.ID)
if err != nil { if err != nil {
return oops.New(err, "failed to delete message") return false, oops.New(err, "failed to delete message")
} }
if msg.Author != nil && !msg.Author.IsBot { if !msg.Author.IsBot {
channel, err := CreateDM(ctx, msg.Author.ID) channel, err := CreateDM(ctx, msg.Author.ID)
if err != nil { if err != nil {
return oops.New(err, "failed to create DM channel") return false, oops.New(err, "failed to create DM channel")
} }
err = SendMessages(ctx, bot.dbConn, MessageToSend{ err = SendMessages(ctx, bot.dbConn, MessageToSend{
@ -47,50 +101,230 @@ func (bot *discordBotInstance) processShowcaseMsg(ctx context.Context, msg *Mess
}, },
}) })
if err != nil { if err != nil {
return oops.New(err, "failed to send showcase warning message") return false, oops.New(err, "failed to send showcase warning message")
} }
} }
} }
return nil return didDelete, nil
} }
func (bot *discordBotInstance) processLibraryMsg(ctx context.Context, msg *Message) error { /*
switch msg.Type { Ensures that a Discord message is stored in the database. This function is
case MessageTypeDefault, MessageTypeReply, MessageTypeApplicationCommand: idempotent and can be called regardless of whether the item already exists in
default: the database.
return nil
}
if !originalMessageHasField(msg, "content") { This does not create snippets or do anything besides save the message itself.
return nil */
} func (bot *discordBotInstance) saveMessage(
ctx context.Context,
tx pgx.Tx,
msg *Message,
) (*models.DiscordMessage, error) {
iDiscordMessage, err := db.QueryOne(ctx, tx, models.DiscordMessage{},
`
SELECT $columns
FROM handmade_discordmessage
WHERE id = $1
`,
msg.ID,
)
if errors.Is(err, db.ErrNoMatchingRows) {
if !msg.OriginalHasFields("author", "timestamp") {
return nil, errNotEnoughInfo
}
if !messageHasLinks(msg.Content) { _, err = tx.Exec(ctx,
err := DeleteMessage(ctx, msg.ChannelID, msg.ID) `
INSERT INTO handmade_discordmessage (id, channel_id, guild_id, url, user_id, sent_at, snippet_created)
VALUES ($1, $2, $3, $4, $5, $6, $7)
`,
msg.ID,
msg.ChannelID,
*msg.GuildID,
msg.JumpURL(),
msg.Author.ID,
msg.Time(),
false,
)
if err != nil { if err != nil {
return oops.New(err, "failed to delete message") return nil, oops.New(err, "failed to save new discord message")
} }
if msg.Author != nil && !msg.Author.IsBot { /*
channel, err := CreateDM(ctx, msg.Author.ID) TODO(db): This is a spot where it would be really nice to be able
if err != nil { to use RETURNING, and avoid this second query.
return oops.New(err, "failed to create DM channel") */
} iDiscordMessage, err = db.QueryOne(ctx, tx, models.DiscordMessage{},
`
SELECT $columns
FROM handmade_discordmessage
WHERE id = $1
`,
msg.ID,
)
if err != nil {
panic(err)
}
} else if err != nil {
return nil, oops.New(err, "failed to check for existing Discord message")
}
err = SendMessages(ctx, bot.dbConn, MessageToSend{ return iDiscordMessage.(*models.DiscordMessage), nil
ChannelID: channel.ID, }
Req: CreateMessageRequest{
Content: "Posts in #the-library are required to have a link. Discuss library content in other relevant channels.", /*
}, Processes a single Discord message, saving as much of the message's content
}) and attachments as allowed by our rules and user settings. Does NOT create
if err != nil { snippets.
return oops.New(err, "failed to send showcase warning message")
} Idempotent; can be called any time whether the message exists or not.
*/
func (bot *discordBotInstance) saveMessageAndContents(
ctx context.Context,
tx pgx.Tx,
msg *Message,
) (*models.DiscordMessage, error) {
newMsg, err := bot.saveMessage(ctx, tx, msg)
if err != nil {
return nil, err
}
// Check for linked Discord user
iDiscordUser, err := db.QueryOne(ctx, tx, models.DiscordUser{},
`
SELECT $columns
FROM handmade_discorduser
WHERE userid = $1
`,
msg.Author.ID,
)
if errors.Is(err, db.ErrNoMatchingRows) {
return newMsg, nil
} else if err != nil {
return nil, oops.New(err, "failed to look up linked Discord user")
}
discordUser := iDiscordUser.(*models.DiscordUser)
// We have a linked Discord account, so save the message contents (regardless of
// whether we create a snippet or not).
_, err = tx.Exec(ctx,
`
INSERT INTO handmade_discordmessagecontent (message_id, discord_id, last_content)
VALUES ($1, $2, $3)
ON CONFLICT (message_id) DO UPDATE SET
discord_id = EXCLUDED.discord_id,
last_content = EXCLUDED.last_content
`,
msg.ID,
discordUser.ID,
msg.Content, // TODO: Add a method that can fill in mentions and stuff (https://discord.com/developers/docs/reference#message-formatting)
)
// Save attachments
for _, attachment := range msg.Attachments {
_, err := bot.saveAttachment(ctx, tx, &attachment, discordUser.HMNUserId, msg.ID)
if err != nil {
return nil, oops.New(err, "failed to save attachment")
} }
} }
return nil // TODO: Save embeds
return newMsg, nil
}
func (bot *discordBotInstance) saveAttachment(ctx context.Context, tx pgx.Tx, attachment *Attachment, hmnUserID int, discordMessageID string) (*models.DiscordMessageAttachment, error) {
// TODO: Return an existing attachment if it exists
width := 0
height := 0
if attachment.Width != nil {
width = *attachment.Width
}
if attachment.Height != nil {
height = *attachment.Height
}
// TODO: Timeouts and stuff, context cancellation
res, err := http.Get(attachment.Url)
if err != nil {
return nil, oops.New(err, "failed to fetch attachment data")
}
defer res.Body.Close()
content, err := io.ReadAll(res.Body)
if err != nil {
panic(err)
}
asset, err := assets.Create(ctx, tx, assets.CreateInput{
Content: content,
Filename: attachment.Filename,
MimeType: attachment.ContentType,
UploaderID: &hmnUserID,
Width: width,
Height: height,
})
if err != nil {
return nil, oops.New(err, "failed to save asset for Discord attachment")
}
// TODO(db): RETURNING plz thanks
_, err = tx.Exec(ctx,
`
INSERT INTO handmade_discordmessageattachment (id, asset_id, message_id)
VALUES ($1, $2, $3)
`,
attachment.ID,
asset.ID,
discordMessageID,
)
if err != nil {
return nil, oops.New(err, "failed to save Discord attachment data")
}
iDiscordAttachment, err := db.QueryOne(ctx, tx, models.DiscordMessageAttachment{},
`
SELECT $columns
FROM handmade_discordmessageattachment
WHERE id = $1
`,
attachment.ID,
)
if err != nil {
return nil, oops.New(err, "failed to fetch new Discord attachment data")
}
return iDiscordAttachment.(*models.DiscordMessageAttachment), nil
}
func (bot *discordBotInstance) allowedToCreateMessageSnippet(ctx context.Context, msg *Message) (bool, error) {
canSave, err := db.QueryBool(ctx, bot.dbConn,
`
SELECT u.discord_save_showcase
FROM
handmade_discorduser AS duser
JOIN auth_user AS u ON duser.hmn_user_id = u.id
WHERE
duser.userid = $1
`,
msg.Author.ID,
)
if errors.Is(err, db.ErrNoMatchingRows) {
return false, nil
} else if err != nil {
return false, oops.New(err, "failed to check if we can save Discord message")
}
return canSave, nil
}
func (bot *discordBotInstance) createMessageSnippet(ctx context.Context, msg *Message) (*models.Snippet, error) {
// TODO: Actually do this
return nil, nil
} }
func messageHasLinks(content string) bool { func messageHasLinks(content string) bool {
@ -104,12 +338,3 @@ func messageHasLinks(content string) bool {
return false return false
} }
func originalMessageHasField(msg *Message, field string) bool {
if msg.originalMap == nil {
return false
}
_, ok := msg.originalMap[field]
return ok
}

43
src/discord/todo.txt Normal file
View File

@ -0,0 +1,43 @@
the goal: port the old discord showcase bot
what it does: save #project-showcase posts to your HMN user profile if you have your account linked
stuff we need to worry about:
- old posts from before you linked your account
- posts that come in while the bot is down
- what to do with posts if you unlink your account
- what to do with posts if you re-link your account
- what to do if you edit the original discord message
- what to do if you delete the original discord message
- the user's preferences re: saving content
- we don't want to save content without the user's consent, especially since it may persist after they disable the integration
- manually adding content for various reasons
- maybe a bug prevented something from saving
- ryan used to post everything in #projects for some reason
real-time stuff:
- on new showcase message
- always save the lightweight record
- if we have permission, create a snippet
- on edit
- re-save the lightweight record and content as if it was new
- create snippet, unconditionally???? (bug??)
- update snippet contents if the edit makes sense
- on delete
- delete snippet if the user so desires
- delete the message records
background stuff:
- watch mode
- every five seconds
- fetch all HMN users with Discord accounts
- check if we have message records without content
- if so, run a full scrape (no snippets)
- every hour
- run a full scrape, creating snippets
- scrape behavior
- look at every message ever in the channel
- do exactly what the real-time bot does on new messages (although maybe don't do snippets depending on context)

View File

@ -2,6 +2,7 @@ package main
import ( import (
_ "git.handmade.network/hmn/hmn/src/admintools" _ "git.handmade.network/hmn/hmn/src/admintools"
_ "git.handmade.network/hmn/hmn/src/assets"
_ "git.handmade.network/hmn/hmn/src/buildscss" _ "git.handmade.network/hmn/hmn/src/buildscss"
_ "git.handmade.network/hmn/hmn/src/initimage" _ "git.handmade.network/hmn/hmn/src/initimage"
_ "git.handmade.network/hmn/hmn/src/migration" _ "git.handmade.network/hmn/hmn/src/migration"