From ac360ccb1c7bfcf9252253f8e4b8a78399d854f5 Mon Sep 17 00:00:00 2001 From: Frederico Santos Date: Thu, 17 Oct 2024 01:36:55 +0100 Subject: [PATCH] Cold storage (#57) * feat: Implement S3 integration for system save data migration and retrieval * feat: Refactor system save data handling to use JSON and improve error logging * feat: Update S3 migration schedule to hourly and limit old account retrieval to 3000 records * fix: Clean up S3 migration code by removing unnecessary blank lines and improving logging messages --- api/daily/common.go | 63 ++++++++++++++++++++- api/endpoints.go | 1 + db/db.go | 5 ++ db/savedata.go | 109 ++++++++++++++++++++++++++++++++++++- docker-compose.Example.yml | 7 ++- go.mod | 20 ++++++- go.sum | 40 +++++++++++++- 7 files changed, 238 insertions(+), 7 deletions(-) diff --git a/api/daily/common.go b/api/daily/common.go index 9c1df94..63353d7 100644 --- a/api/daily/common.go +++ b/api/daily/common.go @@ -18,15 +18,21 @@ package daily import ( + "bytes" + "context" "crypto/md5" "crypto/rand" "encoding/base64" "encoding/binary" + "encoding/json" "fmt" "log" "os" "time" + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/config" + "github.com/aws/aws-sdk-go-v2/service/s3" "github.com/pagefaultgames/rogueserver/db" "github.com/robfig/cron/v3" ) @@ -34,8 +40,9 @@ import ( const secondsPerDay = 60 * 60 * 24 var ( - scheduler = cron.New(cron.WithLocation(time.UTC)) - secret []byte + scheduler = cron.New(cron.WithLocation(time.UTC)) + s3scheduler = cron.New(cron.WithLocation(time.UTC)) + secret []byte ) func Init() error { @@ -84,6 +91,22 @@ func Init() error { scheduler.Start() + if os.Getenv("AWS_ENDPOINT_URL_S3") == "" { + log.Printf("AWS_ENDPOINT_URL_S3 not set, skipping s3 migration") + return nil + } + + _, err = s3scheduler.AddFunc("@hourly", func() { + time.Sleep(time.Second) + S3SaveMigration() + }) + + if err != nil { + return err + } + + s3scheduler.Start() + return nil } @@ -99,3 +122,39 @@ func deriveSeed(seedTime time.Time) []byte { return hashedSeed[:] } + +func S3SaveMigration() { + cfg, _ := config.LoadDefaultConfig(context.TODO()) + + svc := s3.NewFromConfig(cfg, func(o *s3.Options) { + o.BaseEndpoint = aws.String(os.Getenv("AWS_ENDPOINT_URL_S3")) + }) + // retrieve accounts from db + _, err := svc.CreateBucket(context.Background(), &s3.CreateBucketInput{ + Bucket: aws.String("pokerogue-system"), + }) + + if err != nil { + log.Printf("error while creating bucket: %s", err) + } + + accounts := db.RetrieveOldAccounts() + for _, user := range accounts { + data, _ := db.ReadSystemSaveData(user) + username, _ := db.FetchUsernameFromUUID(user) + json, _ := json.Marshal(data) + _, err := svc.PutObject(context.Background(), &s3.PutObjectInput{ + Bucket: aws.String("pokerogue-system"), + Key: aws.String(username), + Body: bytes.NewReader(json), + }) + + if err != nil { + log.Printf("error while saving data in s3 for user %s: %s", username, err) + continue + } + + fmt.Printf("Saved data in s3 for user %s\n", username) + db.UpdateLocation(user, username) + } +} diff --git a/api/endpoints.go b/api/endpoints.go index 162900d..0e9b9bf 100644 --- a/api/endpoints.go +++ b/api/endpoints.go @@ -420,6 +420,7 @@ func handleSystem(w http.ResponseWriter, r *http.Request) { if errors.Is(err, sql.ErrNoRows) { http.Error(w, err.Error(), http.StatusNotFound) } else { + fmt.Printf("failed to get system save data: %s\n", err) httpError(w, r, err, http.StatusInternalServerError) } diff --git a/db/db.go b/db/db.go index 885c3b7..b8b79c0 100644 --- a/db/db.go +++ b/db/db.go @@ -103,6 +103,11 @@ func setupDb(tx *sql.Tx) error { `ALTER TABLE accounts ADD COLUMN IF NOT EXISTS discordId VARCHAR(32) UNIQUE DEFAULT NULL`, `ALTER TABLE accounts ADD COLUMN IF NOT EXISTS googleId VARCHAR(32) UNIQUE DEFAULT NULL`, + + // ---------------------------------- + // MIGRATION 004 + + `ALTER TABLE accounts ADD COLUMN IF NOT EXISTS isInLocalDb TINYINT(1) NOT NULL DEFAULT 1`, } for _, q := range queries { diff --git a/db/savedata.go b/db/savedata.go index 55d40c4..aed1eee 100644 --- a/db/savedata.go +++ b/db/savedata.go @@ -19,10 +19,17 @@ package db import ( "bytes" + "context" "encoding/gob" + "encoding/json" + "fmt" "github.com/klauspost/compress/zstd" "github.com/pagefaultgames/rogueserver/defs" + + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/config" + "github.com/aws/aws-sdk-go-v2/service/s3" ) func TryAddSeedCompletion(uuid []byte, seed string, mode int) (bool, error) { @@ -55,8 +62,16 @@ func ReadSeedCompleted(uuid []byte, seed string) (bool, error) { func ReadSystemSaveData(uuid []byte) (defs.SystemSaveData, error) { var system defs.SystemSaveData + isLocal, err := isSaveInLocalDb(uuid) + if err != nil { + return system, err + } + + if !isLocal { + RetrieveSystemSaveFromS3(uuid) + } var data []byte - err := handle.QueryRow("SELECT data FROM systemSaveData WHERE uuid = ?", uuid).Scan(&data) + err = handle.QueryRow("SELECT data FROM systemSaveData WHERE uuid = ?", uuid).Scan(&data) if err != nil { return system, err } @@ -193,3 +208,95 @@ func RetrievePlaytime(uuid []byte) (int, error) { return playtime, nil } + +func isSaveInLocalDb(uuid []byte) (bool, error) { + var isLocal bool + err := handle.QueryRow("SELECT isInLocalDb FROM accounts WHERE uuid = ?", uuid).Scan(&isLocal) + if err != nil { + return false, err + } + + return isLocal, nil +} + +func RetrieveSystemSaveFromS3(uuid []byte) error { + cfg, err := config.LoadDefaultConfig(context.TODO()) + if err != nil { + return err + } + + client := s3.NewFromConfig(cfg) + + username, err := FetchUsernameFromUUID(uuid) + if err != nil { + return err + } + + s3Object := &s3.GetObjectInput{ + Bucket: aws.String("pokerogue-system"), + Key: aws.String(username), + } + + resp, err := client.GetObject(context.TODO(), s3Object) + if err != nil { + return err + } + + var session defs.SystemSaveData + json.NewDecoder(resp.Body).Decode(&session) + + err = StoreSystemSaveData(uuid, session) + if err != nil { + fmt.Printf("Failed to store system save data from s3 for user %s\n", username) + return err + } + + fmt.Printf("Retrieved system save data from s3 for user %s\n", username) + + _, err = handle.Exec("UPDATE accounts SET isInLocalDb = 1 WHERE uuid = ?", uuid) + if err != nil { + return err + } + + _, err = client.DeleteObject(context.TODO(), &s3.DeleteObjectInput{ + Bucket: aws.String("pokerogue-system"), + Key: aws.String(username), + }) + + if err != nil { + fmt.Printf("Failed to delete object %s from s3: %s\n", username, err) + } + return nil +} + +func RetrieveOldAccounts() [][]byte { + var users [][]byte + rows, err := handle.Query("SELECT uuid FROM accounts WHERE isInLocalDb = 1 && lastActivity < DATE_SUB(NOW(), INTERVAL 3 MONTH) LIMIT 3000") + if err != nil { + return nil + } + defer rows.Close() + + for rows.Next() { + var uuid []byte + if err := rows.Scan(&uuid); err != nil { + return nil + } + users = append(users, uuid) + } + if err := rows.Err(); err != nil { + return nil + } + + return users +} + +func UpdateLocation(uuid []byte, username string) { + _, err := handle.Exec("UPDATE accounts SET isInLocalDb = 0 WHERE uuid = ?", uuid) + if err != nil { + fmt.Printf("Failed to update location for user %s\n", username) + return + } + + DeleteSystemSaveData(uuid) +} diff --git a/docker-compose.Example.yml b/docker-compose.Example.yml index b8d916a..289e074 100644 --- a/docker-compose.Example.yml +++ b/docker-compose.Example.yml @@ -1,6 +1,6 @@ services: server: - image: ghcr.io/pagefaultgames/rogueserver:master + image: rogueserver:latest restart: unless-stopped environment: debug: true @@ -10,6 +10,11 @@ services: dbname: pokeroguedb gameurl: http://localhost:8000 callbackurl: http://localhost:8001 + AWS_ACCESS_KEY_ID: + AWS_SECRET_ACCESS_KEY: + AWS_REGION: + AWS_ENDPOINT_URL_S3: + depends_on: db: condition: service_healthy diff --git a/go.mod b/go.mod index 14f1e75..c21b2ee 100644 --- a/go.mod +++ b/go.mod @@ -9,12 +9,30 @@ require ( ) require ( + github.com/aws/aws-sdk-go-v2 v1.32.2 + github.com/aws/aws-sdk-go-v2/config v1.27.43 + github.com/aws/aws-sdk-go-v2/service/s3 v1.65.0 + github.com/bwmarrin/discordgo v0.28.1 github.com/golang-jwt/jwt/v5 v5.2.1 github.com/klauspost/compress v1.17.9 ) require ( - github.com/bwmarrin/discordgo v0.28.1 // indirect + github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.6 // indirect + github.com/aws/aws-sdk-go-v2/credentials v1.17.41 // indirect + github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.17 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.21 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.21 // indirect + github.com/aws/aws-sdk-go-v2/internal/ini v1.8.1 // indirect + github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.19 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.0 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.4.0 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.2 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.0 // indirect + github.com/aws/aws-sdk-go-v2/service/sso v1.24.2 // indirect + github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.2 // indirect + github.com/aws/aws-sdk-go-v2/service/sts v1.32.2 // indirect + github.com/aws/smithy-go v1.22.0 // indirect github.com/gorilla/websocket v1.4.2 // indirect golang.org/x/sys v0.19.0 // indirect ) diff --git a/go.sum b/go.sum index 51feab8..6f2e494 100644 --- a/go.sum +++ b/go.sum @@ -1,13 +1,49 @@ +github.com/aws/aws-sdk-go-v2 v1.32.2 h1:AkNLZEyYMLnx/Q/mSKkcMqwNFXMAvFto9bNsHqcTduI= +github.com/aws/aws-sdk-go-v2 v1.32.2/go.mod h1:2SK5n0a2karNTv5tbP1SjsX0uhttou00v/HpXKM1ZUo= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.6 h1:pT3hpW0cOHRJx8Y0DfJUEQuqPild8jRGmSFmBgvydr0= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.6/go.mod h1:j/I2++U0xX+cr44QjHay4Cvxj6FUbnxrgmqN3H1jTZA= +github.com/aws/aws-sdk-go-v2/config v1.27.43 h1:p33fDDihFC390dhhuv8nOmX419wjOSDQRb+USt20RrU= +github.com/aws/aws-sdk-go-v2/config v1.27.43/go.mod h1:pYhbtvg1siOOg8h5an77rXle9tVG8T+BWLWAo7cOukc= +github.com/aws/aws-sdk-go-v2/credentials v1.17.41 h1:7gXo+Axmp+R4Z+AK8YFQO0ZV3L0gizGINCOWxSLY9W8= +github.com/aws/aws-sdk-go-v2/credentials v1.17.41/go.mod h1:u4Eb8d3394YLubphT4jLEwN1rLNq2wFOlT6OuxFwPzU= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.17 h1:TMH3f/SCAWdNtXXVPPu5D6wrr4G5hI1rAxbcocKfC7Q= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.17/go.mod h1:1ZRXLdTpzdJb9fwTMXiLipENRxkGMTn1sfKexGllQCw= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.21 h1:UAsR3xA31QGf79WzpG/ixT9FZvQlh5HY1NRqSHBNOCk= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.21/go.mod h1:JNr43NFf5L9YaG3eKTm7HQzls9J+A9YYcGI5Quh1r2Y= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.21 h1:6jZVETqmYCadGFvrYEQfC5fAQmlo80CeL5psbno6r0s= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.21/go.mod h1:1SR0GbLlnN3QUmYaflZNiH1ql+1qrSiB2vwcJ+4UM60= +github.com/aws/aws-sdk-go-v2/internal/ini v1.8.1 h1:VaRN3TlFdd6KxX1x3ILT5ynH6HvKgqdiXoTxAF4HQcQ= +github.com/aws/aws-sdk-go-v2/internal/ini v1.8.1/go.mod h1:FbtygfRFze9usAadmnGJNc8KsP346kEe+y2/oyhGAGc= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.19 h1:FKdiFzTxlTRO71p0C7VrLbkkdW8qfMKF5+ej6bTmkT0= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.19/go.mod h1:abO3pCj7WLQPTllnSeYImqFfkGrmJV0JovWo/gqT5N0= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.0 h1:TToQNkvGguu209puTojY/ozlqy2d/SFNcoLIqTFi42g= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.0/go.mod h1:0jp+ltwkf+SwG2fm/PKo8t4y8pJSgOCO4D8Lz3k0aHQ= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.4.0 h1:FQNWhRuSq8QwW74GtU0MrveNhZbqvHsA4dkA9w8fTDQ= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.4.0/go.mod h1:j/zZ3zmWfGCK91K73YsfHP53BSTLSjL/y6YN39XbBLM= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.2 h1:s7NA1SOw8q/5c0wr8477yOPp0z+uBaXBnLE0XYb0POA= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.2/go.mod h1:fnjjWyAW/Pj5HYOxl9LJqWtEwS7W2qgcRLWP+uWbss0= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.0 h1:1NKXS8XfhMM0bg5wVYa/eOH8AM2f6JijugbKEyQFTIg= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.0/go.mod h1:ph931DUfVfgrhZR7py9olSvHCiRpvaGxNvlWBcXxFds= +github.com/aws/aws-sdk-go-v2/service/s3 v1.65.0 h1:2dSm7frMrw2tdJ0QvyccQNJyPGaP24dyDgZ6h1QJMGU= +github.com/aws/aws-sdk-go-v2/service/s3 v1.65.0/go.mod h1:4XSVpw66upN8wND3JZA29eXl2NOZvfFVq7DIP6xvfuQ= +github.com/aws/aws-sdk-go-v2/service/sso v1.24.2 h1:bSYXVyUzoTHoKalBmwaZxs97HU9DWWI3ehHSAMa7xOk= +github.com/aws/aws-sdk-go-v2/service/sso v1.24.2/go.mod h1:skMqY7JElusiOUjMJMOv1jJsP7YUg7DrhgqZZWuzu1U= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.2 h1:AhmO1fHINP9vFYUE0LHzCWg/LfUWUF+zFPEcY9QXb7o= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.2/go.mod h1:o8aQygT2+MVP0NaV6kbdE1YnnIM8RRVQzoeUH45GOdI= +github.com/aws/aws-sdk-go-v2/service/sts v1.32.2 h1:CiS7i0+FUe+/YY1GvIBLLrR/XNGZ4CtM1Ll0XavNuVo= +github.com/aws/aws-sdk-go-v2/service/sts v1.32.2/go.mod h1:HtaiBI8CjYoNVde8arShXb94UbQQi9L4EMr6D+xGBwo= +github.com/aws/smithy-go v1.22.0 h1:uunKnWlcoL3zO7q+gG2Pk53joueEOsnNB28QdMsmiMM= +github.com/aws/smithy-go v1.22.0/go.mod h1:irrKGvNn1InZwb2d7fkIRNucdfwR8R+Ts3wxYa/cJHg= github.com/bwmarrin/discordgo v0.28.1 h1:gXsuo2GBO7NbR6uqmrrBDplPUx2T3nzu775q/Rd1aG4= github.com/bwmarrin/discordgo v0.28.1/go.mod h1:NJZpH+1AfhIcyQsPeuBKsUtYrRnjkyu0kIVMCHkZtRY= github.com/go-sql-driver/mysql v1.7.1 h1:lUIinVbN1DY0xBg0eMOzmmtGoHwWBbvnWubQUrtU8EI= github.com/go-sql-driver/mysql v1.7.1/go.mod h1:OXbVy3sEdcQ2Doequ6Z5BW6fXNQTmx+9S1MCJN5yJMI= github.com/golang-jwt/jwt/v5 v5.2.1 h1:OuVbFODueb089Lh128TAcimifWaLhJwVflnrgM17wHk= github.com/golang-jwt/jwt/v5 v5.2.1/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk= -github.com/klauspost/compress v1.17.9 h1:6KIumPrER1LHsvBVuDa0r5xaG0Es51mhhB9BQB2qeMA= -github.com/klauspost/compress v1.17.9/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw= github.com/gorilla/websocket v1.4.2 h1:+/TMaTYc4QFitKJxsQ7Yye35DkWvkdLcvGKqM+x0Ufc= github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= +github.com/klauspost/compress v1.17.9 h1:6KIumPrER1LHsvBVuDa0r5xaG0Es51mhhB9BQB2qeMA= +github.com/klauspost/compress v1.17.9/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw= github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs= github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro= golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4=