Cold storage (#57)

* feat: Implement S3 integration for system save data migration and retrieval

* feat: Refactor system save data handling to use JSON and improve error logging

* feat: Update S3 migration schedule to hourly and limit old account retrieval to 3000 records

* fix: Clean up S3 migration code by removing unnecessary blank lines and improving logging messages

api/daily/common.go

@@ -18,15 +18,21 @@
 package daily
 
 import (
+	"bytes"
+	"context"
 	"crypto/md5"
 	"crypto/rand"
 	"encoding/base64"
 	"encoding/binary"
+	"encoding/json"
 	"fmt"
 	"log"
 	"os"
 	"time"
 
+	"github.com/aws/aws-sdk-go-v2/aws"
+	"github.com/aws/aws-sdk-go-v2/config"
+	"github.com/aws/aws-sdk-go-v2/service/s3"
 	"github.com/pagefaultgames/rogueserver/db"
 	"github.com/robfig/cron/v3"
 )
@@ -35,6 +41,7 @@ const secondsPerDay = 60 * 60 * 24
 
 var (
 	scheduler   = cron.New(cron.WithLocation(time.UTC))
+	s3scheduler = cron.New(cron.WithLocation(time.UTC))
 	secret      []byte
 )
 
@@ -84,6 +91,22 @@ func Init() error {
 
 	scheduler.Start()
 
+	if os.Getenv("AWS_ENDPOINT_URL_S3") == "" {
+		log.Printf("AWS_ENDPOINT_URL_S3 not set, skipping s3 migration")
+		return nil
+	}
+
+	_, err = s3scheduler.AddFunc("@hourly", func() {
+		time.Sleep(time.Second)
+		S3SaveMigration()
+	})
+
+	if err != nil {
+		return err
+	}
+
+	s3scheduler.Start()
+
 	return nil
 }
 
@@ -99,3 +122,39 @@ func deriveSeed(seedTime time.Time) []byte {
 
 	return hashedSeed[:]
 }
+
+func S3SaveMigration() {
+	cfg, _ := config.LoadDefaultConfig(context.TODO())
+
+	svc := s3.NewFromConfig(cfg, func(o *s3.Options) {
+		o.BaseEndpoint = aws.String(os.Getenv("AWS_ENDPOINT_URL_S3"))
+	})
+	// retrieve accounts from db
+	_, err := svc.CreateBucket(context.Background(), &s3.CreateBucketInput{
+		Bucket: aws.String("pokerogue-system"),
+	})
+
+	if err != nil {
+		log.Printf("error while creating bucket: %s", err)
+	}
+
+	accounts := db.RetrieveOldAccounts()
+	for _, user := range accounts {
+		data, _ := db.ReadSystemSaveData(user)
+		username, _ := db.FetchUsernameFromUUID(user)
+		json, _ := json.Marshal(data)
+		_, err := svc.PutObject(context.Background(), &s3.PutObjectInput{
+			Bucket: aws.String("pokerogue-system"),
+			Key:    aws.String(username),
+			Body:   bytes.NewReader(json),
+		})
+
+		if err != nil {
+			log.Printf("error while saving data in s3 for user %s: %s", username, err)
+			continue
+		}
+
+		fmt.Printf("Saved data in s3 for user %s\n", username)
+		db.UpdateLocation(user, username)
+	}
+}

api/endpoints.go

@@ -420,6 +420,7 @@ func handleSystem(w http.ResponseWriter, r *http.Request) {
 			if errors.Is(err, sql.ErrNoRows) {
 				http.Error(w, err.Error(), http.StatusNotFound)
 			} else {
+				fmt.Printf("failed to get system save data: %s\n", err)
 				httpError(w, r, err, http.StatusInternalServerError)
 			}
 

db/db.go

@@ -103,6 +103,11 @@ func setupDb(tx *sql.Tx) error {
 
 		`ALTER TABLE accounts ADD COLUMN IF NOT EXISTS discordId VARCHAR(32) UNIQUE DEFAULT NULL`,
 		`ALTER TABLE accounts ADD COLUMN IF NOT EXISTS googleId VARCHAR(32) UNIQUE DEFAULT NULL`,
+
+		// ----------------------------------
+		// MIGRATION 004
+
+		`ALTER TABLE accounts ADD COLUMN IF NOT EXISTS isInLocalDb TINYINT(1) NOT NULL DEFAULT 1`,
 	}
 
 	for _, q := range queries {

db/savedata.go

@@ -19,10 +19,17 @@ package db
 
 import (
 	"bytes"
+	"context"
 	"encoding/gob"
+	"encoding/json"
+	"fmt"
 
 	"github.com/klauspost/compress/zstd"
 	"github.com/pagefaultgames/rogueserver/defs"
+
+	"github.com/aws/aws-sdk-go-v2/aws"
+	"github.com/aws/aws-sdk-go-v2/config"
+	"github.com/aws/aws-sdk-go-v2/service/s3"
 )
 
 func TryAddSeedCompletion(uuid []byte, seed string, mode int) (bool, error) {
@@ -55,8 +62,16 @@ func ReadSeedCompleted(uuid []byte, seed string) (bool, error) {
 func ReadSystemSaveData(uuid []byte) (defs.SystemSaveData, error) {
 	var system defs.SystemSaveData
 
+	isLocal, err := isSaveInLocalDb(uuid)
+	if err != nil {
+		return system, err
+	}
+
+	if !isLocal {
+		RetrieveSystemSaveFromS3(uuid)
+	}
 	var data []byte
-	err := handle.QueryRow("SELECT data FROM systemSaveData WHERE uuid = ?", uuid).Scan(&data)
+	err = handle.QueryRow("SELECT data FROM systemSaveData WHERE uuid = ?", uuid).Scan(&data)
 	if err != nil {
 		return system, err
 	}
@@ -193,3 +208,95 @@ func RetrievePlaytime(uuid []byte) (int, error) {
 
 	return playtime, nil
 }
+
+func isSaveInLocalDb(uuid []byte) (bool, error) {
+	var isLocal bool
+	err := handle.QueryRow("SELECT isInLocalDb FROM accounts WHERE uuid = ?", uuid).Scan(&isLocal)
+	if err != nil {
+		return false, err
+	}
+
+	return isLocal, nil
+}
+
+func RetrieveSystemSaveFromS3(uuid []byte) error {
+	cfg, err := config.LoadDefaultConfig(context.TODO())
+	if err != nil {
+		return err
+	}
+
+	client := s3.NewFromConfig(cfg)
+
+	username, err := FetchUsernameFromUUID(uuid)
+	if err != nil {
+		return err
+	}
+
+	s3Object := &s3.GetObjectInput{
+		Bucket: aws.String("pokerogue-system"),
+		Key:    aws.String(username),
+	}
+
+	resp, err := client.GetObject(context.TODO(), s3Object)
+	if err != nil {
+		return err
+	}
+
+	var session defs.SystemSaveData
+	json.NewDecoder(resp.Body).Decode(&session)
+
+	err = StoreSystemSaveData(uuid, session)
+	if err != nil {
+		fmt.Printf("Failed to store system save data from s3 for user %s\n", username)
+		return err
+	}
+
+	fmt.Printf("Retrieved system save data from s3 for user %s\n", username)
+
+	_, err = handle.Exec("UPDATE accounts SET isInLocalDb = 1 WHERE uuid = ?", uuid)
+	if err != nil {
+		return err
+	}
+
+	_, err = client.DeleteObject(context.TODO(), &s3.DeleteObjectInput{
+		Bucket: aws.String("pokerogue-system"),
+		Key:    aws.String(username),
+	})
+
+	if err != nil {
+		fmt.Printf("Failed to delete object %s from s3: %s\n", username, err)
+	}
+	return nil
+}
+
+func RetrieveOldAccounts() [][]byte {
+	var users [][]byte
+	rows, err := handle.Query("SELECT uuid FROM accounts WHERE isInLocalDb = 1 && lastActivity < DATE_SUB(NOW(), INTERVAL 3 MONTH) LIMIT 3000")
+	if err != nil {
+		return nil
+	}
+	defer rows.Close()
+
+	for rows.Next() {
+		var uuid []byte
+		if err := rows.Scan(&uuid); err != nil {
+			return nil
+		}
+		users = append(users, uuid)
+	}
+	if err := rows.Err(); err != nil {
+		return nil
+	}
+
+	return users
+}
+
+func UpdateLocation(uuid []byte, username string) {
+	_, err := handle.Exec("UPDATE accounts SET isInLocalDb = 0 WHERE uuid = ?", uuid)
+	if err != nil {
+		fmt.Printf("Failed to update location for user %s\n", username)
+		return
+	}
+
+	DeleteSystemSaveData(uuid)
+}

docker-compose.Example.yml

@@ -1,6 +1,6 @@
 services:
   server: 
-    image: ghcr.io/pagefaultgames/rogueserver:master
+    image: rogueserver:latest
     restart: unless-stopped
     environment:
       debug: true
@@ -10,6 +10,11 @@ services:
       dbname: pokeroguedb
       gameurl: http://localhost:8000
       callbackurl: http://localhost:8001
+      AWS_ACCESS_KEY_ID: <access>
+      AWS_SECRET_ACCESS_KEY: <secret>
+      AWS_REGION: <region>
+      AWS_ENDPOINT_URL_S3: <endpoint>
+
     depends_on:
       db:
         condition: service_healthy

go.mod

@@ -9,12 +9,30 @@ require (
 )
 
 require (
+	github.com/aws/aws-sdk-go-v2 v1.32.2
+	github.com/aws/aws-sdk-go-v2/config v1.27.43
+	github.com/aws/aws-sdk-go-v2/service/s3 v1.65.0
+	github.com/bwmarrin/discordgo v0.28.1
 	github.com/golang-jwt/jwt/v5 v5.2.1
 	github.com/klauspost/compress v1.17.9
 )
 
 require (
-	github.com/bwmarrin/discordgo v0.28.1 // indirect
+	github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.6 // indirect
+	github.com/aws/aws-sdk-go-v2/credentials v1.17.41 // indirect
+	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.17 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.21 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.21 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/ini v1.8.1 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.19 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.0 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.4.0 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.2 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.0 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sso v1.24.2 // indirect
+	github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.2 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sts v1.32.2 // indirect
+	github.com/aws/smithy-go v1.22.0 // indirect
 	github.com/gorilla/websocket v1.4.2 // indirect
 	golang.org/x/sys v0.19.0 // indirect
 )

go.sum

@@ -1,13 +1,49 @@
+github.com/aws/aws-sdk-go-v2 v1.32.2 h1:AkNLZEyYMLnx/Q/mSKkcMqwNFXMAvFto9bNsHqcTduI=
+github.com/aws/aws-sdk-go-v2 v1.32.2/go.mod h1:2SK5n0a2karNTv5tbP1SjsX0uhttou00v/HpXKM1ZUo=
+github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.6 h1:pT3hpW0cOHRJx8Y0DfJUEQuqPild8jRGmSFmBgvydr0=
+github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.6/go.mod h1:j/I2++U0xX+cr44QjHay4Cvxj6FUbnxrgmqN3H1jTZA=
+github.com/aws/aws-sdk-go-v2/config v1.27.43 h1:p33fDDihFC390dhhuv8nOmX419wjOSDQRb+USt20RrU=
+github.com/aws/aws-sdk-go-v2/config v1.27.43/go.mod h1:pYhbtvg1siOOg8h5an77rXle9tVG8T+BWLWAo7cOukc=
+github.com/aws/aws-sdk-go-v2/credentials v1.17.41 h1:7gXo+Axmp+R4Z+AK8YFQO0ZV3L0gizGINCOWxSLY9W8=
+github.com/aws/aws-sdk-go-v2/credentials v1.17.41/go.mod h1:u4Eb8d3394YLubphT4jLEwN1rLNq2wFOlT6OuxFwPzU=
+github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.17 h1:TMH3f/SCAWdNtXXVPPu5D6wrr4G5hI1rAxbcocKfC7Q=
+github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.17/go.mod h1:1ZRXLdTpzdJb9fwTMXiLipENRxkGMTn1sfKexGllQCw=
+github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.21 h1:UAsR3xA31QGf79WzpG/ixT9FZvQlh5HY1NRqSHBNOCk=
+github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.21/go.mod h1:JNr43NFf5L9YaG3eKTm7HQzls9J+A9YYcGI5Quh1r2Y=
+github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.21 h1:6jZVETqmYCadGFvrYEQfC5fAQmlo80CeL5psbno6r0s=
+github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.21/go.mod h1:1SR0GbLlnN3QUmYaflZNiH1ql+1qrSiB2vwcJ+4UM60=
+github.com/aws/aws-sdk-go-v2/internal/ini v1.8.1 h1:VaRN3TlFdd6KxX1x3ILT5ynH6HvKgqdiXoTxAF4HQcQ=
+github.com/aws/aws-sdk-go-v2/internal/ini v1.8.1/go.mod h1:FbtygfRFze9usAadmnGJNc8KsP346kEe+y2/oyhGAGc=
+github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.19 h1:FKdiFzTxlTRO71p0C7VrLbkkdW8qfMKF5+ej6bTmkT0=
+github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.19/go.mod h1:abO3pCj7WLQPTllnSeYImqFfkGrmJV0JovWo/gqT5N0=
+github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.0 h1:TToQNkvGguu209puTojY/ozlqy2d/SFNcoLIqTFi42g=
+github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.0/go.mod h1:0jp+ltwkf+SwG2fm/PKo8t4y8pJSgOCO4D8Lz3k0aHQ=
+github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.4.0 h1:FQNWhRuSq8QwW74GtU0MrveNhZbqvHsA4dkA9w8fTDQ=
+github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.4.0/go.mod h1:j/zZ3zmWfGCK91K73YsfHP53BSTLSjL/y6YN39XbBLM=
+github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.2 h1:s7NA1SOw8q/5c0wr8477yOPp0z+uBaXBnLE0XYb0POA=
+github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.2/go.mod h1:fnjjWyAW/Pj5HYOxl9LJqWtEwS7W2qgcRLWP+uWbss0=
+github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.0 h1:1NKXS8XfhMM0bg5wVYa/eOH8AM2f6JijugbKEyQFTIg=
+github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.0/go.mod h1:ph931DUfVfgrhZR7py9olSvHCiRpvaGxNvlWBcXxFds=
+github.com/aws/aws-sdk-go-v2/service/s3 v1.65.0 h1:2dSm7frMrw2tdJ0QvyccQNJyPGaP24dyDgZ6h1QJMGU=
+github.com/aws/aws-sdk-go-v2/service/s3 v1.65.0/go.mod h1:4XSVpw66upN8wND3JZA29eXl2NOZvfFVq7DIP6xvfuQ=
+github.com/aws/aws-sdk-go-v2/service/sso v1.24.2 h1:bSYXVyUzoTHoKalBmwaZxs97HU9DWWI3ehHSAMa7xOk=
+github.com/aws/aws-sdk-go-v2/service/sso v1.24.2/go.mod h1:skMqY7JElusiOUjMJMOv1jJsP7YUg7DrhgqZZWuzu1U=
+github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.2 h1:AhmO1fHINP9vFYUE0LHzCWg/LfUWUF+zFPEcY9QXb7o=
+github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.2/go.mod h1:o8aQygT2+MVP0NaV6kbdE1YnnIM8RRVQzoeUH45GOdI=
+github.com/aws/aws-sdk-go-v2/service/sts v1.32.2 h1:CiS7i0+FUe+/YY1GvIBLLrR/XNGZ4CtM1Ll0XavNuVo=
+github.com/aws/aws-sdk-go-v2/service/sts v1.32.2/go.mod h1:HtaiBI8CjYoNVde8arShXb94UbQQi9L4EMr6D+xGBwo=
+github.com/aws/smithy-go v1.22.0 h1:uunKnWlcoL3zO7q+gG2Pk53joueEOsnNB28QdMsmiMM=
+github.com/aws/smithy-go v1.22.0/go.mod h1:irrKGvNn1InZwb2d7fkIRNucdfwR8R+Ts3wxYa/cJHg=
 github.com/bwmarrin/discordgo v0.28.1 h1:gXsuo2GBO7NbR6uqmrrBDplPUx2T3nzu775q/Rd1aG4=
 github.com/bwmarrin/discordgo v0.28.1/go.mod h1:NJZpH+1AfhIcyQsPeuBKsUtYrRnjkyu0kIVMCHkZtRY=
 github.com/go-sql-driver/mysql v1.7.1 h1:lUIinVbN1DY0xBg0eMOzmmtGoHwWBbvnWubQUrtU8EI=
 github.com/go-sql-driver/mysql v1.7.1/go.mod h1:OXbVy3sEdcQ2Doequ6Z5BW6fXNQTmx+9S1MCJN5yJMI=
 github.com/golang-jwt/jwt/v5 v5.2.1 h1:OuVbFODueb089Lh128TAcimifWaLhJwVflnrgM17wHk=
 github.com/golang-jwt/jwt/v5 v5.2.1/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk=
-github.com/klauspost/compress v1.17.9 h1:6KIumPrER1LHsvBVuDa0r5xaG0Es51mhhB9BQB2qeMA=
-github.com/klauspost/compress v1.17.9/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw=
 github.com/gorilla/websocket v1.4.2 h1:+/TMaTYc4QFitKJxsQ7Yye35DkWvkdLcvGKqM+x0Ufc=
 github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
+github.com/klauspost/compress v1.17.9 h1:6KIumPrER1LHsvBVuDa0r5xaG0Es51mhhB9BQB2qeMA=
+github.com/klauspost/compress v1.17.9/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw=
 github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs=
 github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro=
 golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4=