hotfix: move save data to DB (#2)

* start reading save data from DB

* finish migration code

* remove leftover dir creation

* fix import cycle

* remove more dir creation

* update gitignore

* fix pk

* better table creation

* use REPLACE INTO

* fix typo

.gitignore

@@ -3,3 +3,9 @@
 rogueserver*
 userdata/*
 secret.key
+
+# Jetbrains IDEs
+/.idea/
+*.iml
+*.ipr
+*.iws

api/account/register.go

@@ -20,8 +20,6 @@ package account
 import (
 	"crypto/rand"
 	"fmt"
-	"os"
-
 	"github.com/pagefaultgames/rogueserver/db"
 )
 
@@ -52,10 +50,5 @@ func Register(username, password string) error {
 		return fmt.Errorf("failed to add account record: %s", err)
 	}
 
-	err = os.MkdirAll(fmt.Sprintf("userdata/%x", uuid), 0755)
-	if err != nil && !os.IsExist(err) {
-		return fmt.Errorf(fmt.Sprintf("failed to create userdata folder: %s", err))
-	}
-
 	return nil
 }

api/savedata/clear.go

@@ -18,14 +18,10 @@
 package savedata
 
 import (
-	"encoding/hex"
 	"fmt"
-	"log"
-	"os"
-	"strconv"
-
 	"github.com/pagefaultgames/rogueserver/db"
 	"github.com/pagefaultgames/rogueserver/defs"
+	"log"
 )
 
 type ClearResponse struct {
@@ -66,14 +62,9 @@ func Clear(uuid []byte, slot int, seed string, save defs.SessionSaveData) (Clear
 		}
 	}
 
-	fileName := "session"
+	err = db.DeleteSessionSaveData(uuid, slot)
-	if slot != 0 {
+	if err != nil {
-		fileName += strconv.Itoa(slot)
+		log.Printf("failed to delete session save data: %s", err)
-	}
-
-	err = os.Remove(fmt.Sprintf("userdata/%s/%s.pzs", hex.EncodeToString(uuid), fileName))
-	if err != nil && !os.IsNotExist(err) {
-		return response, fmt.Errorf("failed to delete save file: %s", err)
 	}
 
 	return response, nil

api/savedata/common.go

@@ -18,71 +18,9 @@
 package savedata
 
 import (
-	"encoding/gob"
-	"encoding/hex"
-	"fmt"
-	"os"
-	"strconv"
-
-	"github.com/klauspost/compress/zstd"
 	"github.com/pagefaultgames/rogueserver/defs"
 )
 
-func readSystemSaveData(uuid []byte) (defs.SystemSaveData, error) {
-	var system defs.SystemSaveData
-
-	file, err := os.Open("userdata/" + hex.EncodeToString(uuid) + "/system.pzs")
-	if err != nil {
-		return system, fmt.Errorf("failed to open save file for reading: %s", err)
-	}
-
-	defer file.Close()
-
-	zstdDecoder, err := zstd.NewReader(file)
-	if err != nil {
-		return system, fmt.Errorf("failed to create zstd decoder: %s", err)
-	}
-
-	defer zstdDecoder.Close()
-
-	err = gob.NewDecoder(zstdDecoder).Decode(&system)
-	if err != nil {
-		return system, fmt.Errorf("failed to deserialize save: %s", err)
-	}
-
-	return system, nil
-}
-
-func readSessionSaveData(uuid []byte, slotID int) (defs.SessionSaveData, error) {
-	var session defs.SessionSaveData
-
-	fileName := "session"
-	if slotID != 0 {
-		fileName += strconv.Itoa(slotID)
-	}
-
-	file, err := os.Open(fmt.Sprintf("userdata/%s/%s.pzs", hex.EncodeToString(uuid), fileName))
-	if err != nil {
-		return session, fmt.Errorf("failed to open save file for reading: %s", err)
-	}
-
-	defer file.Close()
-
-	zstdDecoder, err := zstd.NewReader(file)
-	if err != nil {
-		return session, fmt.Errorf("failed to create zstd decoder: %s", err)
-	}
-
-	defer zstdDecoder.Close()
-
-	err = gob.NewDecoder(zstdDecoder).Decode(&session)
-	if err != nil {
-		return session, fmt.Errorf("failed to deserialize save: %s", err)
-	}
-
-	return session, nil
-}
-
 func validateSessionCompleted(session defs.SessionSaveData) bool {
 	switch session.GameMode {
 	case 0:

api/savedata/delete.go

@@ -19,12 +19,9 @@ package savedata
 
 import (
 	"fmt"
-	"log"
-	"os"
-	"strconv"
-
 	"github.com/pagefaultgames/rogueserver/db"
 	"github.com/pagefaultgames/rogueserver/defs"
+	"log"
 )
 
 // /savedata/delete - delete save data
@@ -36,27 +33,14 @@ func Delete(uuid []byte, datatype, slot int) error {
 
 	switch datatype {
 	case 0: // System
-		err := os.Remove(fmt.Sprintf("userdata/%x/system.pzs", uuid))
+		return db.DeleteSystemSaveData(uuid)
-		if err != nil && !os.IsNotExist(err) {
-			return fmt.Errorf("failed to delete save file: %s", err)
-		}
 	case 1: // Session
 		if slot < 0 || slot >= defs.SessionSlotCount {
 			return fmt.Errorf("slot id %d out of range", slot)
 		}
 
-		fileName := "session"
+		return db.DeleteSessionSaveData(uuid, slot)
-		if slot != 0 {
-			fileName += strconv.Itoa(slot)
-		}
-
-		err = os.Remove(fmt.Sprintf("userdata/%x/%s.pzs", uuid, fileName))
-		if err != nil && !os.IsNotExist(err) {
-			return fmt.Errorf("failed to delete save file: %s", err)
-		}
 	default:
 		return fmt.Errorf("invalid data type")
 	}
-
-	return nil
 }

api/savedata/get.go

@@ -29,7 +29,11 @@ import (
 func Get(uuid []byte, datatype, slot int) (any, error) {
 	switch datatype {
 	case 0: // System
-		system, err := readSystemSaveData(uuid)
+		if slot != 0 {
+			return nil, fmt.Errorf("invalid slot id for system data")
+		}
+
+		system, err := db.ReadSystemSaveData(uuid)
 		if err != nil {
 			return nil, err
 		}
@@ -49,7 +53,7 @@ func Get(uuid []byte, datatype, slot int) (any, error) {
 			return nil, fmt.Errorf("slot id %d out of range", slot)
 		}
 
-		session, err := readSessionSaveData(uuid, slot)
+		session, err := db.ReadSessionSaveData(uuid, slot)
 		if err != nil {
 			return nil, err
 		}

api/savedata/update.go

@@ -18,11 +18,8 @@
 package savedata
 
 import (
-	"bytes"
-	"encoding/gob"
 	"fmt"
 	"log"
-	"os"
 	"strconv"
 
 	"github.com/klauspost/compress/zstd"
@@ -39,13 +36,6 @@ func Update(uuid []byte, slot int, save any) error {
 		log.Print("failed to update account last activity")
 	}
 
-	// ideally should have been done at account creation
-	err = os.MkdirAll(fmt.Sprintf("userdata/%x", uuid), 0755)
-	if err != nil && !os.IsExist(err) {
-		return fmt.Errorf(fmt.Sprintf("failed to create userdata folder: %s", err))
-	}
-
-	var filename string
 	switch save := save.(type) {
 	case defs.SystemSaveData: // System
 		if save.TrainerId == 0 && save.SecretId == 0 {
@@ -61,36 +51,26 @@ func Update(uuid []byte, slot int, save any) error {
 			return fmt.Errorf("failed to update account stats: %s", err)
 		}
 
-		filename = "system"
+		err = db.DeleteClaimedAccountCompensations(uuid)
+		if err != nil {
+			return fmt.Errorf("failed to delete claimed compensations: %s", err)
+		}
+
+		return db.StoreSystemSaveData(uuid, save)
 
-		db.DeleteClaimedAccountCompensations(uuid)
 	case defs.SessionSaveData: // Session
 		if slot < 0 || slot >= defs.SessionSlotCount {
 			return fmt.Errorf("slot id %d out of range", slot)
 		}
 
-		filename = "session"
+		filename := "session"
 		if slot != 0 {
 			filename += strconv.Itoa(slot)
 		}
-	default:
-		return fmt.Errorf("invalid data type")
-	}
 
-	var buf bytes.Buffer
+		return db.StoreSessionSaveData(uuid, save, slot)
-	err = gob.NewEncoder(&buf).Encode(save)
-	if err != nil {
-		return fmt.Errorf("failed to serialize save: %s", err)
-	}
-
-	if buf.Len() == 0 {
-		return fmt.Errorf("tried to write empty save file")
-	}
 
-	err = os.WriteFile(fmt.Sprintf("userdata/%x/%s.pzs", uuid, filename), zstdEncoder.EncodeAll(buf.Bytes(), nil), 0644)
+	default:
-	if err != nil {
+		return fmt.Errorf("invalid data type")
-		return fmt.Errorf("failed to write save to disk: %s", err)
 	}
-
-	return nil
 }

db/db.go

@@ -19,9 +19,11 @@ package db
 
 import (
 	"database/sql"
+	"encoding/hex"
 	"fmt"
-
 	_ "github.com/go-sql-driver/mysql"
+	"log"
+	"os"
 )
 
 var handle *sql.DB
@@ -36,5 +38,79 @@ func Init(username, password, protocol, address, database string) error {
 
 	handle.SetMaxOpenConns(1000)
 
+	tx, err := handle.Begin()
+	if err != nil {
+		panic(err)
+	}
+	tx.Exec("CREATE TABLE IF NOT EXISTS systemSaveData (uuid BINARY(16) PRIMARY KEY, data BLOB, timestamp TIMESTAMP)")
+	tx.Exec("CREATE TABLE IF NOT EXISTS sessionSaveData (uuid BINARY(16), slot TINYINT, data BLOB, timestamp TIMESTAMP, PRIMARY KEY (uuid, slot))")
+	err = tx.Commit()
+	if err != nil {
+		panic(err)
+	}
+
+	// TODO temp code
+	entries, err := os.ReadDir("userdata")
+	if err != nil {
+		log.Fatalln(err)
+		return nil
+	}
+
+	for _, entry := range entries {
+		if !entry.IsDir() {
+			continue
+		}
+
+		uuidString := entry.Name()
+		uuid, err := hex.DecodeString(uuidString)
+		if err != nil {
+			log.Printf("failed to decode uuid: %s", err)
+			continue
+		}
+
+		// store new system data
+		systemData, err := LegacyReadSystemSaveData(uuid)
+		if err != nil {
+			log.Printf("failed to read system save data for %v: %s", uuidString, err)
+			continue
+		}
+
+		err = StoreSystemSaveData(uuid, systemData)
+		if err != nil {
+			log.Fatalf("failed to store system save data for %v: %s\n", uuidString, err)
+			continue
+		}
+
+		// delete old system data
+		err = os.Remove("userdata/" + uuidString + "/system.pzs")
+		if err != nil {
+			log.Fatalf("failed to remove legacy system save data for %v: %s", uuidString, err)
+		}
+
+		for i := 0; i < 5; i++ {
+			sessionData, err := LegacyReadSessionSaveData(uuid, i)
+			if err != nil {
+				log.Printf("failed to read session save data %v for %v: %s", i, uuidString, err)
+				continue
+			}
+
+			// store new session data
+			err = StoreSessionSaveData(uuid, sessionData, i)
+			if err != nil {
+				log.Fatalf("failed to store session save data for %v: %s\n", uuidString, err)
+			}
+
+			// delete old session data
+			filename := "session"
+			if i != 0 {
+				filename += fmt.Sprintf("%d", i)
+			}
+			err = os.Remove(fmt.Sprintf("userdata/%s/%s.pzs", uuidString, filename))
+			if err != nil {
+				log.Fatalf("failed to remove legacy session save data %v for %v: %s", i, uuidString, err)
+			}
+		}
+	}
+
 	return nil
 }

db/legacy.go

@@ -0,0 +1,84 @@
+/*
+	Copyright (C) 2024  Pagefault Games
+
+	This program is free software: you can redistribute it and/or modify
+	it under the terms of the GNU Affero General Public License as published by
+	the Free Software Foundation, either version 3 of the License, or
+	(at your option) any later version.
+
+	This program is distributed in the hope that it will be useful,
+	but WITHOUT ANY WARRANTY; without even the implied warranty of
+	MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+	GNU Affero General Public License for more details.
+
+	You should have received a copy of the GNU Affero General Public License
+	along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+package db
+
+import (
+	"encoding/gob"
+	"encoding/hex"
+	"fmt"
+	"os"
+	"strconv"
+
+	"github.com/klauspost/compress/zstd"
+	"github.com/pagefaultgames/rogueserver/defs"
+)
+
+func LegacyReadSystemSaveData(uuid []byte) (defs.SystemSaveData, error) {
+	var system defs.SystemSaveData
+
+	file, err := os.Open("userdata/" + hex.EncodeToString(uuid) + "/system.pzs")
+	if err != nil {
+		return system, fmt.Errorf("failed to open save file for reading: %s", err)
+	}
+
+	defer file.Close()
+
+	zstdDecoder, err := zstd.NewReader(file)
+	if err != nil {
+		return system, fmt.Errorf("failed to create zstd decoder: %s", err)
+	}
+
+	defer zstdDecoder.Close()
+
+	err = gob.NewDecoder(zstdDecoder).Decode(&system)
+	if err != nil {
+		return system, fmt.Errorf("failed to deserialize save: %s", err)
+	}
+
+	return system, nil
+}
+
+func LegacyReadSessionSaveData(uuid []byte, slotID int) (defs.SessionSaveData, error) {
+	var session defs.SessionSaveData
+
+	fileName := "session"
+	if slotID != 0 {
+		fileName += strconv.Itoa(slotID)
+	}
+
+	file, err := os.Open(fmt.Sprintf("userdata/%s/%s.pzs", hex.EncodeToString(uuid), fileName))
+	if err != nil {
+		return session, fmt.Errorf("failed to open save file for reading: %s", err)
+	}
+
+	defer file.Close()
+
+	zstdDecoder, err := zstd.NewReader(file)
+	if err != nil {
+		return session, fmt.Errorf("failed to create zstd decoder: %s", err)
+	}
+
+	defer zstdDecoder.Close()
+
+	err = gob.NewDecoder(zstdDecoder).Decode(&session)
+	if err != nil {
+		return session, fmt.Errorf("failed to deserialize save: %s", err)
+	}
+
+	return session, nil
+}

db/savedata.go

@@ -17,6 +17,12 @@
 
 package db
 
+import (
+	"bytes"
+	"encoding/gob"
+	"github.com/pagefaultgames/rogueserver/defs"
+)
+
 func TryAddDailyRunCompletion(uuid []byte, seed string, mode int) (bool, error) {
 	var count int
 	err := handle.QueryRow("SELECT COUNT(*) FROM dailyRunCompletions WHERE uuid = ? AND seed = ?", uuid, seed).Scan(&count)
@@ -33,3 +39,60 @@ func TryAddDailyRunCompletion(uuid []byte, seed string, mode int) (bool, error)
 
 	return true, nil
 }
+
+func ReadSystemSaveData(uuid []byte) (defs.SystemSaveData, error) {
+	var data []byte
+	err := handle.QueryRow("SELECT data FROM systemSaveData WHERE uuid = ?", uuid).Scan(&data)
+
+	reader := bytes.NewReader(data)
+	system := defs.SystemSaveData{}
+	err = gob.NewDecoder(reader).Decode(&system)
+	return system, err
+}
+
+func StoreSystemSaveData(uuid []byte, data defs.SystemSaveData) error {
+
+	var buf bytes.Buffer
+	err := gob.NewEncoder(&buf).Encode(data)
+	if err != nil {
+		return err
+	}
+
+	_, err = handle.Exec("INSERT INTO systemSaveData (uuid, data, timestamp) VALUES (?, ?, UTC_TIMESTAMP()) ON DUPLICATE KEY UPDATE data = VALUES(data), timestamp = VALUES(timestamp)", uuid, buf.Bytes())
+
+	return err
+}
+
+func DeleteSystemSaveData(uuid []byte) error {
+	_, err := handle.Exec("DELETE FROM systemSaveData WHERE uuid = ?", uuid)
+	return err
+}
+
+func ReadSessionSaveData(uuid []byte, slot int) (defs.SessionSaveData, error) {
+	var data []byte
+	err := handle.QueryRow("SELECT data FROM sessionSaveData WHERE uuid = ? AND slot = ?", uuid, slot).Scan(&data)
+
+	reader := bytes.NewReader(data)
+	save := defs.SessionSaveData{}
+	err = gob.NewDecoder(reader).Decode(&save)
+
+	return save, err
+}
+
+func StoreSessionSaveData(uuid []byte, data defs.SessionSaveData, slot int) error {
+
+	var buf bytes.Buffer
+	err := gob.NewEncoder(&buf).Encode(data)
+	if err != nil {
+		return err
+	}
+
+	_, err = handle.Exec("REPLACE INTO sessionSaveData (uuid, slot, data, timestamp) VALUES (?, ?, ?, UTC_TIMESTAMP())", uuid, slot, buf.Bytes())
+
+	return err
+}
+
+func DeleteSessionSaveData(uuid []byte, slot int) error {
+	_, err := handle.Exec("DELETE FROM sessionSaveData WHERE uuid = ? AND slot = ?", uuid, slot)
+	return err
+}