Move darkstorm_backend to just backend

Added MongoDB instances of DB tables
Updated some DB interfaces
Added logging to count cleanup

internal/backend/app.go

@@ -1,4 +1,4 @@
-package darkstorm
+package backend
 
 import "net/http"
 

internal/backend/crash.go

@@ -1,4 +1,4 @@
-package darkstorm
+package backend
 
 import (
 	"encoding/json"

internal/backend/darkstorm.go

@@ -1,9 +1,10 @@
-package darkstorm
+package backend
 
 import (
 	"crypto/ed25519"
 	"encoding/json"
 	"errors"
+	"log"
 	"net/http"
 	"sync"
 	"time"
@@ -60,12 +61,17 @@ func NewBackend(keyTable Table[ApiKey], apps ...App) (*Backend, error) {
 func (b *Backend) cleanupLoop() {
 	for range time.Tick(24 * time.Hour) {
 		old := getDate(time.Now().Add(-30 * 24 * time.Hour))
+		var err error
 		for _, a := range b.apps {
+			log.Printf("Removing logs for %v", a.AppID())
 			tab := a.CountTable()
 			if tab == nil {
 				continue
 			}
-			tab.RemoveOldLogs(old)
+			err = tab.RemoveOldLogs(old)
+			if err != nil {
+				log.Printf("error removing old logs for %v: %v\n", a.AppID(), err)
+			}
 		}
 	}
 }

internal/backend/darkstorm_test.go

@@ -1,4 +1,4 @@
-package darkstorm
+package backend
 
 import (
 	"fmt"

internal/backend/db.go

@@ -1,4 +1,4 @@
-package darkstorm
+package backend
 
 import "errors"
 
@@ -11,7 +11,7 @@ type IDStruct interface {
 }
 
 type Table[T IDStruct] interface {
-	Get(ID string) (data T, err error)
+	Get(ID string) (data *T, err error)
 	Find(values map[string]any) ([]T, error)
 	Insert(data T) error
 	Remove(ID string) error
@@ -22,9 +22,9 @@ type Table[T IDStruct] interface {
 type CountTable interface {
 	Table[CountLog]
 	// Remove all Log items that have a CountLog.Date value less then the given value.
-	RemoveOldLogs(date int)
+	RemoveOldLogs(date int) error
 	// Get count. If platform is an empty string or "all", the full count should be given
-	Count(platform string) int
+	Count(platform string) (int, error)
 }
 
 type CrashTable interface {

internal/backend/db/mongo.go

@@ -0,0 +1,83 @@
+package db
+
+import (
+	"context"
+
+	"github.com/CalebQ42/darkstorm-server/internal/backend"
+	"go.mongodb.org/mongo-driver/bson"
+	"go.mongodb.org/mongo-driver/mongo"
+)
+
+type MongoTable[T backend.IDStruct] struct {
+	col *mongo.Collection
+}
+
+func NewMongoTable[T backend.IDStruct](col *mongo.Collection) *MongoTable[T] {
+	return &MongoTable[T]{
+		col: col,
+	}
+}
+
+func (m *MongoTable[T]) Get(ID string) (data *T, err error) {
+	res := m.col.FindOne(context.Background(), bson.M{"_id": ID})
+	if res.Err() == mongo.ErrNoDocuments {
+		return nil, backend.ErrNotFound
+	} else if res.Err() != nil {
+		return nil, res.Err()
+	}
+	var out T
+	err = res.Decode(&out)
+	return &out, err
+}
+
+func (m *MongoTable[T]) Find(values map[string]any) ([]T, error) {
+	res, err := m.col.Find(context.Background(), values)
+	if err == mongo.ErrNoDocuments {
+		return nil, backend.ErrNotFound
+	} else if err != nil {
+		return nil, err
+	}
+	var out []T
+	err = res.All(context.Background(), &out)
+	return out, err
+}
+
+func (m *MongoTable[T]) Insert(data T) error {
+	_, err := m.col.InsertOne(context.Background(), data)
+	return err
+}
+
+func (m *MongoTable[T]) Remove(ID string) error {
+	res := m.col.FindOneAndDelete(context.Background(), bson.M{"_id": ID})
+	return res.Err()
+}
+
+func (m *MongoTable[T]) FullUpdate(ID string, data T) error {
+	res := m.col.FindOneAndReplace(context.Background(), bson.M{"_id": ID}, data)
+	if res.Err() == mongo.ErrNoDocuments {
+		return backend.ErrNotFound
+	}
+	return res.Err()
+}
+
+func (m *MongoTable[T]) PartUpdate(ID string, update map[string]any) error {
+	res := m.col.FindOneAndUpdate(context.Background(), bson.M{"_id": ID}, update)
+	if res.Err() == mongo.ErrNoDocuments {
+		return backend.ErrNotFound
+	}
+	return res.Err()
+}
+
+func (m *MongoTable[CountLog]) RemoveOldLogs(date int) {
+	m.col.DeleteMany(context.Background(), bson.M{"date": bson.M{"$lt": date}})
+}
+func (m *MongoTable[CountLog]) Count(platform string) (int, error) {
+	var filter bson.M
+	if platform == "" || platform == "all" {
+		filter = bson.M{}
+	} else {
+		filter = bson.M{"platform": platform}
+	}
+	out, err := m.col.CountDocuments(context.Background(), filter)
+	return int(out), err
+}

internal/backend/db/mongo_crash.go

@@ -0,0 +1,70 @@
+package db
+
+import (
+	"context"
+	"strings"
+
+	"github.com/CalebQ42/darkstorm-server/internal/backend"
+	"github.com/google/uuid"
+	"go.mongodb.org/mongo-driver/bson"
+	"go.mongodb.org/mongo-driver/mongo"
+)
+
+type MongoCrashTable struct {
+	*MongoTable[backend.CrashReport]
+	archiveCol *mongo.Collection
+}
+
+func NewMongoCrashTable(crashCol *mongo.Collection, archiveCol *mongo.Collection) *MongoCrashTable {
+	return &MongoCrashTable{
+		MongoTable: NewMongoTable[backend.CrashReport](crashCol),
+		archiveCol: archiveCol,
+	}
+}
+
+func (m *MongoCrashTable) Archive(toArchive backend.ArchivedCrash) error {
+	if toArchive.Platform == "" {
+		toArchive.Platform = "all"
+	}
+	_, err := m.archiveCol.InsertOne(context.Background(), toArchive)
+	return err
+}
+
+func (m *MongoCrashTable) IsArchived(ind backend.IndividualCrash) bool {
+	res := m.archiveCol.FindOne(context.Background(),
+		bson.M{"error": ind.Error, "stack": ind.Stack, "platform": bson.M{"$in": []string{ind.Platform, "all"}}},
+	)
+	return res.Err() == nil
+}
+
+func (m *MongoCrashTable) InsertCrash(ind backend.IndividualCrash) error {
+	first, _, _ := strings.Cut(ind.Stack, "\n")
+	_, err := m.col.UpdateOne(context.Background(),
+		bson.M{"error": ind.Error, "firstLine": first, //filter main report
+			"individual.stack": ind.Stack, "individual.platform": ind.Platform}, //filter individual
+		bson.M{"$inc": bson.M{"individual.count": 1}}, //increment count
+	)
+	if err == mongo.ErrNoDocuments {
+		ind.Count = 1
+		_, err = m.col.UpdateOne(context.Background(),
+			bson.M{"error": ind.Error, "firstLine": first}, //filter
+			bson.M{"$push": bson.M{"individual": ind}},     //Add new individual report
+		)
+		if err == mongo.ErrNoDocuments {
+			var id uuid.UUID
+			id, err = uuid.NewV7()
+			if err != nil {
+				return err
+			}
+			_, err = m.col.InsertOne(context.Background(),
+				backend.CrashReport{
+					ID:         id.String(),
+					Error:      ind.Error,
+					FirstLine:  first,
+					Individual: []backend.IndividualCrash{ind},
+				},
+			)
+		}
+	}
+	return err
+}

internal/backend/db/valkey.go

@@ -0,0 +1 @@
+package db

internal/backend/header.go

@@ -1,4 +1,4 @@
-package darkstorm
+package backend
 
 import (
 	"errors"
@@ -36,7 +36,7 @@ func (b *Backend) ParseHeader(r *http.Request) (*ParsedHeader, error) {
 		if apiKey.Death > 0 && time.Unix(apiKey.Death, 0).Before(time.Now()) {
 			return nil, ErrApiKeyUnauthorized
 		}
-		out.Key = &apiKey
+		out.Key = apiKey
 	}
 	if token != "" && b.userTable != nil {
 		t, err := jwt.Parse(token, func(t *jwt.Token) (interface{}, error) {

internal/backend/key.go

@@ -1,4 +1,4 @@
-package darkstorm
+package backend
 
 type ApiKey struct {
 	Perm  map[string]bool

internal/backend/log.go

@@ -1,4 +1,4 @@
-package darkstorm
+package backend
 
 import (
 	"encoding/json"
@@ -120,6 +120,10 @@ func (b *Backend) getCount(w http.ResponseWriter, r *http.Request) {
 		ReturnError(w, http.StatusBadRequest, "badRequest", "Trying to get user count on app that doesn't have a count table")
 		return
 	}
-	out := count.Count(r.URL.Query().Get("platform"))
+	out, err := count.Count(r.URL.Query().Get("platform"))
+	if err != nil {
+		ReturnError(w, http.StatusInternalServerError, "internal", "Server error")
+		return
+	}
 	json.NewEncoder(w).Encode(map[string]int{"count": out})
 }

internal/backend/user.go

@@ -1,4 +1,4 @@
-package darkstorm
+package backend
 
 import (
 	"crypto/rand"