Move darkstorm_backend to just backend

Added MongoDB instances of DB tables
Updated some DB interfaces
Added logging to count cleanup

go.mod

@@ -4,10 +4,20 @@ go 1.22.3
 
 require (
 	github.com/google/uuid v1.6.0
-	golang.org/x/crypto v0.23.0
+	golang.org/x/crypto v0.24.0
+	github.com/golang-jwt/jwt/v5 v5.2.1
+	go.mongodb.org/mongo-driver v1.15.0
 )
 
 require (
-	github.com/golang-jwt/jwt/v5 v5.2.1
+	github.com/golang/snappy v0.0.1 // indirect
-	golang.org/x/sys v0.20.0 // indirect
+	github.com/klauspost/compress v1.13.6 // indirect
+	github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe // indirect
+	github.com/xdg-go/pbkdf2 v1.0.0 // indirect
+	github.com/xdg-go/scram v1.1.2 // indirect
+	github.com/xdg-go/stringprep v1.0.4 // indirect
+	github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d // indirect
+	golang.org/x/sync v0.7.0 // indirect
+	golang.org/x/text v0.16.0 // indirect
+	golang.org/x/sys v0.21.0 // indirect
 )

go.sum

@@ -1,8 +1,58 @@
+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/golang-jwt/jwt/v5 v5.2.1 h1:OuVbFODueb089Lh128TAcimifWaLhJwVflnrgM17wHk=
 github.com/golang-jwt/jwt/v5 v5.2.1/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk=
+github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4=
+github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
+github.com/google/go-cmp v0.5.2 h1:X2ev0eStA3AbceY54o37/0PQ/UWqKEiiO2dKL5OPaFM=
+github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
 github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
-golang.org/x/crypto v0.23.0 h1:dIJU/v2J8Mdglj/8rJ6UUOM3Zc9zLZxVZwwxMooUSAI=
+github.com/klauspost/compress v1.13.6 h1:P76CopJELS0TiO2mebmnzgWaajssP/EszplttgQxcgc=
-golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8=
+github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk=
-golang.org/x/sys v0.20.0 h1:Od9JTbYCk261bKm4M/mw7AklTlFYIa0bIp9BgSm1S8Y=
+github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe h1:iruDEfMl2E6fbMZ9s0scYfZQ84/6SPL6zC8ACM2oIL0=
-golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc=
+github.com/xdg-go/pbkdf2 v1.0.0 h1:Su7DPu48wXMwC3bs7MCNG+z4FhcyEuz5dlvchbq0B0c=
+github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI=
+github.com/xdg-go/scram v1.1.2 h1:FHX5I5B4i4hKRVRBCFRxq1iQRej7WO3hhBuJf+UUySY=
+github.com/xdg-go/scram v1.1.2/go.mod h1:RT/sEzTbU5y00aCK8UOx6R7YryM0iF1N2MOmC3kKLN4=
+github.com/xdg-go/stringprep v1.0.4 h1:XLI/Ng3O1Atzq0oBs3TWm+5ZVgkq2aqdlvP9JtoZ6c8=
+github.com/xdg-go/stringprep v1.0.4/go.mod h1:mPGuuIYwz7CmR2bT9j4GbQqutWS1zV24gijq1dTyGkM=
+github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d h1:splanxYIlg+5LfHAM6xpdFEAYOk8iySO56hMFq6uLyA=
+github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d/go.mod h1:rHwXgn7JulP+udvsHwJoVG1YGAP6VLg4y9I5dyZdqmA=
+github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
+go.mongodb.org/mongo-driver v1.15.0 h1:rJCKC8eEliewXjZGf0ddURtl7tTVy1TK3bfl0gkUSLc=
+go.mongodb.org/mongo-driver v1.15.0/go.mod h1:Vzb0Mk/pa7e6cWw85R4F/endUC3u0U9jGcNU603k65c=
+golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
+golang.org/x/crypto v0.24.0 h1:mnl8DM0o513X8fdIkmyFE/5hTYxbwYOjDS/+rK6qpRI=
+golang.org/x/crypto v0.24.0/go.mod h1:Z1PMYSOR5nyMcyAVAIQSKCDwalqy85Aqn1x3Ws4L5DM=
+golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
+golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
+golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
+golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M=
+golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
+golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.21.0 h1:rF+pYz3DAGSQAxAu1CbC7catZg4ebC4UIeIhKxBZvws=
+golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
+golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
+golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
+golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ=
+golang.org/x/text v0.16.0 h1:a94ExnEXNtEwYLGJSIUxnWoxoRz/ZcCsV63ROupILh4=
+golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI=
+golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
+golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4=
+golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=

internal/backend/app.go

@@ -1,4 +1,4 @@
-package darkstorm
+package backend
 
 import "net/http"
 

internal/backend/crash.go

@@ -1,4 +1,4 @@
-package darkstorm
+package backend
 
 import (
 	"encoding/json"

internal/backend/darkstorm.go

@@ -1,9 +1,10 @@
-package darkstorm
+package backend
 
 import (
 	"crypto/ed25519"
 	"encoding/json"
 	"errors"
+	"log"
 	"net/http"
 	"sync"
 	"time"
@@ -60,12 +61,17 @@ func NewBackend(keyTable Table[ApiKey], apps ...App) (*Backend, error) {
 func (b *Backend) cleanupLoop() {
 	for range time.Tick(24 * time.Hour) {
 		old := getDate(time.Now().Add(-30 * 24 * time.Hour))
+		var err error
 		for _, a := range b.apps {
+			log.Printf("Removing logs for %v", a.AppID())
 			tab := a.CountTable()
 			if tab == nil {
 				continue
 			}
-			tab.RemoveOldLogs(old)
+			err = tab.RemoveOldLogs(old)
+			if err != nil {
+				log.Printf("error removing old logs for %v: %v\n", a.AppID(), err)
+			}
 		}
 	}
 }

internal/backend/darkstorm_test.go

@@ -1,4 +1,4 @@
-package darkstorm
+package backend
 
 import (
 	"fmt"

internal/backend/db.go

@@ -1,4 +1,4 @@
-package darkstorm
+package backend
 
 import "errors"
 
@@ -11,7 +11,7 @@ type IDStruct interface {
 }
 
 type Table[T IDStruct] interface {
-	Get(ID string) (data T, err error)
+	Get(ID string) (data *T, err error)
 	Find(values map[string]any) ([]T, error)
 	Insert(data T) error
 	Remove(ID string) error
@@ -22,9 +22,9 @@ type Table[T IDStruct] interface {
 type CountTable interface {
 	Table[CountLog]
 	// Remove all Log items that have a CountLog.Date value less then the given value.
-	RemoveOldLogs(date int)
+	RemoveOldLogs(date int) error
 	// Get count. If platform is an empty string or "all", the full count should be given
-	Count(platform string) int
+	Count(platform string) (int, error)
 }
 
 type CrashTable interface {

internal/backend/db/mongo.go

@@ -0,0 +1,83 @@
+package db
+
+import (
+	"context"
+
+	"github.com/CalebQ42/darkstorm-server/internal/backend"
+	"go.mongodb.org/mongo-driver/bson"
+	"go.mongodb.org/mongo-driver/mongo"
+)
+
+type MongoTable[T backend.IDStruct] struct {
+	col *mongo.Collection
+}
+
+func NewMongoTable[T backend.IDStruct](col *mongo.Collection) *MongoTable[T] {
+	return &MongoTable[T]{
+		col: col,
+	}
+}
+
+func (m *MongoTable[T]) Get(ID string) (data *T, err error) {
+	res := m.col.FindOne(context.Background(), bson.M{"_id": ID})
+	if res.Err() == mongo.ErrNoDocuments {
+		return nil, backend.ErrNotFound
+	} else if res.Err() != nil {
+		return nil, res.Err()
+	}
+	var out T
+	err = res.Decode(&out)
+	return &out, err
+}
+
+func (m *MongoTable[T]) Find(values map[string]any) ([]T, error) {
+	res, err := m.col.Find(context.Background(), values)
+	if err == mongo.ErrNoDocuments {
+		return nil, backend.ErrNotFound
+	} else if err != nil {
+		return nil, err
+	}
+	var out []T
+	err = res.All(context.Background(), &out)
+	return out, err
+}
+
+func (m *MongoTable[T]) Insert(data T) error {
+	_, err := m.col.InsertOne(context.Background(), data)
+	return err
+}
+
+func (m *MongoTable[T]) Remove(ID string) error {
+	res := m.col.FindOneAndDelete(context.Background(), bson.M{"_id": ID})
+	return res.Err()
+}
+
+func (m *MongoTable[T]) FullUpdate(ID string, data T) error {
+	res := m.col.FindOneAndReplace(context.Background(), bson.M{"_id": ID}, data)
+	if res.Err() == mongo.ErrNoDocuments {
+		return backend.ErrNotFound
+	}
+	return res.Err()
+}
+
+func (m *MongoTable[T]) PartUpdate(ID string, update map[string]any) error {
+	res := m.col.FindOneAndUpdate(context.Background(), bson.M{"_id": ID}, update)
+	if res.Err() == mongo.ErrNoDocuments {
+		return backend.ErrNotFound
+	}
+	return res.Err()
+}
+
+func (m *MongoTable[CountLog]) RemoveOldLogs(date int) {
+	m.col.DeleteMany(context.Background(), bson.M{"date": bson.M{"$lt": date}})
+}
+func (m *MongoTable[CountLog]) Count(platform string) (int, error) {
+	var filter bson.M
+	if platform == "" || platform == "all" {
+		filter = bson.M{}
+	} else {
+		filter = bson.M{"platform": platform}
+	}
+	out, err := m.col.CountDocuments(context.Background(), filter)
+	return int(out), err
+}

internal/backend/db/mongo_crash.go

@@ -0,0 +1,70 @@
+package db
+
+import (
+	"context"
+	"strings"
+
+	"github.com/CalebQ42/darkstorm-server/internal/backend"
+	"github.com/google/uuid"
+	"go.mongodb.org/mongo-driver/bson"
+	"go.mongodb.org/mongo-driver/mongo"
+)
+
+type MongoCrashTable struct {
+	*MongoTable[backend.CrashReport]
+	archiveCol *mongo.Collection
+}
+
+func NewMongoCrashTable(crashCol *mongo.Collection, archiveCol *mongo.Collection) *MongoCrashTable {
+	return &MongoCrashTable{
+		MongoTable: NewMongoTable[backend.CrashReport](crashCol),
+		archiveCol: archiveCol,
+	}
+}
+
+func (m *MongoCrashTable) Archive(toArchive backend.ArchivedCrash) error {
+	if toArchive.Platform == "" {
+		toArchive.Platform = "all"
+	}
+	_, err := m.archiveCol.InsertOne(context.Background(), toArchive)
+	return err
+}
+
+func (m *MongoCrashTable) IsArchived(ind backend.IndividualCrash) bool {
+	res := m.archiveCol.FindOne(context.Background(),
+		bson.M{"error": ind.Error, "stack": ind.Stack, "platform": bson.M{"$in": []string{ind.Platform, "all"}}},
+	)
+	return res.Err() == nil
+}
+
+func (m *MongoCrashTable) InsertCrash(ind backend.IndividualCrash) error {
+	first, _, _ := strings.Cut(ind.Stack, "\n")
+	_, err := m.col.UpdateOne(context.Background(),
+		bson.M{"error": ind.Error, "firstLine": first, //filter main report
+			"individual.stack": ind.Stack, "individual.platform": ind.Platform}, //filter individual
+		bson.M{"$inc": bson.M{"individual.count": 1}}, //increment count
+	)
+	if err == mongo.ErrNoDocuments {
+		ind.Count = 1
+		_, err = m.col.UpdateOne(context.Background(),
+			bson.M{"error": ind.Error, "firstLine": first}, //filter
+			bson.M{"$push": bson.M{"individual": ind}},     //Add new individual report
+		)
+		if err == mongo.ErrNoDocuments {
+			var id uuid.UUID
+			id, err = uuid.NewV7()
+			if err != nil {
+				return err
+			}
+			_, err = m.col.InsertOne(context.Background(),
+				backend.CrashReport{
+					ID:         id.String(),
+					Error:      ind.Error,
+					FirstLine:  first,
+					Individual: []backend.IndividualCrash{ind},
+				},
+			)
+		}
+	}
+	return err
+}

internal/backend/db/valkey.go

@@ -0,0 +1 @@
+package db

internal/backend/header.go

@@ -1,4 +1,4 @@
-package darkstorm
+package backend
 
 import (
 	"errors"
@@ -36,7 +36,7 @@ func (b *Backend) ParseHeader(r *http.Request) (*ParsedHeader, error) {
 		if apiKey.Death > 0 && time.Unix(apiKey.Death, 0).Before(time.Now()) {
 			return nil, ErrApiKeyUnauthorized
 		}
-		out.Key = &apiKey
+		out.Key = apiKey
 	}
 	if token != "" && b.userTable != nil {
 		t, err := jwt.Parse(token, func(t *jwt.Token) (interface{}, error) {

internal/backend/key.go

@@ -1,4 +1,4 @@
-package darkstorm
+package backend
 
 type ApiKey struct {
 	Perm  map[string]bool

internal/backend/log.go

@@ -1,4 +1,4 @@
-package darkstorm
+package backend
 
 import (
 	"encoding/json"
@@ -120,6 +120,10 @@ func (b *Backend) getCount(w http.ResponseWriter, r *http.Request) {
 		ReturnError(w, http.StatusBadRequest, "badRequest", "Trying to get user count on app that doesn't have a count table")
 		return
 	}
-	out := count.Count(r.URL.Query().Get("platform"))
+	out, err := count.Count(r.URL.Query().Get("platform"))
+	if err != nil {
+		ReturnError(w, http.StatusInternalServerError, "internal", "Server error")
+		return
+	}
 	json.NewEncoder(w).Encode(map[string]int{"count": out})
 }

internal/backend/user.go

@@ -1,4 +1,4 @@
-package darkstorm
+package backend
 
 import (
 	"crypto/rand"