diff --git a/go.mod b/go.mod index 9131155..b115052 100644 --- a/go.mod +++ b/go.mod @@ -4,10 +4,20 @@ go 1.22.3 require ( github.com/google/uuid v1.6.0 - golang.org/x/crypto v0.23.0 + golang.org/x/crypto v0.24.0 + github.com/golang-jwt/jwt/v5 v5.2.1 + go.mongodb.org/mongo-driver v1.15.0 ) require ( - github.com/golang-jwt/jwt/v5 v5.2.1 - golang.org/x/sys v0.20.0 // indirect + github.com/golang/snappy v0.0.1 // indirect + github.com/klauspost/compress v1.13.6 // indirect + github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe // indirect + github.com/xdg-go/pbkdf2 v1.0.0 // indirect + github.com/xdg-go/scram v1.1.2 // indirect + github.com/xdg-go/stringprep v1.0.4 // indirect + github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d // indirect + golang.org/x/sync v0.7.0 // indirect + golang.org/x/text v0.16.0 // indirect + golang.org/x/sys v0.21.0 // indirect ) diff --git a/go.sum b/go.sum index c5c1062..ea5a8fd 100644 --- a/go.sum +++ b/go.sum @@ -1,8 +1,58 @@ +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/golang-jwt/jwt/v5 v5.2.1 h1:OuVbFODueb089Lh128TAcimifWaLhJwVflnrgM17wHk= github.com/golang-jwt/jwt/v5 v5.2.1/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk= +github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4= +github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/google/go-cmp v0.5.2 h1:X2ev0eStA3AbceY54o37/0PQ/UWqKEiiO2dKL5OPaFM= +github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -golang.org/x/crypto v0.23.0 h1:dIJU/v2J8Mdglj/8rJ6UUOM3Zc9zLZxVZwwxMooUSAI= -golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8= -golang.org/x/sys v0.20.0 h1:Od9JTbYCk261bKm4M/mw7AklTlFYIa0bIp9BgSm1S8Y= -golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +github.com/klauspost/compress v1.13.6 h1:P76CopJELS0TiO2mebmnzgWaajssP/EszplttgQxcgc= +github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= +github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe h1:iruDEfMl2E6fbMZ9s0scYfZQ84/6SPL6zC8ACM2oIL0= +github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc= +github.com/xdg-go/pbkdf2 v1.0.0 h1:Su7DPu48wXMwC3bs7MCNG+z4FhcyEuz5dlvchbq0B0c= +github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI= +github.com/xdg-go/scram v1.1.2 h1:FHX5I5B4i4hKRVRBCFRxq1iQRej7WO3hhBuJf+UUySY= +github.com/xdg-go/scram v1.1.2/go.mod h1:RT/sEzTbU5y00aCK8UOx6R7YryM0iF1N2MOmC3kKLN4= +github.com/xdg-go/stringprep v1.0.4 h1:XLI/Ng3O1Atzq0oBs3TWm+5ZVgkq2aqdlvP9JtoZ6c8= +github.com/xdg-go/stringprep v1.0.4/go.mod h1:mPGuuIYwz7CmR2bT9j4GbQqutWS1zV24gijq1dTyGkM= +github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d h1:splanxYIlg+5LfHAM6xpdFEAYOk8iySO56hMFq6uLyA= +github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d/go.mod h1:rHwXgn7JulP+udvsHwJoVG1YGAP6VLg4y9I5dyZdqmA= +github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= +go.mongodb.org/mongo-driver v1.15.0 h1:rJCKC8eEliewXjZGf0ddURtl7tTVy1TK3bfl0gkUSLc= +go.mongodb.org/mongo-driver v1.15.0/go.mod h1:Vzb0Mk/pa7e6cWw85R4F/endUC3u0U9jGcNU603k65c= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.24.0 h1:mnl8DM0o513X8fdIkmyFE/5hTYxbwYOjDS/+rK6qpRI= +golang.org/x/crypto v0.24.0/go.mod h1:Z1PMYSOR5nyMcyAVAIQSKCDwalqy85Aqn1x3Ws4L5DM= +golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M= +golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.21.0 h1:rF+pYz3DAGSQAxAu1CbC7catZg4ebC4UIeIhKxBZvws= +golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= +golang.org/x/text v0.16.0 h1:a94ExnEXNtEwYLGJSIUxnWoxoRz/ZcCsV63ROupILh4= +golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= diff --git a/internal/darkstorm_backend/README.md b/internal/backend/README.md similarity index 100% rename from internal/darkstorm_backend/README.md rename to internal/backend/README.md diff --git a/internal/darkstorm_backend/app.go b/internal/backend/app.go similarity index 95% rename from internal/darkstorm_backend/app.go rename to internal/backend/app.go index 6b3d812..3db5d1c 100644 --- a/internal/darkstorm_backend/app.go +++ b/internal/backend/app.go @@ -1,4 +1,4 @@ -package darkstorm +package backend import "net/http" diff --git a/internal/darkstorm_backend/crash.go b/internal/backend/crash.go similarity index 99% rename from internal/darkstorm_backend/crash.go rename to internal/backend/crash.go index 73b660e..9ee767b 100644 --- a/internal/darkstorm_backend/crash.go +++ b/internal/backend/crash.go @@ -1,4 +1,4 @@ -package darkstorm +package backend import ( "encoding/json" diff --git a/internal/darkstorm_backend/darkstorm.go b/internal/backend/darkstorm.go similarity index 92% rename from internal/darkstorm_backend/darkstorm.go rename to internal/backend/darkstorm.go index 06e5d05..1b9aa57 100644 --- a/internal/darkstorm_backend/darkstorm.go +++ b/internal/backend/darkstorm.go @@ -1,9 +1,10 @@ -package darkstorm +package backend import ( "crypto/ed25519" "encoding/json" "errors" + "log" "net/http" "sync" "time" @@ -60,12 +61,17 @@ func NewBackend(keyTable Table[ApiKey], apps ...App) (*Backend, error) { func (b *Backend) cleanupLoop() { for range time.Tick(24 * time.Hour) { old := getDate(time.Now().Add(-30 * 24 * time.Hour)) + var err error for _, a := range b.apps { + log.Printf("Removing logs for %v", a.AppID()) tab := a.CountTable() if tab == nil { continue } - tab.RemoveOldLogs(old) + err = tab.RemoveOldLogs(old) + if err != nil { + log.Printf("error removing old logs for %v: %v\n", a.AppID(), err) + } } } } diff --git a/internal/darkstorm_backend/darkstorm_test.go b/internal/backend/darkstorm_test.go similarity index 94% rename from internal/darkstorm_backend/darkstorm_test.go rename to internal/backend/darkstorm_test.go index fd1346e..c33a15a 100644 --- a/internal/darkstorm_backend/darkstorm_test.go +++ b/internal/backend/darkstorm_test.go @@ -1,4 +1,4 @@ -package darkstorm +package backend import ( "fmt" diff --git a/internal/darkstorm_backend/db.go b/internal/backend/db.go similarity index 89% rename from internal/darkstorm_backend/db.go rename to internal/backend/db.go index 189f993..83b228c 100644 --- a/internal/darkstorm_backend/db.go +++ b/internal/backend/db.go @@ -1,4 +1,4 @@ -package darkstorm +package backend import "errors" @@ -11,7 +11,7 @@ type IDStruct interface { } type Table[T IDStruct] interface { - Get(ID string) (data T, err error) + Get(ID string) (data *T, err error) Find(values map[string]any) ([]T, error) Insert(data T) error Remove(ID string) error @@ -22,9 +22,9 @@ type Table[T IDStruct] interface { type CountTable interface { Table[CountLog] // Remove all Log items that have a CountLog.Date value less then the given value. - RemoveOldLogs(date int) + RemoveOldLogs(date int) error // Get count. If platform is an empty string or "all", the full count should be given - Count(platform string) int + Count(platform string) (int, error) } type CrashTable interface { diff --git a/internal/backend/db/mongo.go b/internal/backend/db/mongo.go new file mode 100644 index 0000000..c49d718 --- /dev/null +++ b/internal/backend/db/mongo.go @@ -0,0 +1,83 @@ +package db + +import ( + "context" + + "github.com/CalebQ42/darkstorm-server/internal/backend" + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/mongo" +) + +type MongoTable[T backend.IDStruct] struct { + col *mongo.Collection +} + +func NewMongoTable[T backend.IDStruct](col *mongo.Collection) *MongoTable[T] { + return &MongoTable[T]{ + col: col, + } +} + +func (m *MongoTable[T]) Get(ID string) (data *T, err error) { + res := m.col.FindOne(context.Background(), bson.M{"_id": ID}) + if res.Err() == mongo.ErrNoDocuments { + return nil, backend.ErrNotFound + } else if res.Err() != nil { + return nil, res.Err() + } + var out T + err = res.Decode(&out) + return &out, err +} + +func (m *MongoTable[T]) Find(values map[string]any) ([]T, error) { + res, err := m.col.Find(context.Background(), values) + if err == mongo.ErrNoDocuments { + return nil, backend.ErrNotFound + } else if err != nil { + return nil, err + } + var out []T + err = res.All(context.Background(), &out) + return out, err +} + +func (m *MongoTable[T]) Insert(data T) error { + _, err := m.col.InsertOne(context.Background(), data) + return err +} + +func (m *MongoTable[T]) Remove(ID string) error { + res := m.col.FindOneAndDelete(context.Background(), bson.M{"_id": ID}) + return res.Err() +} + +func (m *MongoTable[T]) FullUpdate(ID string, data T) error { + res := m.col.FindOneAndReplace(context.Background(), bson.M{"_id": ID}, data) + if res.Err() == mongo.ErrNoDocuments { + return backend.ErrNotFound + } + return res.Err() +} + +func (m *MongoTable[T]) PartUpdate(ID string, update map[string]any) error { + res := m.col.FindOneAndUpdate(context.Background(), bson.M{"_id": ID}, update) + if res.Err() == mongo.ErrNoDocuments { + return backend.ErrNotFound + } + return res.Err() +} + +func (m *MongoTable[CountLog]) RemoveOldLogs(date int) { + m.col.DeleteMany(context.Background(), bson.M{"date": bson.M{"$lt": date}}) +} +func (m *MongoTable[CountLog]) Count(platform string) (int, error) { + var filter bson.M + if platform == "" || platform == "all" { + filter = bson.M{} + } else { + filter = bson.M{"platform": platform} + } + out, err := m.col.CountDocuments(context.Background(), filter) + return int(out), err +} diff --git a/internal/backend/db/mongo_crash.go b/internal/backend/db/mongo_crash.go new file mode 100644 index 0000000..9ee092e --- /dev/null +++ b/internal/backend/db/mongo_crash.go @@ -0,0 +1,70 @@ +package db + +import ( + "context" + "strings" + + "github.com/CalebQ42/darkstorm-server/internal/backend" + "github.com/google/uuid" + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/mongo" +) + +type MongoCrashTable struct { + *MongoTable[backend.CrashReport] + archiveCol *mongo.Collection +} + +func NewMongoCrashTable(crashCol *mongo.Collection, archiveCol *mongo.Collection) *MongoCrashTable { + return &MongoCrashTable{ + MongoTable: NewMongoTable[backend.CrashReport](crashCol), + archiveCol: archiveCol, + } +} + +func (m *MongoCrashTable) Archive(toArchive backend.ArchivedCrash) error { + if toArchive.Platform == "" { + toArchive.Platform = "all" + } + _, err := m.archiveCol.InsertOne(context.Background(), toArchive) + return err +} + +func (m *MongoCrashTable) IsArchived(ind backend.IndividualCrash) bool { + res := m.archiveCol.FindOne(context.Background(), + bson.M{"error": ind.Error, "stack": ind.Stack, "platform": bson.M{"$in": []string{ind.Platform, "all"}}}, + ) + return res.Err() == nil +} + +func (m *MongoCrashTable) InsertCrash(ind backend.IndividualCrash) error { + first, _, _ := strings.Cut(ind.Stack, "\n") + _, err := m.col.UpdateOne(context.Background(), + bson.M{"error": ind.Error, "firstLine": first, //filter main report + "individual.stack": ind.Stack, "individual.platform": ind.Platform}, //filter individual + bson.M{"$inc": bson.M{"individual.count": 1}}, //increment count + ) + if err == mongo.ErrNoDocuments { + ind.Count = 1 + _, err = m.col.UpdateOne(context.Background(), + bson.M{"error": ind.Error, "firstLine": first}, //filter + bson.M{"$push": bson.M{"individual": ind}}, //Add new individual report + ) + if err == mongo.ErrNoDocuments { + var id uuid.UUID + id, err = uuid.NewV7() + if err != nil { + return err + } + _, err = m.col.InsertOne(context.Background(), + backend.CrashReport{ + ID: id.String(), + Error: ind.Error, + FirstLine: first, + Individual: []backend.IndividualCrash{ind}, + }, + ) + } + } + return err +} diff --git a/internal/backend/db/valkey.go b/internal/backend/db/valkey.go new file mode 100644 index 0000000..10060e9 --- /dev/null +++ b/internal/backend/db/valkey.go @@ -0,0 +1 @@ +package db \ No newline at end of file diff --git a/internal/darkstorm_backend/header.go b/internal/backend/header.go similarity index 99% rename from internal/darkstorm_backend/header.go rename to internal/backend/header.go index cfcbe0c..09ea247 100644 --- a/internal/darkstorm_backend/header.go +++ b/internal/backend/header.go @@ -1,4 +1,4 @@ -package darkstorm +package backend import ( "errors" @@ -36,7 +36,7 @@ func (b *Backend) ParseHeader(r *http.Request) (*ParsedHeader, error) { if apiKey.Death > 0 && time.Unix(apiKey.Death, 0).Before(time.Now()) { return nil, ErrApiKeyUnauthorized } - out.Key = &apiKey + out.Key = apiKey } if token != "" && b.userTable != nil { t, err := jwt.Parse(token, func(t *jwt.Token) (interface{}, error) { diff --git a/internal/darkstorm_backend/key.go b/internal/backend/key.go similarity index 88% rename from internal/darkstorm_backend/key.go rename to internal/backend/key.go index 0fca0fd..f37a093 100644 --- a/internal/darkstorm_backend/key.go +++ b/internal/backend/key.go @@ -1,4 +1,4 @@ -package darkstorm +package backend type ApiKey struct { Perm map[string]bool diff --git a/internal/darkstorm_backend/log.go b/internal/backend/log.go similarity index 94% rename from internal/darkstorm_backend/log.go rename to internal/backend/log.go index 0d6972d..c84fa3d 100644 --- a/internal/darkstorm_backend/log.go +++ b/internal/backend/log.go @@ -1,4 +1,4 @@ -package darkstorm +package backend import ( "encoding/json" @@ -120,6 +120,10 @@ func (b *Backend) getCount(w http.ResponseWriter, r *http.Request) { ReturnError(w, http.StatusBadRequest, "badRequest", "Trying to get user count on app that doesn't have a count table") return } - out := count.Count(r.URL.Query().Get("platform")) + out, err := count.Count(r.URL.Query().Get("platform")) + if err != nil { + ReturnError(w, http.StatusInternalServerError, "internal", "Server error") + return + } json.NewEncoder(w).Encode(map[string]int{"count": out}) } diff --git a/internal/darkstorm_backend/user.go b/internal/backend/user.go similarity index 99% rename from internal/darkstorm_backend/user.go rename to internal/backend/user.go index 4555326..120712c 100644 --- a/internal/darkstorm_backend/user.go +++ b/internal/backend/user.go @@ -1,4 +1,4 @@ -package darkstorm +package backend import ( "crypto/rand"