diff --git a/cmd/server.go b/cmd/server.go index 3715cac..7e3641b 100644 --- a/cmd/server.go +++ b/cmd/server.go @@ -111,7 +111,9 @@ func InitDatabase() { } func DoRecordsWork() { - ticker := time.NewTicker(time.Hour * 1) + ticker := time.NewTicker(time.Minute * 30) + records.DeleteRecordBefore(time.Now().Add(-time.Hour * 24 * 30)) + records.CompactRecord() for range ticker.C { records.DeleteRecordBefore(time.Now().Add(-time.Hour * 24 * 30)) records.CompactRecord() diff --git a/database/clients/report.go b/database/clients/report.go index 0134c50..2a6e366 100644 --- a/database/clients/report.go +++ b/database/clients/report.go @@ -3,6 +3,7 @@ package clients import ( "encoding/json" "fmt" + "time" "github.com/komari-monitor/komari/common" "github.com/komari-monitor/komari/database/dbcore" @@ -56,6 +57,7 @@ func SaveClientReport(clientUUID string, report common.Report) (err error) { Record := models.Record{ Client: clientUUID, + Time: time.Now(), CPU: float32(report.CPU.Usage), GPU: 0, // Report 未提供 GPU Usage,设为 0(与原 nil 行为类似) RAM: report.Ram.Used, diff --git a/database/records/records.go b/database/records/records.go index 3ca2f2d..681bb7a 100644 --- a/database/records/records.go +++ b/database/records/records.go @@ -1,6 +1,7 @@ package records import ( + "log" "time" "gorm.io/gorm" @@ -21,7 +22,7 @@ func DeleteAll() error { func GetLatestRecord(uuid string) (Record []models.Record, err error) { db := dbcore.GetDBInstance() - err = db.Where("ClientUUID = ?", uuid).Order("time DESC").Limit(1).Find(&Record).Error + err = db.Where("client = ?", uuid).Order("time DESC").Limit(1).Find(&Record).Error return } @@ -64,187 +65,164 @@ func QuantileMean(values []float32) float32 { return sum / float32(count) } -// 压缩数据库,针对每个ClientUUID,3小时内的数据不动,24小时内的数据精简为每分钟一条,3天为每15分钟一条,7天为每1小时一条,30天为每12小时一条。所有精简的数据是取 [0.02,0.98] 区间内的平均值 +// 压缩数据库,针对每个ClientUUID,10分钟内的数据不动,24小时内的数据精简为每15分钟一条,3天为每30分钟一条,7天为每1小时一条,30天为每12小时一条。所有精简的数据是取 [0.02,0.98] 区间内的平均值 func CompactRecord() error { db := dbcore.GetDBInstance() - + log.Printf("Compacting records...") var clientUUIDs []string - if err := db.Model(&models.Record{}).Distinct("ClientUUID").Pluck("ClientUUID", &clientUUIDs).Error; err != nil { + if err := db.Model(&models.Record{}).Distinct("client").Pluck("client", &clientUUIDs).Error; err != nil { return err } now := time.Now() - threeHoursAgo := now.Add(-3 * time.Hour) + tenMinutesAgo := now.Add(-10 * time.Minute) oneDayAgo := now.Add(-24 * time.Hour) threeDaysAgo := now.Add(-3 * 24 * time.Hour) sevenDaysAgo := now.Add(-7 * 24 * time.Hour) thirtyDaysAgo := now.Add(-30 * 24 * time.Hour) for _, clientUUID := range clientUUIDs { - // Process each time window - if err := db.Transaction(func(tx *gorm.DB) error { - // 24 hours to 3 hours: compact to 1-minute intervals - if err := compactTimeWindow(tx, clientUUID, oneDayAgo, threeHoursAgo, time.Minute, func(recordss []models.Record) models.Record { - return aggregateRecords(recordss, time.Minute) - }); err != nil { - return err - } - - // 3 days to 24 hours: compact to 15-minute intervals - if err := compactTimeWindow(tx, clientUUID, threeDaysAgo, oneDayAgo, 15*time.Minute, func(recordss []models.Record) models.Record { - return aggregateRecords(recordss, 15*time.Minute) - }); err != nil { - return err - } - - // 7 days to 3 days: compact to 1-hour intervals - if err := compactTimeWindow(tx, clientUUID, sevenDaysAgo, threeDaysAgo, time.Hour, func(recordss []models.Record) models.Record { - return aggregateRecords(recordss, time.Hour) - }); err != nil { - return err - } - - // 30 days to 7 days: compact to 12-hour intervals - if err := compactTimeWindow(tx, clientUUID, thirtyDaysAgo, sevenDaysAgo, 12*time.Hour, func(recordss []models.Record) models.Record { - return aggregateRecords(recordss, 12*time.Hour) - }); err != nil { - return err - } - - return nil - }); err != nil { - return err + // Process each time range with specific aggregation intervals + if err := processTimeRange(db, clientUUID, thirtyDaysAgo, sevenDaysAgo, 12*time.Hour, now); err != nil { + log.Printf("Error compacting 30d-7d for client %s: %v", clientUUID, err) + continue + } + if err := processTimeRange(db, clientUUID, sevenDaysAgo, threeDaysAgo, 1*time.Hour, now); err != nil { + log.Printf("Error compacting 7d-3d for client %s: %v", clientUUID, err) + continue + } + if err := processTimeRange(db, clientUUID, threeDaysAgo, oneDayAgo, 30*time.Minute, now); err != nil { + log.Printf("Error compacting 3d-1d for client %s: %v", clientUUID, err) + continue + } + if err := processTimeRange(db, clientUUID, oneDayAgo, tenMinutesAgo, 15*time.Minute, now); err != nil { + log.Printf("Error compacting 1d-10m for client %s: %v", clientUUID, err) + continue } } + if err := db.Exec("VACUUM").Error; err != nil { + log.Printf("Error vacuuming the database: %v", err) + } + log.Printf("Record compaction completed") return nil } -// compactTimeWindow compacts recordss within a specific time window -func compactTimeWindow(db *gorm.DB, clientUUID string, startTime, endTime time.Time, interval time.Duration, aggregator func([]models.Record) models.Record) error { - var recordss []models.Record - if err := db.Where("ClientUUID = ? AND Time >= ? AND Time < ?", clientUUID, startTime, endTime). - Order("Time ASC").Find(&recordss).Error; err != nil { +func processTimeRange(db *gorm.DB, clientUUID string, start, end time.Time, interval time.Duration, now time.Time) error { + // Round start time to the nearest interval + start = start.Truncate(interval) + end = end.Truncate(interval) + + // Get all records in the time range + var records []models.Record + if err := db.Where("client = ? AND time >= ? AND time < ?", clientUUID, start, end). + Order("time ASC"). + Find(&records).Error; err != nil { return err } - if len(recordss) == 0 { + if len(records) == 0 { return nil } - // Group recordss by interval - groups := make(map[int64][]models.Record) - for _, records := range recordss { - // Truncate time to the start of the interval - intervalStart := records.Time.Truncate(interval).Unix() - groups[intervalStart] = append(groups[intervalStart], records) + // Group records by interval + type aggregatedRecord struct { + StartTime time.Time + Records []models.Record + } + var aggregatedRecords []aggregatedRecord + currentGroup := aggregatedRecord{StartTime: start} + currentGroup.Records = make([]models.Record, 0) + + for _, record := range records { + groupTime := record.Time.Truncate(interval) + if !groupTime.Equal(currentGroup.StartTime) { + if len(currentGroup.Records) > 0 { + aggregatedRecords = append(aggregatedRecords, currentGroup) + } + currentGroup = aggregatedRecord{StartTime: groupTime} + currentGroup.Records = []models.Record{record} + } else { + currentGroup.Records = append(currentGroup.Records, record) + } + } + if len(currentGroup.Records) > 0 { + aggregatedRecords = append(aggregatedRecords, currentGroup) } - // Process each group - for intervalStart, group := range groups { - if len(group) <= 1 { - continue // Skip groups with single records + // Begin transaction + tx := db.Begin() + if tx.Error != nil { + return tx.Error + } + + // Delete original records in the time range + if err := tx.Where("client = ? AND time >= ? AND time < ?", clientUUID, start, end).Delete(&models.Record{}).Error; err != nil { + tx.Rollback() + return err + } + + // Insert aggregated records + for _, agg := range aggregatedRecords { + if len(agg.Records) == 0 { + continue } - // Aggregate recordss - aggregated := aggregator(group) + // Calculate averages and sums + var sumCPU, sumGPU, sumLOAD, sumTEMP float32 + var sumRAM, sumRAMTotal, sumSWAP, sumSWAPTotal, sumDISK, sumDISKTotal int64 + var sumNETIn, sumNETOut, sumNETTotalUp, sumNETTotalDown int64 + var sumPROCESS, sumConnections, sumConnectionsUDP int + count := len(agg.Records) - // Delete original recordss - if err := db.Where("ClientUUID = ? AND Time >= ? AND Time < ?", - clientUUID, time.Unix(intervalStart, 0), time.Unix(intervalStart, 0).Add(interval)). - Delete(&models.Record{}).Error; err != nil { - return err + for _, r := range agg.Records { + sumCPU += r.CPU + sumGPU += r.GPU + sumLOAD += r.LOAD + sumTEMP += r.TEMP + sumRAM += r.RAM + sumRAMTotal += r.RAMTotal + sumSWAP += r.SWAP + sumSWAPTotal += r.SWAPTotal + sumDISK += r.DISK + sumDISKTotal += r.DISKTotal + sumNETIn += r.NETIn + sumNETOut += r.NETOut + sumNETTotalUp += r.NETTotalUp + sumNETTotalDown += r.NETTotalDown + sumPROCESS += r.PROCESS + sumConnections += r.Connections + sumConnectionsUDP += r.ConnectionsUDP } - // Insert aggregated records - if err := db.Create(&aggregated).Error; err != nil { + // Create new aggregated record + newRecord := models.Record{ + Client: clientUUID, + Time: agg.StartTime, + CPU: sumCPU / float32(count), + GPU: sumGPU / float32(count), + RAM: sumRAM / int64(count), + RAMTotal: sumRAMTotal / int64(count), + SWAP: sumSWAP / int64(count), + SWAPTotal: sumSWAPTotal / int64(count), + LOAD: sumLOAD / float32(count), + TEMP: sumTEMP / float32(count), + DISK: sumDISK / int64(count), + DISKTotal: sumDISKTotal / int64(count), + NETIn: sumNETIn / int64(count), + NETOut: sumNETOut / int64(count), + NETTotalUp: sumNETTotalUp / int64(count), + NETTotalDown: sumNETTotalDown / int64(count), + PROCESS: sumPROCESS / count, + Connections: sumConnections / count, + ConnectionsUDP: sumConnectionsUDP / count, + } + + if err := tx.Create(&newRecord).Error; err != nil { + tx.Rollback() return err } } - return nil -} - -// aggregateRecords aggregates a group of recordss into a single records -func aggregateRecords(recordss []models.Record, interval time.Duration) models.Record { - if len(recordss) == 0 { - return models.Record{} - } - - // Initialize result with first records's metadata - result := models.Record{ - Client: recordss[0].Client, - Time: recordss[0].Time.Truncate(interval), - } - - // Collect values for quantile mean calculation - var cpuValues, gpuValues, loadValues, tempValues []float32 - var ram, ramTotal, swap, swapTotal, disk, diskTotal, netIn, netOut, netTotalUp, netTotalDown []int64 - var process, connections, connectionsUDP []int - - for _, r := range recordss { - cpuValues = append(cpuValues, r.CPU) - - gpuValues = append(gpuValues, r.GPU) - loadValues = append(loadValues, r.LOAD) - tempValues = append(tempValues, r.TEMP) - ram = append(ram, r.RAM) - ramTotal = append(ramTotal, r.RAMTotal) - swap = append(swap, r.SWAP) - swapTotal = append(swapTotal, r.SWAPTotal) - disk = append(disk, r.DISK) - diskTotal = append(diskTotal, r.DISKTotal) - netIn = append(netIn, r.NETIn) - netOut = append(netOut, r.NETOut) - netTotalUp = append(netTotalUp, r.NETTotalUp) - netTotalDown = append(netTotalDown, r.NETTotalDown) - process = append(process, r.PROCESS) - connections = append(connections, r.Connections) - connectionsUDP = append(connectionsUDP, r.ConnectionsUDP) - } - - // Apply quantile mean for float32 fields - result.CPU = QuantileMean(cpuValues) - result.GPU = QuantileMean(gpuValues) - result.LOAD = QuantileMean(loadValues) - result.TEMP = QuantileMean(tempValues) - - // Simple mean for integer fields - sumInt64 := func(values []int64) int64 { - if len(values) == 0 { - return 0 - } - sum := int64(0) - for _, v := range values { - sum += v - } - return sum / int64(len(values)) - } - - sumInt := func(values []int) int { - if len(values) == 0 { - return 0 - } - sum := 0 - for _, v := range values { - sum += v - } - return sum / len(values) - } - - result.RAM = sumInt64(ram) - result.RAMTotal = sumInt64(ramTotal) - result.SWAP = sumInt64(swap) - result.SWAPTotal = sumInt64(swapTotal) - result.DISK = sumInt64(disk) - result.DISKTotal = sumInt64(diskTotal) - result.NETIn = sumInt64(netIn) - result.NETOut = sumInt64(netOut) - result.NETTotalUp = sumInt64(netTotalUp) - result.NETTotalDown = sumInt64(netTotalDown) - result.PROCESS = sumInt(process) - result.Connections = sumInt(connections) - result.ConnectionsUDP = sumInt(connectionsUDP) - - return result + return tx.Commit().Error }