fix: #12 压缩数据库未按预期工

This commit is contained in:
Akizon77 committed 2025-04-30 18:14:46 +08:00
1 parent 4c65049bd8
commit 053592f26c
3 files changed
+131 -149

No files matched your search

+3 -1
View File
@@ -111,7 +111,9 @@ func InitDatabase() {
}
func DoRecordsWork() {
ticker := time.NewTicker(time.Hour * 1)
ticker := time.NewTicker(time.Minute * 30)
records.DeleteRecordBefore(time.Now().Add(-time.Hour * 24 * 30))
records.CompactRecord()
for range ticker.C {
records.DeleteRecordBefore(time.Now().Add(-time.Hour * 24 * 30))
records.CompactRecord()
+2
View File
@@ -3,6 +3,7 @@ package clients
import (
"encoding/json"
"fmt"
"time"
"github.com/komari-monitor/komari/common"
"github.com/komari-monitor/komari/database/dbcore"
@@ -56,6 +57,7 @@ func SaveClientReport(clientUUID string, report common.Report) (err error) {
Record := models.Record{
Client: clientUUID,
Time: time.Now(),
CPU: float32(report.CPU.Usage),
GPU: 0, // Report 未提供 GPU Usage,设为 0(与原 nil 行为类似)
RAM: report.Ram.Used,
+126 -148
View File
@@ -1,6 +1,7 @@
package records
import (
"log"
"time"
"gorm.io/gorm"
@@ -21,7 +22,7 @@ func DeleteAll() error {
func GetLatestRecord(uuid string) (Record []models.Record, err error) {
db := dbcore.GetDBInstance()
err = db.Where("ClientUUID = ?", uuid).Order("time DESC").Limit(1).Find(&Record).Error
err = db.Where("client = ?", uuid).Order("time DESC").Limit(1).Find(&Record).Error
return
}
@@ -64,187 +65,164 @@ func QuantileMean(values []float32) float32 {
return sum / float32(count)
}
// 压缩数据库,针对每个ClientUUID,3小时内的数据不动,24小时内的数据精简为每分钟一条,3天为每15分钟一条,7天为每1小时一条,30天为每12小时一条。所有精简的数据是取 [0.02,0.98] 区间内的平均值
// 压缩数据库,针对每个ClientUUID,10分钟内的数据不动,24小时内的数据精简为每15分钟一条,3天为每30分钟一条,7天为每1小时一条,30天为每12小时一条。所有精简的数据是取 [0.02,0.98] 区间内的平均值
func CompactRecord() error {
db := dbcore.GetDBInstance()
log.Printf("Compacting records...")
var clientUUIDs []string
if err := db.Model(&models.Record{}).Distinct("ClientUUID").Pluck("ClientUUID", &clientUUIDs).Error; err != nil {
if err := db.Model(&models.Record{}).Distinct("client").Pluck("client", &clientUUIDs).Error; err != nil {
return err
}
now := time.Now()
threeHoursAgo := now.Add(-3 * time.Hour)
tenMinutesAgo := now.Add(-10 * time.Minute)
oneDayAgo := now.Add(-24 * time.Hour)
threeDaysAgo := now.Add(-3 * 24 * time.Hour)
sevenDaysAgo := now.Add(-7 * 24 * time.Hour)
thirtyDaysAgo := now.Add(-30 * 24 * time.Hour)
for _, clientUUID := range clientUUIDs {
// Process each time window
if err := db.Transaction(func(tx *gorm.DB) error {
// 24 hours to 3 hours: compact to 1-minute intervals
if err := compactTimeWindow(tx, clientUUID, oneDayAgo, threeHoursAgo, time.Minute, func(recordss []models.Record) models.Record {
return aggregateRecords(recordss, time.Minute)
}); err != nil {
return err
}
// 3 days to 24 hours: compact to 15-minute intervals
if err := compactTimeWindow(tx, clientUUID, threeDaysAgo, oneDayAgo, 15*time.Minute, func(recordss []models.Record) models.Record {
return aggregateRecords(recordss, 15*time.Minute)
}); err != nil {
return err
}
// 7 days to 3 days: compact to 1-hour intervals
if err := compactTimeWindow(tx, clientUUID, sevenDaysAgo, threeDaysAgo, time.Hour, func(recordss []models.Record) models.Record {
return aggregateRecords(recordss, time.Hour)
}); err != nil {
return err
}
// 30 days to 7 days: compact to 12-hour intervals
if err := compactTimeWindow(tx, clientUUID, thirtyDaysAgo, sevenDaysAgo, 12*time.Hour, func(recordss []models.Record) models.Record {
return aggregateRecords(recordss, 12*time.Hour)
}); err != nil {
return err
}
return nil
}); err != nil {
return err
// Process each time range with specific aggregation intervals
if err := processTimeRange(db, clientUUID, thirtyDaysAgo, sevenDaysAgo, 12*time.Hour, now); err != nil {
log.Printf("Error compacting 30d-7d for client %s: %v", clientUUID, err)
continue
}
if err := processTimeRange(db, clientUUID, sevenDaysAgo, threeDaysAgo, 1*time.Hour, now); err != nil {
log.Printf("Error compacting 7d-3d for client %s: %v", clientUUID, err)
continue
}
if err := processTimeRange(db, clientUUID, threeDaysAgo, oneDayAgo, 30*time.Minute, now); err != nil {
log.Printf("Error compacting 3d-1d for client %s: %v", clientUUID, err)
continue
}
if err := processTimeRange(db, clientUUID, oneDayAgo, tenMinutesAgo, 15*time.Minute, now); err != nil {
log.Printf("Error compacting 1d-10m for client %s: %v", clientUUID, err)
continue
}
}
if err := db.Exec("VACUUM").Error; err != nil {
log.Printf("Error vacuuming the database: %v", err)
}
log.Printf("Record compaction completed")
return nil
}
// compactTimeWindow compacts recordss within a specific time window
func compactTimeWindow(db *gorm.DB, clientUUID string, startTime, endTime time.Time, interval time.Duration, aggregator func([]models.Record) models.Record) error {
var recordss []models.Record
if err := db.Where("ClientUUID = ? AND Time >= ? AND Time < ?", clientUUID, startTime, endTime).
Order("Time ASC").Find(&recordss).Error; err != nil {
func processTimeRange(db *gorm.DB, clientUUID string, start, end time.Time, interval time.Duration, now time.Time) error {
// Round start time to the nearest interval
start = start.Truncate(interval)
end = end.Truncate(interval)
// Get all records in the time range
var records []models.Record
if err := db.Where("client = ? AND time >= ? AND time < ?", clientUUID, start, end).
Order("time ASC").
Find(&records).Error; err != nil {
return err
}
if len(recordss) == 0 {
if len(records) == 0 {
return nil
}
// Group recordss by interval
groups := make(map[int64][]models.Record)
for _, records := range recordss {
// Truncate time to the start of the interval
intervalStart := records.Time.Truncate(interval).Unix()
groups[intervalStart] = append(groups[intervalStart], records)
// Group records by interval
type aggregatedRecord struct {
StartTime time.Time
Records []models.Record
}
var aggregatedRecords []aggregatedRecord
currentGroup := aggregatedRecord{StartTime: start}
currentGroup.Records = make([]models.Record, 0)
for _, record := range records {
groupTime := record.Time.Truncate(interval)
if !groupTime.Equal(currentGroup.StartTime) {
if len(currentGroup.Records) > 0 {
aggregatedRecords = append(aggregatedRecords, currentGroup)
}
currentGroup = aggregatedRecord{StartTime: groupTime}
currentGroup.Records = []models.Record{record}
} else {
currentGroup.Records = append(currentGroup.Records, record)
}
}
if len(currentGroup.Records) > 0 {
aggregatedRecords = append(aggregatedRecords, currentGroup)
}
// Process each group
for intervalStart, group := range groups {
if len(group) <= 1 {
continue // Skip groups with single records
// Begin transaction
tx := db.Begin()
if tx.Error != nil {
return tx.Error
}
// Delete original records in the time range
if err := tx.Where("client = ? AND time >= ? AND time < ?", clientUUID, start, end).Delete(&models.Record{}).Error; err != nil {
tx.Rollback()
return err
}
// Insert aggregated records
for _, agg := range aggregatedRecords {
if len(agg.Records) == 0 {
continue
}
// Aggregate recordss
aggregated := aggregator(group)
// Calculate averages and sums
var sumCPU, sumGPU, sumLOAD, sumTEMP float32
var sumRAM, sumRAMTotal, sumSWAP, sumSWAPTotal, sumDISK, sumDISKTotal int64
var sumNETIn, sumNETOut, sumNETTotalUp, sumNETTotalDown int64
var sumPROCESS, sumConnections, sumConnectionsUDP int
count := len(agg.Records)
// Delete original recordss
if err := db.Where("ClientUUID = ? AND Time >= ? AND Time < ?",
clientUUID, time.Unix(intervalStart, 0), time.Unix(intervalStart, 0).Add(interval)).
Delete(&models.Record{}).Error; err != nil {
return err
for _, r := range agg.Records {
sumCPU += r.CPU
sumGPU += r.GPU
sumLOAD += r.LOAD
sumTEMP += r.TEMP
sumRAM += r.RAM
sumRAMTotal += r.RAMTotal
sumSWAP += r.SWAP
sumSWAPTotal += r.SWAPTotal
sumDISK += r.DISK
sumDISKTotal += r.DISKTotal
sumNETIn += r.NETIn
sumNETOut += r.NETOut
sumNETTotalUp += r.NETTotalUp
sumNETTotalDown += r.NETTotalDown
sumPROCESS += r.PROCESS
sumConnections += r.Connections
sumConnectionsUDP += r.ConnectionsUDP
}
// Insert aggregated records
if err := db.Create(&aggregated).Error; err != nil {
// Create new aggregated record
newRecord := models.Record{
Client: clientUUID,
Time: agg.StartTime,
CPU: sumCPU / float32(count),
GPU: sumGPU / float32(count),
RAM: sumRAM / int64(count),
RAMTotal: sumRAMTotal / int64(count),
SWAP: sumSWAP / int64(count),
SWAPTotal: sumSWAPTotal / int64(count),
LOAD: sumLOAD / float32(count),
TEMP: sumTEMP / float32(count),
DISK: sumDISK / int64(count),
DISKTotal: sumDISKTotal / int64(count),
NETIn: sumNETIn / int64(count),
NETOut: sumNETOut / int64(count),
NETTotalUp: sumNETTotalUp / int64(count),
NETTotalDown: sumNETTotalDown / int64(count),
PROCESS: sumPROCESS / count,
Connections: sumConnections / count,
ConnectionsUDP: sumConnectionsUDP / count,
}
if err := tx.Create(&newRecord).Error; err != nil {
tx.Rollback()
return err
}
}
return nil
}
// aggregateRecords aggregates a group of recordss into a single records
func aggregateRecords(recordss []models.Record, interval time.Duration) models.Record {
if len(recordss) == 0 {
return models.Record{}
}
// Initialize result with first records's metadata
result := models.Record{
Client: recordss[0].Client,
Time: recordss[0].Time.Truncate(interval),
}
// Collect values for quantile mean calculation
var cpuValues, gpuValues, loadValues, tempValues []float32
var ram, ramTotal, swap, swapTotal, disk, diskTotal, netIn, netOut, netTotalUp, netTotalDown []int64
var process, connections, connectionsUDP []int
for _, r := range recordss {
cpuValues = append(cpuValues, r.CPU)
gpuValues = append(gpuValues, r.GPU)
loadValues = append(loadValues, r.LOAD)
tempValues = append(tempValues, r.TEMP)
ram = append(ram, r.RAM)
ramTotal = append(ramTotal, r.RAMTotal)
swap = append(swap, r.SWAP)
swapTotal = append(swapTotal, r.SWAPTotal)
disk = append(disk, r.DISK)
diskTotal = append(diskTotal, r.DISKTotal)
netIn = append(netIn, r.NETIn)
netOut = append(netOut, r.NETOut)
netTotalUp = append(netTotalUp, r.NETTotalUp)
netTotalDown = append(netTotalDown, r.NETTotalDown)
process = append(process, r.PROCESS)
connections = append(connections, r.Connections)
connectionsUDP = append(connectionsUDP, r.ConnectionsUDP)
}
// Apply quantile mean for float32 fields
result.CPU = QuantileMean(cpuValues)
result.GPU = QuantileMean(gpuValues)
result.LOAD = QuantileMean(loadValues)
result.TEMP = QuantileMean(tempValues)
// Simple mean for integer fields
sumInt64 := func(values []int64) int64 {
if len(values) == 0 {
return 0
}
sum := int64(0)
for _, v := range values {
sum += v
}
return sum / int64(len(values))
}
sumInt := func(values []int) int {
if len(values) == 0 {
return 0
}
sum := 0
for _, v := range values {
sum += v
}
return sum / len(values)
}
result.RAM = sumInt64(ram)
result.RAMTotal = sumInt64(ramTotal)
result.SWAP = sumInt64(swap)
result.SWAPTotal = sumInt64(swapTotal)
result.DISK = sumInt64(disk)
result.DISKTotal = sumInt64(diskTotal)
result.NETIn = sumInt64(netIn)
result.NETOut = sumInt64(netOut)
result.NETTotalUp = sumInt64(netTotalUp)
result.NETTotalDown = sumInt64(netTotalDown)
result.PROCESS = sumInt(process)
result.Connections = sumInt(connections)
result.ConnectionsUDP = sumInt(connectionsUDP)
return result
return tx.Commit().Error
}