mirror of
https://github.com/komari-monitor/komari.git
synced 2026-11-17 12:10:56 +00:00
fix: #12 压缩数据库未按预期工
This commit is contained in:
1 parent
4c65049bd8
commit
053592f26c
3 files changed
+131
-149
No files matched your search
+3
-1
@@ -111,7 +111,9 @@ func InitDatabase() {
|
||||
}
|
||||
|
||||
func DoRecordsWork() {
|
||||
ticker := time.NewTicker(time.Hour * 1)
|
||||
ticker := time.NewTicker(time.Minute * 30)
|
||||
records.DeleteRecordBefore(time.Now().Add(-time.Hour * 24 * 30))
|
||||
records.CompactRecord()
|
||||
for range ticker.C {
|
||||
records.DeleteRecordBefore(time.Now().Add(-time.Hour * 24 * 30))
|
||||
records.CompactRecord()
|
||||
|
||||
@@ -3,6 +3,7 @@ package clients
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"github.com/komari-monitor/komari/common"
|
||||
"github.com/komari-monitor/komari/database/dbcore"
|
||||
@@ -56,6 +57,7 @@ func SaveClientReport(clientUUID string, report common.Report) (err error) {
|
||||
|
||||
Record := models.Record{
|
||||
Client: clientUUID,
|
||||
Time: time.Now(),
|
||||
CPU: float32(report.CPU.Usage),
|
||||
GPU: 0, // Report 未提供 GPU Usage,设为 0(与原 nil 行为类似)
|
||||
RAM: report.Ram.Used,
|
||||
|
||||
+126
-148
@@ -1,6 +1,7 @@
|
||||
package records
|
||||
|
||||
import (
|
||||
"log"
|
||||
"time"
|
||||
|
||||
"gorm.io/gorm"
|
||||
@@ -21,7 +22,7 @@ func DeleteAll() error {
|
||||
|
||||
func GetLatestRecord(uuid string) (Record []models.Record, err error) {
|
||||
db := dbcore.GetDBInstance()
|
||||
err = db.Where("ClientUUID = ?", uuid).Order("time DESC").Limit(1).Find(&Record).Error
|
||||
err = db.Where("client = ?", uuid).Order("time DESC").Limit(1).Find(&Record).Error
|
||||
return
|
||||
}
|
||||
|
||||
@@ -64,187 +65,164 @@ func QuantileMean(values []float32) float32 {
|
||||
return sum / float32(count)
|
||||
}
|
||||
|
||||
// 压缩数据库,针对每个ClientUUID,3小时内的数据不动,24小时内的数据精简为每分钟一条,3天为每15分钟一条,7天为每1小时一条,30天为每12小时一条。所有精简的数据是取 [0.02,0.98] 区间内的平均值
|
||||
// 压缩数据库,针对每个ClientUUID,10分钟内的数据不动,24小时内的数据精简为每15分钟一条,3天为每30分钟一条,7天为每1小时一条,30天为每12小时一条。所有精简的数据是取 [0.02,0.98] 区间内的平均值
|
||||
func CompactRecord() error {
|
||||
db := dbcore.GetDBInstance()
|
||||
|
||||
log.Printf("Compacting records...")
|
||||
var clientUUIDs []string
|
||||
if err := db.Model(&models.Record{}).Distinct("ClientUUID").Pluck("ClientUUID", &clientUUIDs).Error; err != nil {
|
||||
if err := db.Model(&models.Record{}).Distinct("client").Pluck("client", &clientUUIDs).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
now := time.Now()
|
||||
threeHoursAgo := now.Add(-3 * time.Hour)
|
||||
tenMinutesAgo := now.Add(-10 * time.Minute)
|
||||
oneDayAgo := now.Add(-24 * time.Hour)
|
||||
threeDaysAgo := now.Add(-3 * 24 * time.Hour)
|
||||
sevenDaysAgo := now.Add(-7 * 24 * time.Hour)
|
||||
thirtyDaysAgo := now.Add(-30 * 24 * time.Hour)
|
||||
|
||||
for _, clientUUID := range clientUUIDs {
|
||||
// Process each time window
|
||||
if err := db.Transaction(func(tx *gorm.DB) error {
|
||||
// 24 hours to 3 hours: compact to 1-minute intervals
|
||||
if err := compactTimeWindow(tx, clientUUID, oneDayAgo, threeHoursAgo, time.Minute, func(recordss []models.Record) models.Record {
|
||||
return aggregateRecords(recordss, time.Minute)
|
||||
}); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// 3 days to 24 hours: compact to 15-minute intervals
|
||||
if err := compactTimeWindow(tx, clientUUID, threeDaysAgo, oneDayAgo, 15*time.Minute, func(recordss []models.Record) models.Record {
|
||||
return aggregateRecords(recordss, 15*time.Minute)
|
||||
}); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// 7 days to 3 days: compact to 1-hour intervals
|
||||
if err := compactTimeWindow(tx, clientUUID, sevenDaysAgo, threeDaysAgo, time.Hour, func(recordss []models.Record) models.Record {
|
||||
return aggregateRecords(recordss, time.Hour)
|
||||
}); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// 30 days to 7 days: compact to 12-hour intervals
|
||||
if err := compactTimeWindow(tx, clientUUID, thirtyDaysAgo, sevenDaysAgo, 12*time.Hour, func(recordss []models.Record) models.Record {
|
||||
return aggregateRecords(recordss, 12*time.Hour)
|
||||
}); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
}); err != nil {
|
||||
return err
|
||||
// Process each time range with specific aggregation intervals
|
||||
if err := processTimeRange(db, clientUUID, thirtyDaysAgo, sevenDaysAgo, 12*time.Hour, now); err != nil {
|
||||
log.Printf("Error compacting 30d-7d for client %s: %v", clientUUID, err)
|
||||
continue
|
||||
}
|
||||
if err := processTimeRange(db, clientUUID, sevenDaysAgo, threeDaysAgo, 1*time.Hour, now); err != nil {
|
||||
log.Printf("Error compacting 7d-3d for client %s: %v", clientUUID, err)
|
||||
continue
|
||||
}
|
||||
if err := processTimeRange(db, clientUUID, threeDaysAgo, oneDayAgo, 30*time.Minute, now); err != nil {
|
||||
log.Printf("Error compacting 3d-1d for client %s: %v", clientUUID, err)
|
||||
continue
|
||||
}
|
||||
if err := processTimeRange(db, clientUUID, oneDayAgo, tenMinutesAgo, 15*time.Minute, now); err != nil {
|
||||
log.Printf("Error compacting 1d-10m for client %s: %v", clientUUID, err)
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
if err := db.Exec("VACUUM").Error; err != nil {
|
||||
log.Printf("Error vacuuming the database: %v", err)
|
||||
}
|
||||
log.Printf("Record compaction completed")
|
||||
return nil
|
||||
}
|
||||
|
||||
// compactTimeWindow compacts recordss within a specific time window
|
||||
func compactTimeWindow(db *gorm.DB, clientUUID string, startTime, endTime time.Time, interval time.Duration, aggregator func([]models.Record) models.Record) error {
|
||||
var recordss []models.Record
|
||||
if err := db.Where("ClientUUID = ? AND Time >= ? AND Time < ?", clientUUID, startTime, endTime).
|
||||
Order("Time ASC").Find(&recordss).Error; err != nil {
|
||||
func processTimeRange(db *gorm.DB, clientUUID string, start, end time.Time, interval time.Duration, now time.Time) error {
|
||||
// Round start time to the nearest interval
|
||||
start = start.Truncate(interval)
|
||||
end = end.Truncate(interval)
|
||||
|
||||
// Get all records in the time range
|
||||
var records []models.Record
|
||||
if err := db.Where("client = ? AND time >= ? AND time < ?", clientUUID, start, end).
|
||||
Order("time ASC").
|
||||
Find(&records).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if len(recordss) == 0 {
|
||||
if len(records) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
// Group recordss by interval
|
||||
groups := make(map[int64][]models.Record)
|
||||
for _, records := range recordss {
|
||||
// Truncate time to the start of the interval
|
||||
intervalStart := records.Time.Truncate(interval).Unix()
|
||||
groups[intervalStart] = append(groups[intervalStart], records)
|
||||
// Group records by interval
|
||||
type aggregatedRecord struct {
|
||||
StartTime time.Time
|
||||
Records []models.Record
|
||||
}
|
||||
var aggregatedRecords []aggregatedRecord
|
||||
currentGroup := aggregatedRecord{StartTime: start}
|
||||
currentGroup.Records = make([]models.Record, 0)
|
||||
|
||||
for _, record := range records {
|
||||
groupTime := record.Time.Truncate(interval)
|
||||
if !groupTime.Equal(currentGroup.StartTime) {
|
||||
if len(currentGroup.Records) > 0 {
|
||||
aggregatedRecords = append(aggregatedRecords, currentGroup)
|
||||
}
|
||||
currentGroup = aggregatedRecord{StartTime: groupTime}
|
||||
currentGroup.Records = []models.Record{record}
|
||||
} else {
|
||||
currentGroup.Records = append(currentGroup.Records, record)
|
||||
}
|
||||
}
|
||||
if len(currentGroup.Records) > 0 {
|
||||
aggregatedRecords = append(aggregatedRecords, currentGroup)
|
||||
}
|
||||
|
||||
// Process each group
|
||||
for intervalStart, group := range groups {
|
||||
if len(group) <= 1 {
|
||||
continue // Skip groups with single records
|
||||
// Begin transaction
|
||||
tx := db.Begin()
|
||||
if tx.Error != nil {
|
||||
return tx.Error
|
||||
}
|
||||
|
||||
// Delete original records in the time range
|
||||
if err := tx.Where("client = ? AND time >= ? AND time < ?", clientUUID, start, end).Delete(&models.Record{}).Error; err != nil {
|
||||
tx.Rollback()
|
||||
return err
|
||||
}
|
||||
|
||||
// Insert aggregated records
|
||||
for _, agg := range aggregatedRecords {
|
||||
if len(agg.Records) == 0 {
|
||||
continue
|
||||
}
|
||||
|
||||
// Aggregate recordss
|
||||
aggregated := aggregator(group)
|
||||
// Calculate averages and sums
|
||||
var sumCPU, sumGPU, sumLOAD, sumTEMP float32
|
||||
var sumRAM, sumRAMTotal, sumSWAP, sumSWAPTotal, sumDISK, sumDISKTotal int64
|
||||
var sumNETIn, sumNETOut, sumNETTotalUp, sumNETTotalDown int64
|
||||
var sumPROCESS, sumConnections, sumConnectionsUDP int
|
||||
count := len(agg.Records)
|
||||
|
||||
// Delete original recordss
|
||||
if err := db.Where("ClientUUID = ? AND Time >= ? AND Time < ?",
|
||||
clientUUID, time.Unix(intervalStart, 0), time.Unix(intervalStart, 0).Add(interval)).
|
||||
Delete(&models.Record{}).Error; err != nil {
|
||||
return err
|
||||
for _, r := range agg.Records {
|
||||
sumCPU += r.CPU
|
||||
sumGPU += r.GPU
|
||||
sumLOAD += r.LOAD
|
||||
sumTEMP += r.TEMP
|
||||
sumRAM += r.RAM
|
||||
sumRAMTotal += r.RAMTotal
|
||||
sumSWAP += r.SWAP
|
||||
sumSWAPTotal += r.SWAPTotal
|
||||
sumDISK += r.DISK
|
||||
sumDISKTotal += r.DISKTotal
|
||||
sumNETIn += r.NETIn
|
||||
sumNETOut += r.NETOut
|
||||
sumNETTotalUp += r.NETTotalUp
|
||||
sumNETTotalDown += r.NETTotalDown
|
||||
sumPROCESS += r.PROCESS
|
||||
sumConnections += r.Connections
|
||||
sumConnectionsUDP += r.ConnectionsUDP
|
||||
}
|
||||
|
||||
// Insert aggregated records
|
||||
if err := db.Create(&aggregated).Error; err != nil {
|
||||
// Create new aggregated record
|
||||
newRecord := models.Record{
|
||||
Client: clientUUID,
|
||||
Time: agg.StartTime,
|
||||
CPU: sumCPU / float32(count),
|
||||
GPU: sumGPU / float32(count),
|
||||
RAM: sumRAM / int64(count),
|
||||
RAMTotal: sumRAMTotal / int64(count),
|
||||
SWAP: sumSWAP / int64(count),
|
||||
SWAPTotal: sumSWAPTotal / int64(count),
|
||||
LOAD: sumLOAD / float32(count),
|
||||
TEMP: sumTEMP / float32(count),
|
||||
DISK: sumDISK / int64(count),
|
||||
DISKTotal: sumDISKTotal / int64(count),
|
||||
NETIn: sumNETIn / int64(count),
|
||||
NETOut: sumNETOut / int64(count),
|
||||
NETTotalUp: sumNETTotalUp / int64(count),
|
||||
NETTotalDown: sumNETTotalDown / int64(count),
|
||||
PROCESS: sumPROCESS / count,
|
||||
Connections: sumConnections / count,
|
||||
ConnectionsUDP: sumConnectionsUDP / count,
|
||||
}
|
||||
|
||||
if err := tx.Create(&newRecord).Error; err != nil {
|
||||
tx.Rollback()
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// aggregateRecords aggregates a group of recordss into a single records
|
||||
func aggregateRecords(recordss []models.Record, interval time.Duration) models.Record {
|
||||
if len(recordss) == 0 {
|
||||
return models.Record{}
|
||||
}
|
||||
|
||||
// Initialize result with first records's metadata
|
||||
result := models.Record{
|
||||
Client: recordss[0].Client,
|
||||
Time: recordss[0].Time.Truncate(interval),
|
||||
}
|
||||
|
||||
// Collect values for quantile mean calculation
|
||||
var cpuValues, gpuValues, loadValues, tempValues []float32
|
||||
var ram, ramTotal, swap, swapTotal, disk, diskTotal, netIn, netOut, netTotalUp, netTotalDown []int64
|
||||
var process, connections, connectionsUDP []int
|
||||
|
||||
for _, r := range recordss {
|
||||
cpuValues = append(cpuValues, r.CPU)
|
||||
|
||||
gpuValues = append(gpuValues, r.GPU)
|
||||
loadValues = append(loadValues, r.LOAD)
|
||||
tempValues = append(tempValues, r.TEMP)
|
||||
ram = append(ram, r.RAM)
|
||||
ramTotal = append(ramTotal, r.RAMTotal)
|
||||
swap = append(swap, r.SWAP)
|
||||
swapTotal = append(swapTotal, r.SWAPTotal)
|
||||
disk = append(disk, r.DISK)
|
||||
diskTotal = append(diskTotal, r.DISKTotal)
|
||||
netIn = append(netIn, r.NETIn)
|
||||
netOut = append(netOut, r.NETOut)
|
||||
netTotalUp = append(netTotalUp, r.NETTotalUp)
|
||||
netTotalDown = append(netTotalDown, r.NETTotalDown)
|
||||
process = append(process, r.PROCESS)
|
||||
connections = append(connections, r.Connections)
|
||||
connectionsUDP = append(connectionsUDP, r.ConnectionsUDP)
|
||||
}
|
||||
|
||||
// Apply quantile mean for float32 fields
|
||||
result.CPU = QuantileMean(cpuValues)
|
||||
result.GPU = QuantileMean(gpuValues)
|
||||
result.LOAD = QuantileMean(loadValues)
|
||||
result.TEMP = QuantileMean(tempValues)
|
||||
|
||||
// Simple mean for integer fields
|
||||
sumInt64 := func(values []int64) int64 {
|
||||
if len(values) == 0 {
|
||||
return 0
|
||||
}
|
||||
sum := int64(0)
|
||||
for _, v := range values {
|
||||
sum += v
|
||||
}
|
||||
return sum / int64(len(values))
|
||||
}
|
||||
|
||||
sumInt := func(values []int) int {
|
||||
if len(values) == 0 {
|
||||
return 0
|
||||
}
|
||||
sum := 0
|
||||
for _, v := range values {
|
||||
sum += v
|
||||
}
|
||||
return sum / len(values)
|
||||
}
|
||||
|
||||
result.RAM = sumInt64(ram)
|
||||
result.RAMTotal = sumInt64(ramTotal)
|
||||
result.SWAP = sumInt64(swap)
|
||||
result.SWAPTotal = sumInt64(swapTotal)
|
||||
result.DISK = sumInt64(disk)
|
||||
result.DISKTotal = sumInt64(diskTotal)
|
||||
result.NETIn = sumInt64(netIn)
|
||||
result.NETOut = sumInt64(netOut)
|
||||
result.NETTotalUp = sumInt64(netTotalUp)
|
||||
result.NETTotalDown = sumInt64(netTotalDown)
|
||||
result.PROCESS = sumInt(process)
|
||||
result.Connections = sumInt(connections)
|
||||
result.ConnectionsUDP = sumInt(connectionsUDP)
|
||||
|
||||
return result
|
||||
return tx.Commit().Error
|
||||
}
|
||||
Reference in new issue
Block a user