mirror of
https://github.com/komari-monitor/komari.git
synced 2026-11-12 09:24:35 +00:00
Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
bf6b45ec3a | ||
|
|
4a3400bee6 | ||
|
|
4caa067556 | ||
|
|
2f3959adde | ||
|
|
63a7ffddcf | ||
|
|
92d679ffdb | ||
|
|
7c30fc4dc9 | ||
|
|
773a3715dc | ||
|
|
75d7b6c7a3 | ||
|
|
c294c17642 | ||
|
|
2883bf6295 | ||
|
|
14bf8bde7c | ||
|
|
f043f2890c | ||
|
|
057108a2f3 | ||
|
|
5879d4b3da | ||
|
|
16769d5687 | ||
|
|
5ea32c48b6 | ||
|
|
a06d922d78 | ||
|
|
0ed14fc73d | ||
|
|
61bd0c8bdd |
No files matched your search
+1
-4
@@ -8,10 +8,7 @@ ARG TARGETARCH
|
||||
|
||||
RUN apk add --no-cache ca-certificates curl tzdata
|
||||
|
||||
COPY komari-${TARGETOS}-${TARGETARCH} /app/komari
|
||||
|
||||
|
||||
RUN chmod +x /app/komari
|
||||
COPY --chmod=755 komari-${TARGETOS}-${TARGETARCH} /app/komari
|
||||
|
||||
ENV GIN_MODE=release
|
||||
ENV KOMARI_LISTEN=0.0.0.0:25774
|
||||
|
||||
@@ -5,43 +5,62 @@
|
||||
|
||||

|
||||
|
||||
[简体中文](./README_zh-cn.md)
|
||||
[English](./README.md) | [简体中文](./README_zh-cn.md)
|
||||
|
||||
Komari is a lightweight, self-hosted server monitoring tool designed to provide a simple and efficient solution for monitoring server performance. It supports viewing server status through a web interface and collects data through a lightweight agent.
|
||||
Komari is a lightweight, self-hosted server monitoring solution. It provides a simple and efficient way to track server performance through a web interface, with metrics collected by a lightweight agent.
|
||||
|
||||
> [!WARNING]
|
||||
> Komari is a self-hosted monitoring and control program intended only for systems you own or are authorized to administer. Do not weaponize Komari, deploy it without consent, or use it for unauthorized access, persistence, command execution, or other abusive activity. For context on real-world abuse risks, see Huntress' analysis: [Komari C2 agent abuse](https://www.huntress.com/blog/komari-c2-agent-abuse).
|
||||
> Users are solely responsible for how they deploy and operate Komari. The developers do not accept responsibility for unauthorized or abusive use, or for any resulting consequences.
|
||||
> On Windows, when remote control is enabled, the client displays a Windows notification at each user login to remind the user that Komari is remote control software.
|
||||
> Komari is a self-hosted monitoring and control application. Deploy it only on systems you own or are authorized to manage. You are solely responsible for how you deploy and use Komari. The developers accept no liability for unauthorized access, persistence, command execution, other misuse, or any resulting consequences.
|
||||
|
||||
[文档/Documentation](https://komari-document.pages.dev/) | [文档(镜像站 By Geekertao)](https://www.komari.wiki) | [Telegram Group](https://t.me/komari_monitor)
|
||||
[Documentation](https://www.komari.wiki/) | [Telegram Group](https://t.me/komari_monitor)
|
||||
|
||||
## Features
|
||||
|
||||
- **Lightweight and Efficient**: Low resource consumption, suitable for servers of all sizes.
|
||||
- **Self-hosted**: Complete control over data privacy, easy to deploy.
|
||||
- **Web Interface**: Intuitive monitoring dashboard, easy to use.
|
||||
- **Real-time monitoring**: Displays monitoring data at one-second intervals.
|
||||
- **Lightweight and efficient**: Uses minimal system resources and works well on servers of any size.
|
||||
- **Self-hosted**: Keeps you in control of your data and privacy.
|
||||
- **Web interface**: Provides an intuitive, easy-to-use monitoring dashboard.
|
||||
- **Extensible**: Supports custom themes and plugins.
|
||||
|
||||
## One-click Deployment
|
||||
## Quick Start
|
||||
|
||||
- Rainyun - CNY 4.5/month
|
||||
| Platform | Description |
|
||||
| --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------ |
|
||||
| <a href="https://app.rainyun.com/apps/rca/store/6780/NzYxNzAz_"><img src="https://rainyun-apps.cn-nb1.rains3.com/materials/deploy-on-rainyun-cn.svg" alt="Rainyun" width="180"></a> | Deploy websites, databases, and hundreds of popular apps in seconds with flexible hourly billing. |
|
||||
| <a href="https://apps.fit2cloud.com/1panel/komari"><img src="https://raw.githubusercontent.com/komari-monitor/public/refs/heads/main/images/1panel-logo-blue.png" alt="1Panel App Store" width="180"></a> | A modern, open-source Linux server management panel for websites, databases, containers, files, backups, security, and AI, with one-click deployment from its app store. |
|
||||
|
||||
[](https://app.rainyun.com/apps/rca/store/6780/NzYxNzAz_)
|
||||
For instructions on Docker deployment, binary installation, building from source, and updates, see the [installation guide](https://www.komari.wiki/en/install/quick-start).
|
||||
|
||||
- 1Panel App Store
|
||||
## Screenshots
|
||||
|
||||
Available on 1Panel App Store. Install via **App Store > Utilities > Komari**.
|
||||
| Page | Screenshot |
|
||||
| ------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
|
||||
| Home Dashboard | <img src="https://b2.akz.moe/awesome-pictures/komari-screenshot/%E4%B8%BB%E9%A1%B5%E4%BB%AA%E8%A1%A8%E7%9B%98-en.webp" width="800" alt="Home Dashboard"> |
|
||||
| Admin Dashboard | <img src="https://b2.akz.moe/awesome-pictures/komari-screenshot/%E5%90%8E%E5%8F%B0%E4%BB%AA%E8%A1%A8%E7%9B%98-en.webp" width="800" alt="Admin Dashboard"> |
|
||||
| History Charts | <img src="https://b2.akz.moe/awesome-pictures/komari-screenshot/%E5%8E%86%E5%8F%B2%E5%9B%BE%E8%A1%A8-en.webp" width="800" alt="History Charts"> |
|
||||
| Web Terminal | <img src="https://b2.akz.moe/awesome-pictures/komari-screenshot/%E7%BD%91%E9%A1%B5%E7%BB%88%E7%AB%AF.webp" width="800" alt="Web Terminal"> |
|
||||
| Customizable Themes | <img src="https://b2.akz.moe/awesome-pictures/komari-screenshot/%E4%B8%BB%E9%A2%98%E5%8F%AF%E8%87%AA%E5%AE%9A%E4%B9%89-en.webp" width="800" alt="Customizable Themes"> |
|
||||
| Theme Market | <img src="https://b2.akz.moe/awesome-pictures/komari-screenshot/%E4%B8%BB%E9%A2%98%E5%B8%82%E5%9C%BA-en.webp" width="800" alt="Theme Market"> |
|
||||
|
||||
For Docker, binary, source-build, and update instructions, see the [installation guide](https://komari-document.pages.dev/en/install/quick-start).
|
||||
## Sponsors
|
||||
|
||||
## Contributing
|
||||
Interested in sponsoring Komari? Contact the developer via [email](mailto:komari@akz.moe) or [Telegram](https://t.me/mamomoe).
|
||||
|
||||
Issues and Pull Requests are welcome!
|
||||
| Sponsor | Description |
|
||||
| ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
|
||||
| <a href="https://whmcs.as211392.com/aff.php?aff=110"><img src="https://raw.githubusercontent.com/komari-monitor/public/refs/heads/main/images/dreamcloud.png" alt="Dream Cloud" width="180"></a> | Cost-effective Asia-Pacific hosting with direct connectivity and robust DDoS protection, backed by transparent capacity claims. |
|
||||
| <a href="https://sharon.io"><img src="https://raw.githubusercontent.com/komari-monitor/public/refs/heads/main/images/sharon-networks.webp" alt="Sharon Networks" width="180"></a> | Premium China-optimized connectivity from Asia-Pacific data centers, featuring low latency, high bandwidth, and Tbps-scale local DDoS mitigation. Join the [Telegram community](https://t.me/SharonNetwork) to participate in charitable initiatives and community giveaways. |
|
||||
|
||||
## Acknowledgements
|
||||
## Contributors
|
||||
|
||||
| Acknowledgement | Details |
|
||||
| ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
|
||||
| <a href="https://whmcs.as211392.com/aff.php?aff=110"><img src="https://raw.githubusercontent.com/komari-monitor/public/refs/heads/main/images/dreamcloud.png" alt="Dream Cloud" width="180"></a> | 极高性价比解锁直连亚太高防,真高防,不虚标,打死退款 |
|
||||
| <a href="https://sharon.io"><img src="https://raw.githubusercontent.com/komari-monitor/public/refs/heads/main/images/sharon-networks.webp" alt="Sharon Networks" width="180"></a> | 亚太数据中心提供顶级的中国优化网络接入 · 低延时&高带宽&提供Tbps级本地清洗高防服务, 为您的业务保驾护航, 为您的客户提供极致体验. 加入社区 [Telegram群组](https://t.me/SharonNetwork) 可参与公益募捐或群内抽奖免费使用 |
|
||||
| Open-source community | Everyone who has submitted pull requests or created themes. |
|
||||
Thanks to everyone who has contributed code, themes, plugins, documentation, translations, bug reports, or feedback to Komari.
|
||||
|
||||
<a href="https://github.com/komari-monitor/komari/graphs/contributors"><img src="https://contributors-img.web.app/image?repo=komari-monitor/komari" alt="Komari contributors" width="600"></a>
|
||||
|
||||
## Support the Project
|
||||
|
||||
If Komari has been useful to you, consider buying me a coffee. Thank you for your support!
|
||||
|
||||
| WeChat Pay | TRON Network |
|
||||
| ------------------------------------------------------------------------------------------------------------ | ------------------------------------------------------------------------------------------- |
|
||||
|  |  |
|
||||
+35
-18
@@ -5,45 +5,62 @@
|
||||
|
||||

|
||||
|
||||
[简体中文](./README_zh-cn.md)
|
||||
[English](./README.md) | [简体中文](./README_zh-cn.md)
|
||||
|
||||
Komari 是一款轻量级的自托管服务器监控工具,旨在提供简单、高效的服务器性能监控解决方案。它支持通过 Web 界面查看服务器状态,并通过轻量级 Agent 收集数据。
|
||||
|
||||
> [!WARNING]
|
||||
> Komari 是一款自托管的监控/控制程序,仅应部署在你拥有或已获得授权管理的系统上。请勿将 Komari 武器化,或在未获授权的情况下部署、访问、持久化、执行命令及从事其他滥用行为。关于现实中的滥用风险,可参考 Huntress 的分析:[Komari C2 agent abuse](https://www.huntress.com/blog/komari-c2-agent-abuse)。
|
||||
> 用户需要自行承担部署和使用 Komari 的责任。开发者不对未经授权或滥用行为及其后果承担责任。
|
||||
> 在 Windows 端开启远程控制后,客户端会在每次用户登录时通过 Windows 通知提醒用户 Komari 是一款远程控制软件。
|
||||
> Komari 是一款自托管的监控/控制程序,仅应部署在你拥有或已获得授权管理的系统上。在未获授权的情况下部署、访问、持久化、执行命令及从事其他滥用行为,用户需要自行承担部署和使用 Komari 的责任。开发者不对未经授权或滥用行为及其后果承担责任。
|
||||
|
||||
[文档/Documentation](https://www.komari.wiki/) | [Telegram 群](https://t.me/komari_monitor)
|
||||
[文档](https://www.komari.wiki/) | [Telegram 群](https://t.me/komari_monitor)
|
||||
|
||||
## 特性
|
||||
|
||||
- **实时监控**: 秒级实时数据展示。
|
||||
- **轻量高效**:低资源占用,适合各种规模的服务器。
|
||||
- **自托管**:完全掌控数据隐私,部署简单。
|
||||
- **Web 界面**:直观的监控仪表盘,易于使用。
|
||||
- **极强的可扩展性**: 支持自定义主题和插件。
|
||||
|
||||
## 快速开始
|
||||
|
||||
### 0. 容器云一键部署
|
||||
|
||||
- 雨云云应用 - CNY 4.5/月
|
||||
|
||||
[](https://app.rainyun.com/apps/rca/store/6780/NzYxNzAz_)
|
||||
|
||||
- 1Panel 应用商店
|
||||
|
||||
已上架1Panel应用商店,应用商店-实用工具-Komari 即可安装
|
||||
| 平台 | 介绍 |
|
||||
| -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------- |
|
||||
| <a href="https://app.rainyun.com/apps/rca/store/6780/NzYxNzAz_"><img src="https://rainyun-apps.cn-nb1.rains3.com/materials/deploy-on-rainyun-cn.svg" alt="Rainyun" width="180"></a> | 秒级部署网站、数据库及数百款热门 App,并采用按小时灵活计费。 |
|
||||
| <a href="https://apps.fit2cloud.com/1panel/komari"><img src="https://raw.githubusercontent.com/komari-monitor/public/refs/heads/main/images/1panel-logo-blue.png" alt="1Panel Appstore" width="180"></a> | 现代化、开源的Linux 服务器运维管理面板,提供网站、数据库、容器、文件、备份、安全与AI 管理能力,支持应用商店一键部署。 |
|
||||
|
||||
Docker、二进制文件、源码构建和更新说明,请参阅 [安装指南](https://www.komari.wiki/install/quick-start).
|
||||
|
||||
## 贡献
|
||||
## 截图
|
||||
|
||||
欢迎提交 Issue 或 Pull Request!
|
||||
| 页面 | 截图 |
|
||||
| ------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------ |
|
||||
| 主页仪表盘 | <img src="https://b2.akz.moe/awesome-pictures/komari-screenshot/%E4%B8%BB%E9%A1%B5%E4%BB%AA%E8%A1%A8%E7%9B%98.webp" width="800" alt="主页仪表盘"> |
|
||||
| 后台仪表盘 | <img src="https://b2.akz.moe/awesome-pictures/komari-screenshot/%E5%90%8E%E5%8F%B0%E4%BB%AA%E8%A1%A8%E7%9B%98.webp" width="800" alt="后台仪表盘"> |
|
||||
| 历史图表 | <img src="https://b2.akz.moe/awesome-pictures/komari-screenshot/%E5%8E%86%E5%8F%B2%E5%9B%BE%E8%A1%A8.webp" width="800" alt="历史图表"> |
|
||||
| 网页终端 | <img src="https://b2.akz.moe/awesome-pictures/komari-screenshot/%E7%BD%91%E9%A1%B5%E7%BB%88%E7%AB%AF.webp" width="800" alt="网页终端"> |
|
||||
| 主题可自定义 | <img src="https://b2.akz.moe/awesome-pictures/komari-screenshot/%E4%B8%BB%E9%A2%98%E5%8F%AF%E8%87%AA%E5%AE%9A%E4%B9%89.webp" width="800" alt="主题可自定义"> |
|
||||
| 主题市场 | <img src="https://b2.akz.moe/awesome-pictures/komari-screenshot/%E4%B8%BB%E9%A2%98%E5%B8%82%E5%9C%BA.webp" width="800" alt="主题市场"> |
|
||||
|
||||
## 鸣谢
|
||||
## 赞助商
|
||||
|
||||
有意赞助 Komari?请通过 [电子邮箱](mailto:komari@akz.moe) 或 [Telegram](https://t.me/mamomoe) 联系开发者。
|
||||
|
||||
| 赞助商 | 描述 |
|
||||
| ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
|
||||
| <a href="https://whmcs.as211392.com/aff.php?aff=110"><img src="https://raw.githubusercontent.com/komari-monitor/public/refs/heads/main/images/dreamcloud.png" alt="Dream Cloud" width="180"></a> | 极高性价比解锁直连亚太高防,真高防,不虚标,打死退款 |
|
||||
| <a href="https://sharon.io"><img src="https://raw.githubusercontent.com/komari-monitor/public/refs/heads/main/images/sharon-networks.webp" alt="Sharon Networks" width="180"></a> | 亚太数据中心提供顶级的中国优化网络接入 · 低延时&高带宽&提供Tbps级本地清洗高防服务, 为您的业务保驾护航, 为您的客户提供极致体验. 加入社区 [Telegram群组](https://t.me/SharonNetwork) 可参与公益募捐或群内抽奖免费使用 |
|
||||
| 开源社区 | 提交 PR、制作主题的各位开发者 |
|
||||
|
||||
## 贡献者
|
||||
|
||||
感谢所有为 Komari 贡献代码、主题、插件、文档、翻译、问题报告或反馈的朋友。
|
||||
|
||||
<a href="https://github.com/komari-monitor/komari/graphs/contributors"><img src="https://contributors-img.web.app/image?repo=komari-monitor/komari" alt="Komari 贡献者" width="600"></a>
|
||||
|
||||
## 支持项目
|
||||
|
||||
如果 Komari 对你有所帮助,欢迎请作者喝一杯奶茶。感谢你的支持!
|
||||
|
||||
| 微信赞赏码 | TRON Network |
|
||||
| ------------------------------------------------------------------------------------------------ | --------------------------------------------------------------------------- |
|
||||
|  |  |
|
||||
@@ -15,16 +15,37 @@ const (
|
||||
|
||||
// Theme represents a komari theme information
|
||||
type Theme struct {
|
||||
Name string `json:"name"` // 主题名称
|
||||
Name any `json:"name"` // 主题名称,支持字符串或多语言对象
|
||||
Short string `json:"short"` // 短名称,用作文件夹名
|
||||
Description string `json:"description"` // 主题描述
|
||||
Description any `json:"description"` // 主题描述,支持字符串或多语言对象
|
||||
Version string `json:"version"` // 版本号
|
||||
Author string `json:"author"` // 作者
|
||||
Author any `json:"author"` // 作者,支持字符串或多语言对象
|
||||
URL string `json:"url"` // 主题URL
|
||||
Preview string `json:"preview"` // 预览图片相对路径
|
||||
Configuration Configuration `json:"configuration"` // 声明配置项
|
||||
}
|
||||
|
||||
// IsLocalizedText reports whether a manifest text field has a usable value.
|
||||
func IsLocalizedText(value any) bool {
|
||||
switch text := value.(type) {
|
||||
case string:
|
||||
return strings.TrimSpace(text) != ""
|
||||
case map[string]any:
|
||||
for _, item := range text {
|
||||
if itemText, ok := item.(string); ok && strings.TrimSpace(itemText) != "" {
|
||||
return true
|
||||
}
|
||||
}
|
||||
case map[string]string:
|
||||
for _, item := range text {
|
||||
if strings.TrimSpace(item) != "" {
|
||||
return true
|
||||
}
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
type Configuration struct {
|
||||
Type string `json:"type"` // managed raw redirect
|
||||
Icon string `json:"icon"` // 图标
|
||||
@@ -36,7 +57,7 @@ type ManagedThemeConfigurationItem struct {
|
||||
Key string `json:"key"`
|
||||
Name any `json:"name"`
|
||||
Required bool `json:"required"`
|
||||
Type string `json:"type"` // string number select switch title
|
||||
Type string `json:"type"` // string number select switch title textbox richtext nodes pingtasks
|
||||
Options string `json:"options"`
|
||||
Default any `json:"default"`
|
||||
Help any `json:"help"`
|
||||
|
||||
+31
-52
@@ -3,16 +3,17 @@ package database
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
logger "github.com/komari-monitor/komari/utils/log"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
|
||||
"github.com/gin-gonic/gin"
|
||||
"github.com/komari-monitor/komari/database/dbcore"
|
||||
"github.com/komari-monitor/komari/internal/metricstore"
|
||||
"github.com/komari-monitor/komari/database/models"
|
||||
"github.com/komari-monitor/komari/internal/config"
|
||||
"github.com/komari-monitor/komari/internal/managedconfig"
|
||||
"github.com/komari-monitor/komari/internal/metricstore"
|
||||
logger "github.com/komari-monitor/komari/utils/log"
|
||||
"github.com/komari-monitor/komari/web/public"
|
||||
)
|
||||
|
||||
func GetPublicInfo() (map[string]interface{}, error) {
|
||||
@@ -69,58 +70,20 @@ func GetPublicInfo() (map[string]interface{}, error) {
|
||||
if err != nil {
|
||||
logger.Infof("database", "%v", err)
|
||||
}
|
||||
// Try to load theme declaration file and merge defaults for managed configuration
|
||||
// Theme declarations live in ./data/theme/<short>/komari-theme.json
|
||||
if cst.Theme != "" && cst.Theme != "default" {
|
||||
themeConfigPath := filepath.Join("./data/theme", cst.Theme, "komari-theme.json")
|
||||
if _, err := os.Stat(themeConfigPath); err == nil {
|
||||
b, err := os.ReadFile(themeConfigPath)
|
||||
if err == nil {
|
||||
var themeDecl struct {
|
||||
Configuration struct {
|
||||
Type string `json:"type"`
|
||||
Data []models.ManagedThemeConfigurationItem `json:"data"`
|
||||
} `json:"configuration"`
|
||||
}
|
||||
if err := json.Unmarshal(b, &themeDecl); err == nil {
|
||||
if themeDecl.Configuration.Type == "managed" {
|
||||
for _, item := range themeDecl.Configuration.Data {
|
||||
if item.Key == "" {
|
||||
continue
|
||||
}
|
||||
// missing
|
||||
if _, exists := tc_data[item.Key]; !exists {
|
||||
var def any = item.Default
|
||||
// select
|
||||
if item.Type == "select" {
|
||||
if def == nil || def == "" {
|
||||
if item.Options != "" {
|
||||
opts := strings.Split(item.Options, ",")
|
||||
if len(opts) > 0 {
|
||||
def = strings.TrimSpace(opts[0])
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
// number->0, string->"", switch->false
|
||||
if def == nil {
|
||||
switch item.Type {
|
||||
case "number":
|
||||
def = 0
|
||||
case "switch":
|
||||
def = false
|
||||
default:
|
||||
def = ""
|
||||
}
|
||||
}
|
||||
tc_data[item.Key] = def
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
items := themeConfigurationItems(cst.Theme)
|
||||
if cst.Theme != "default" {
|
||||
for _, item := range items {
|
||||
if item.Key == "" {
|
||||
continue
|
||||
}
|
||||
if _, exists := tc_data[item.Key]; !exists {
|
||||
tc_data[item.Key] = managedconfig.DefaultValue(item)
|
||||
}
|
||||
}
|
||||
}
|
||||
if err := managedconfig.ResolveForOutput(tc_data, items); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return gin.H{
|
||||
"sitename": cst.Sitename,
|
||||
@@ -140,3 +103,19 @@ func GetPublicInfo() (map[string]interface{}, error) {
|
||||
"theme_settings": tc_data,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func themeConfigurationItems(short string) []models.ManagedThemeConfigurationItem {
|
||||
var manifest models.Theme
|
||||
if short == "default" {
|
||||
data, err := public.PublicFS.ReadFile("defaultTheme/komari-theme.json")
|
||||
if err != nil || json.Unmarshal(data, &manifest) != nil {
|
||||
return nil
|
||||
}
|
||||
} else {
|
||||
data, err := os.ReadFile(filepath.Join("./data/theme", short, "komari-theme.json"))
|
||||
if err != nil || json.Unmarshal(data, &manifest) != nil {
|
||||
return nil
|
||||
}
|
||||
}
|
||||
return managedconfig.Items(manifest.Configuration)
|
||||
}
|
||||
+38
-5
@@ -29,6 +29,8 @@ INSTALL_DIR="/opt/komari"
|
||||
DATA_DIR="/opt/komari"
|
||||
SERVICE_NAME="komari"
|
||||
BINARY_PATH="$INSTALL_DIR/komari"
|
||||
BACKUP_DIR="$INSTALL_DIR/backup"
|
||||
DATA_BACKUP_DIR="$DATA_DIR/data/backup"
|
||||
DEFAULT_PORT="25774"
|
||||
LISTEN_PORT=""
|
||||
REPO="komari-monitor/komari"
|
||||
@@ -419,6 +421,26 @@ show_access_info() {
|
||||
ui_msgbox "安装完成" "$content"
|
||||
}
|
||||
|
||||
# Remove historical upgrade backups
|
||||
cleanup_backups() {
|
||||
if ! ui_yesno "确认清理备份" "将删除 Komari 的二进制升级备份和数据升级压缩包。\n\n您确定要继续吗?"; then
|
||||
log_info "备份清理已取消"
|
||||
return 0
|
||||
fi
|
||||
|
||||
log_step "清理升级历史备份..."
|
||||
rm -f -- "${BINARY_PATH}.backup."*
|
||||
|
||||
local backup_dir
|
||||
for backup_dir in "$BACKUP_DIR" "$DATA_BACKUP_DIR"; do
|
||||
if [ -d "$backup_dir" ]; then
|
||||
find "$backup_dir" -maxdepth 1 -type f -name '*.zip' -delete
|
||||
fi
|
||||
done
|
||||
|
||||
ui_msgbox "清理完成" "升级历史备份已清理。\n\n清理范围:\n 二进制备份: ${BINARY_PATH}.backup.*\n 数据压缩包: $BACKUP_DIR\n 数据压缩包: $DATA_BACKUP_DIR"
|
||||
}
|
||||
|
||||
# Upgrade function
|
||||
upgrade_komari() {
|
||||
log_step "升级 Komari..."
|
||||
@@ -439,14 +461,23 @@ upgrade_komari() {
|
||||
log_step "停止 Komari 服务..."
|
||||
systemctl stop ${SERVICE_NAME}.service
|
||||
|
||||
log_step "清理旧的二进制备份..."
|
||||
rm -f -- "${BINARY_PATH}.backup."*
|
||||
|
||||
local backup_path="${BINARY_PATH}.backup.$(date +%Y%m%d_%H%M%S)"
|
||||
log_step "备份当前二进制文件..."
|
||||
cp "$BINARY_PATH" "${BINARY_PATH}.backup.$(date +%Y%m%d_%H%M%S)"
|
||||
if ! cp "$BINARY_PATH" "$backup_path"; then
|
||||
log_error "备份当前二进制文件失败,正在启动服务"
|
||||
systemctl start ${SERVICE_NAME}.service
|
||||
ui_msgbox "错误" "备份当前二进制文件失败,升级已取消。"
|
||||
return 1
|
||||
fi
|
||||
|
||||
local arch=$(detect_arch)
|
||||
local download_url=$(get_download_url "$arch")
|
||||
if [ $? -ne 0 ]; then
|
||||
log_error "获取下载链接失败,正在从备份恢复"
|
||||
mv "${BINARY_PATH}.backup."* "$BINARY_PATH"
|
||||
mv "$backup_path" "$BINARY_PATH"
|
||||
systemctl start ${SERVICE_NAME}.service
|
||||
ui_msgbox "错误" "获取下载链接失败,已从备份恢复。"
|
||||
return 1
|
||||
@@ -455,7 +486,7 @@ upgrade_komari() {
|
||||
log_step "下载最新版本..."
|
||||
if ! curl -L -o "$BINARY_PATH" "$download_url"; then
|
||||
log_error "下载失败,正在从备份恢复"
|
||||
mv "${BINARY_PATH}.backup."* "$BINARY_PATH"
|
||||
mv "$backup_path" "$BINARY_PATH"
|
||||
systemctl start ${SERVICE_NAME}.service
|
||||
ui_msgbox "错误" "下载失败,已从备份恢复。"
|
||||
return 1
|
||||
@@ -593,7 +624,8 @@ main_menu() {
|
||||
"5" "查看日志" \
|
||||
"6" "重启服务" \
|
||||
"7" "停止服务" \
|
||||
"8" "退出")
|
||||
"8" "清理升级历史备份" \
|
||||
"9" "退出")
|
||||
|
||||
# 用户在 TUI 中取消(ESC/Cancel)则退出
|
||||
if [ $? -ne 0 ] && tui_enabled; then
|
||||
@@ -609,7 +641,8 @@ main_menu() {
|
||||
5) show_logs ;;
|
||||
6) restart_service ;;
|
||||
7) stop_service ;;
|
||||
8)
|
||||
8) cleanup_backups ;;
|
||||
9)
|
||||
tui_enabled && clear
|
||||
exit 0
|
||||
;;
|
||||
|
||||
@@ -0,0 +1,144 @@
|
||||
// Package managedconfig implements the shared managed-configuration behavior
|
||||
// used by themes and plugins.
|
||||
package managedconfig
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"strings"
|
||||
|
||||
"github.com/komari-monitor/komari/database/dbcore"
|
||||
"github.com/komari-monitor/komari/database/models"
|
||||
)
|
||||
|
||||
const (
|
||||
TypeNodes = "nodes"
|
||||
TypePingTasks = "pingtasks"
|
||||
)
|
||||
|
||||
// Items returns the declared managed configuration items. An omitted type is
|
||||
// managed, matching the theme configuration default.
|
||||
func Items(configuration models.Configuration) []models.ManagedThemeConfigurationItem {
|
||||
if configuration.Type != "" && !strings.EqualFold(configuration.Type, models.ThemeConfigurationManaged) {
|
||||
return nil
|
||||
}
|
||||
raw, err := json.Marshal(configuration.Data)
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
var items []models.ManagedThemeConfigurationItem
|
||||
if err := json.Unmarshal(raw, &items); err != nil {
|
||||
return nil
|
||||
}
|
||||
return items
|
||||
}
|
||||
|
||||
func DefaultValue(item models.ManagedThemeConfigurationItem) any {
|
||||
value := item.Default
|
||||
if item.Type == "select" && (value == nil || value == "") && item.Options != "" {
|
||||
options := strings.Split(item.Options, ",")
|
||||
if len(options) > 0 {
|
||||
return strings.TrimSpace(options[0])
|
||||
}
|
||||
}
|
||||
if value != nil {
|
||||
return value
|
||||
}
|
||||
switch item.Type {
|
||||
case "number":
|
||||
return float64(0)
|
||||
case "switch":
|
||||
return false
|
||||
case TypeNodes, TypePingTasks:
|
||||
return "[]"
|
||||
default:
|
||||
return ""
|
||||
}
|
||||
}
|
||||
|
||||
// ResolveForOutput decodes declared selectors, drops deleted references, and
|
||||
// returns typed arrays suitable for public theme settings and plugin config.
|
||||
func ResolveForOutput(values map[string]any, items []models.ManagedThemeConfigurationItem) error {
|
||||
hasNodes := false
|
||||
hasPingTasks := false
|
||||
for _, item := range items {
|
||||
switch item.Type {
|
||||
case TypeNodes:
|
||||
hasNodes = true
|
||||
case TypePingTasks:
|
||||
hasPingTasks = true
|
||||
}
|
||||
}
|
||||
|
||||
db := dbcore.GetDBInstance()
|
||||
liveNodes := map[string]struct{}{}
|
||||
if hasNodes {
|
||||
var nodes []models.Client
|
||||
if err := db.Select("uuid").Find(&nodes).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
for _, node := range nodes {
|
||||
liveNodes[node.UUID] = struct{}{}
|
||||
}
|
||||
}
|
||||
livePingTasks := map[uint]struct{}{}
|
||||
if hasPingTasks {
|
||||
var tasks []models.PingTask
|
||||
if err := db.Select("id").Find(&tasks).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
for _, task := range tasks {
|
||||
livePingTasks[task.Id] = struct{}{}
|
||||
}
|
||||
}
|
||||
|
||||
for _, item := range items {
|
||||
if item.Key == "" {
|
||||
continue
|
||||
}
|
||||
switch item.Type {
|
||||
case TypeNodes:
|
||||
selected := NodeIDs(values[item.Key])
|
||||
filtered := make([]string, 0, len(selected))
|
||||
for _, id := range selected {
|
||||
if _, ok := liveNodes[id]; ok {
|
||||
filtered = append(filtered, id)
|
||||
}
|
||||
}
|
||||
values[item.Key] = filtered
|
||||
case TypePingTasks:
|
||||
selected := PingTaskIDs(values[item.Key])
|
||||
filtered := make([]uint, 0, len(selected))
|
||||
for _, id := range selected {
|
||||
if _, ok := livePingTasks[id]; ok {
|
||||
filtered = append(filtered, id)
|
||||
}
|
||||
}
|
||||
values[item.Key] = filtered
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func NodeIDs(value any) []string {
|
||||
raw, ok := value.(string)
|
||||
if !ok {
|
||||
return nil
|
||||
}
|
||||
var ids []string
|
||||
if json.Unmarshal([]byte(raw), &ids) != nil {
|
||||
return nil
|
||||
}
|
||||
return ids
|
||||
}
|
||||
|
||||
func PingTaskIDs(value any) []uint {
|
||||
raw, ok := value.(string)
|
||||
if !ok {
|
||||
return nil
|
||||
}
|
||||
var ids []uint
|
||||
if json.Unmarshal([]byte(raw), &ids) != nil {
|
||||
return nil
|
||||
}
|
||||
return ids
|
||||
}
|
||||
@@ -92,9 +92,6 @@ func createMetricDefinitionsWithDefaultRetention(ctx context.Context, s *metric.
|
||||
}
|
||||
if err == nil {
|
||||
if existing.RetentionDays == 0 {
|
||||
if _, err := s.SetMetricRetention(ctx, def.Name, 0); err != nil {
|
||||
return fmt.Errorf("failed to preserve disabled metric %s: %w", def.Name, err)
|
||||
}
|
||||
continue
|
||||
}
|
||||
def.RetentionDays = existing.RetentionDays
|
||||
@@ -103,11 +100,5 @@ func createMetricDefinitionsWithDefaultRetention(ctx context.Context, s *metric.
|
||||
return fmt.Errorf("failed to create metric %s: %w", def.Name, err)
|
||||
}
|
||||
}
|
||||
for _, name := range obsoleteBuiltinMetricNames {
|
||||
if err := s.DeleteMetric(ctx, name); err != nil {
|
||||
return fmt.Errorf("failed to remove obsolete metric %s: %w", name, err)
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
@@ -84,7 +84,31 @@ func ReclaimSpace(ctx context.Context) (MaintenanceResult, error) {
|
||||
}
|
||||
maintenanceCtx := context.Background()
|
||||
result.Before, result.BeforeSizeError = activeStore.StorageSize(maintenanceCtx)
|
||||
maintenanceErr := activeStore.ReclaimSpace(maintenanceCtx)
|
||||
maintenanceErr := deleteUndefinedMetrics(maintenanceCtx, activeStore)
|
||||
if maintenanceErr == nil {
|
||||
maintenanceErr = activeStore.ReclaimSpace(maintenanceCtx)
|
||||
}
|
||||
result.After, result.AfterSizeError = activeStore.StorageSize(maintenanceCtx)
|
||||
return result, maintenanceErr
|
||||
}
|
||||
|
||||
func deleteUndefinedMetrics(ctx context.Context, s *metric.Store) error {
|
||||
definitions, err := s.ListMetrics(ctx)
|
||||
if err != nil {
|
||||
return fmt.Errorf("list metric definitions before reclaim: %w", err)
|
||||
}
|
||||
|
||||
builtin := make(map[string]struct{}, len(builtinMetricNames))
|
||||
for _, name := range builtinMetricNames {
|
||||
builtin[name] = struct{}{}
|
||||
}
|
||||
for _, definition := range definitions {
|
||||
if _, ok := builtin[definition.Name]; ok {
|
||||
continue
|
||||
}
|
||||
if err := s.DeleteMetric(ctx, definition.Name); err != nil {
|
||||
return fmt.Errorf("delete undefined metric %q before reclaim: %w", definition.Name, err)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -32,13 +32,6 @@ var loadRecordMetricNames = []string{
|
||||
MetricProcess, MetricConnections, MetricConnectionsUDP,
|
||||
}
|
||||
|
||||
var obsoleteBuiltinMetricNames = []string{
|
||||
"memory.total",
|
||||
"swap.total",
|
||||
"temperature",
|
||||
"disk.total",
|
||||
}
|
||||
|
||||
// gpuDeviceRecordMetricNames are stored separately from the entity-level GPU
|
||||
// average and are included when deleting all system records.
|
||||
var gpuDeviceRecordMetricNames = []string{
|
||||
@@ -50,6 +43,8 @@ var recordMetricNames = joinMetricNames(loadRecordMetricNames, gpuDeviceRecordMe
|
||||
// Ping has an independent retention and cleanup boundary.
|
||||
var pingMetricNames = []string{MetricPingLatency, MetricPingLoss}
|
||||
|
||||
var builtinMetricNames = joinMetricNames(recordMetricNames, pingMetricNames)
|
||||
|
||||
func metricNameForRecordField(name string) (string, bool) {
|
||||
switch name {
|
||||
case "cpu":
|
||||
|
||||
@@ -3,7 +3,6 @@ package metricstore
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"errors"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
"time"
|
||||
@@ -283,7 +282,7 @@ func TestCreateMetricDefinitionsUsesExplicitRetentionAndPreservesOverrides(t *te
|
||||
}
|
||||
}
|
||||
|
||||
func TestCreateMetricDefinitionsRemovesObsoleteMetrics(t *testing.T) {
|
||||
func TestCreateMetricDefinitionsKeepsExistingMetrics(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
s, err := metric.Open(ctx, metric.SQLite(":memory:", metric.WithMaxOpenConns(1)))
|
||||
if err != nil {
|
||||
@@ -299,15 +298,19 @@ func TestCreateMetricDefinitionsRemovesObsoleteMetrics(t *testing.T) {
|
||||
if err := createMetricDefinitions(ctx, s); err != nil {
|
||||
t.Fatalf("refresh built-in definitions: %v", err)
|
||||
}
|
||||
if _, err := s.GetMetric(ctx, "memory.total"); !errors.Is(err, metric.ErrNotFound) {
|
||||
t.Fatalf("obsolete definition error = %v, want ErrNotFound", err)
|
||||
definition, err := s.GetMetric(ctx, "memory.total")
|
||||
if err != nil {
|
||||
t.Fatalf("existing definition was removed: %v", err)
|
||||
}
|
||||
if definition.RetentionDays != 1 {
|
||||
t.Fatalf("existing retention = %d, want 1", definition.RetentionDays)
|
||||
}
|
||||
points, err := s.Query(ctx, metric.Query{MetricName: "memory.total", EntityID: "node-a", Start: time.Now().UTC().Add(-time.Hour), End: time.Now().UTC().Add(time.Hour)})
|
||||
if err != nil {
|
||||
t.Fatalf("query obsolete points: %v", err)
|
||||
t.Fatalf("query existing points: %v", err)
|
||||
}
|
||||
if len(points) != 0 {
|
||||
t.Fatalf("obsolete points remain: %#v", points)
|
||||
if len(points) != 1 || points[0].Value != 1024 {
|
||||
t.Fatalf("existing points were removed: %#v", points)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,63 @@
|
||||
package metricstore
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"fmt"
|
||||
|
||||
"github.com/komari-monitor/komari/pkg/metric"
|
||||
)
|
||||
|
||||
// QueryContext runs a raw read query against the active metric store. The
|
||||
// caller must invoke the returned release function after closing rows.
|
||||
func QueryContext(ctx context.Context, query string, args ...any) (*sql.Rows, metric.Driver, func(), error) {
|
||||
return QueryForDriver(ctx, func(metric.Driver) (string, error) {
|
||||
return query, nil
|
||||
}, args...)
|
||||
}
|
||||
|
||||
// QueryForDriver runs a raw read query whose SQL depends on the active store's
|
||||
// driver. The caller must invoke the returned release function after closing rows.
|
||||
func QueryForDriver(ctx context.Context, query func(metric.Driver) (string, error), args ...any) (*sql.Rows, metric.Driver, func(), error) {
|
||||
if err := storeOperations.AcquireShared(ctx); err != nil {
|
||||
return nil, "", nil, fmt.Errorf("wait for metric store operations before query: %w", err)
|
||||
}
|
||||
|
||||
storeMu.RLock()
|
||||
activeStore := store
|
||||
storeMu.RUnlock()
|
||||
if activeStore == nil {
|
||||
storeOperations.ReleaseShared()
|
||||
return nil, "", nil, ErrStoreNotInitialized
|
||||
}
|
||||
|
||||
statement, err := query(activeStore.Driver())
|
||||
if err != nil {
|
||||
storeOperations.ReleaseShared()
|
||||
return nil, "", nil, err
|
||||
}
|
||||
rows, err := activeStore.QueryContext(ctx, statement, args...)
|
||||
if err != nil {
|
||||
storeOperations.ReleaseShared()
|
||||
return nil, "", nil, err
|
||||
}
|
||||
return rows, activeStore.Driver(), storeOperations.ReleaseShared, nil
|
||||
}
|
||||
|
||||
// ExecContext runs a raw statement against the active metric store.
|
||||
func ExecContext(ctx context.Context, query string, args ...any) (sql.Result, metric.Driver, error) {
|
||||
if err := storeOperations.Acquire(ctx); err != nil {
|
||||
return nil, "", fmt.Errorf("wait for metric store operations before execution: %w", err)
|
||||
}
|
||||
defer storeOperations.Release()
|
||||
|
||||
storeMu.RLock()
|
||||
activeStore := store
|
||||
storeMu.RUnlock()
|
||||
if activeStore == nil {
|
||||
return nil, "", ErrStoreNotInitialized
|
||||
}
|
||||
|
||||
result, err := activeStore.ExecContext(ctx, query, args...)
|
||||
return result, activeStore.Driver(), err
|
||||
}
|
||||
@@ -27,6 +27,8 @@ type reportTrafficValues struct {
|
||||
totalUp int64
|
||||
hasDown bool
|
||||
totalDown int64
|
||||
hasUptime bool
|
||||
uptime int64
|
||||
}
|
||||
|
||||
var reportTrafficStates sync.Map
|
||||
@@ -384,12 +386,15 @@ func writeReportBatch(ctx context.Context, reports []v1.Report) ([]v1.Report, er
|
||||
if !values.timestamp.IsZero() && !report.UpdatedAt.After(values.timestamp) {
|
||||
report.UpdatedAt = values.timestamp.Add(time.Millisecond)
|
||||
}
|
||||
agentRestart := values.hasUptime && report.Uptime < values.uptime
|
||||
counterResetUp := values.hasUp && report.Network.TotalUp < values.totalUp
|
||||
counterResetDown := values.hasDown && report.Network.TotalDown < values.totalDown
|
||||
trafficUp := int64(0)
|
||||
if values.hasUp {
|
||||
if values.hasUp && !agentRestart && !counterResetUp {
|
||||
trafficUp = TrafficCounterDelta(report.Network.TotalUp, values.totalUp)
|
||||
}
|
||||
trafficDown := int64(0)
|
||||
if values.hasDown {
|
||||
if values.hasDown && !agentRestart && !counterResetDown {
|
||||
trafficDown = TrafficCounterDelta(report.Network.TotalDown, values.totalDown)
|
||||
}
|
||||
points = append(points, reportMetricPoints(report, trafficUp, trafficDown)...)
|
||||
@@ -398,6 +403,8 @@ func writeReportBatch(ctx context.Context, reports []v1.Report) ([]v1.Report, er
|
||||
values.totalUp = report.Network.TotalUp
|
||||
values.hasDown = true
|
||||
values.totalDown = report.Network.TotalDown
|
||||
values.hasUptime = true
|
||||
values.uptime = report.Uptime
|
||||
pendingStates[state] = values
|
||||
prepared[i] = report
|
||||
}
|
||||
|
||||
@@ -87,11 +87,11 @@ func TestWriteReportStoresMinuteMetricsAndResetAwareTraffic(t *testing.T) {
|
||||
t.Fatalf("write reset report: %v", err)
|
||||
}
|
||||
|
||||
assertMetricValues(t, s, MetricTrafficUp, report.UUID, base.Add(-time.Second), base.Add(time.Minute), []float64{0, 50, 20})
|
||||
assertMetricValues(t, s, MetricTrafficDown, report.UUID, base.Add(-time.Second), base.Add(time.Minute), []float64{0, 60, 30})
|
||||
assertMetricValues(t, s, MetricTrafficUp, report.UUID, base.Add(-time.Second), base.Add(time.Minute), []float64{0, 50, 0})
|
||||
assertMetricValues(t, s, MetricTrafficDown, report.UUID, base.Add(-time.Second), base.Add(time.Minute), []float64{0, 60, 0})
|
||||
assertMetricValues(t, s, MetricNetTotalUp, report.UUID, base.Add(-time.Second), base.Add(time.Minute), []float64{100, 150, 20})
|
||||
assertMetricAggregate(t, s, MetricTrafficUp, report.UUID, base.Add(-time.Second), base.Add(time.Minute), metric.AggSum, 70, 3)
|
||||
assertMetricAggregate(t, s, MetricTrafficDown, report.UUID, base.Add(-time.Second), base.Add(time.Minute), metric.AggSum, 90, 3)
|
||||
assertMetricAggregate(t, s, MetricTrafficUp, report.UUID, base.Add(-time.Second), base.Add(time.Minute), metric.AggSum, 50, 3)
|
||||
assertMetricAggregate(t, s, MetricTrafficDown, report.UUID, base.Add(-time.Second), base.Add(time.Minute), metric.AggSum, 60, 3)
|
||||
|
||||
gpuPoints, err := s.Query(ctx, metric.Query{
|
||||
MetricName: MetricGPUDeviceUsage,
|
||||
@@ -370,6 +370,48 @@ func TestTrafficCounterDelta(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestWriteReportRebasesTrafficAfterAgentRestart(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
s := useReportTestStore(t, nil)
|
||||
base := time.Now().UTC().Truncate(time.Minute).Add(5 * time.Second)
|
||||
report := v1.Report{
|
||||
UUID: "restarted-node",
|
||||
UpdatedAt: base,
|
||||
Uptime: 1000,
|
||||
Network: v1.NetworkReport{TotalUp: 100, TotalDown: 200},
|
||||
}
|
||||
if _, err := WriteReport(ctx, report); err != nil {
|
||||
t.Fatalf("write first report: %v", err)
|
||||
}
|
||||
|
||||
report.UpdatedAt = base.Add(3 * time.Second)
|
||||
report.Uptime = 1003
|
||||
report.Network.TotalUp = 150
|
||||
report.Network.TotalDown = 260
|
||||
if _, err := WriteReport(ctx, report); err != nil {
|
||||
t.Fatalf("write continuous report: %v", err)
|
||||
}
|
||||
|
||||
report.UpdatedAt = base.Add(6 * time.Second)
|
||||
report.Uptime = 1
|
||||
report.Network.TotalUp = 155
|
||||
report.Network.TotalDown = 265
|
||||
if _, err := WriteReport(ctx, report); err != nil {
|
||||
t.Fatalf("write report after agent restart: %v", err)
|
||||
}
|
||||
|
||||
report.UpdatedAt = base.Add(9 * time.Second)
|
||||
report.Uptime = 4
|
||||
report.Network.TotalUp = 180
|
||||
report.Network.TotalDown = 300
|
||||
if _, err := WriteReport(ctx, report); err != nil {
|
||||
t.Fatalf("write report after new baseline: %v", err)
|
||||
}
|
||||
|
||||
assertMetricValues(t, s, MetricTrafficUp, report.UUID, base.Add(-time.Second), base.Add(time.Minute), []float64{0, 50, 0, 25})
|
||||
assertMetricValues(t, s, MetricTrafficDown, report.UUID, base.Add(-time.Second), base.Add(time.Minute), []float64{0, 60, 0, 35})
|
||||
}
|
||||
|
||||
func TestWriteReportNormalizesReceiveTimeToUTC(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
s := useReportTestStore(t, nil)
|
||||
|
||||
@@ -5,10 +5,10 @@ import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
|
||||
"github.com/komari-monitor/komari/database/dbcore"
|
||||
"github.com/komari-monitor/komari/database/models"
|
||||
"github.com/komari-monitor/komari/internal/managedconfig"
|
||||
"gorm.io/gorm"
|
||||
)
|
||||
|
||||
@@ -26,7 +26,11 @@ func GetConfiguration(short string) (map[string]any, error) {
|
||||
if err != nil {
|
||||
return values, nil // not installed (or unreadable): keep saved values
|
||||
}
|
||||
mergeConfigurationDefaults(values, info.Configuration)
|
||||
items := managedconfig.Items(info.Configuration)
|
||||
mergeConfigurationDefaults(values, items)
|
||||
if err := managedconfig.ResolveForOutput(values, items); err != nil {
|
||||
return nil, fmt.Errorf("resolve plugin configuration: %w", err)
|
||||
}
|
||||
return values, nil
|
||||
}
|
||||
|
||||
@@ -48,18 +52,7 @@ func savedConfiguration(short string) (map[string]any, error) {
|
||||
|
||||
// mergeConfigurationDefaults fills manifest-declared defaults for keys that
|
||||
// are missing from the saved values, following the theme merge rules.
|
||||
func mergeConfigurationDefaults(values map[string]any, configuration models.Configuration) {
|
||||
if configuration.Type != models.ThemeConfigurationManaged {
|
||||
return
|
||||
}
|
||||
raw, err := json.Marshal(configuration.Data)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
var items []models.ManagedThemeConfigurationItem
|
||||
if err := json.Unmarshal(raw, &items); err != nil {
|
||||
return
|
||||
}
|
||||
func mergeConfigurationDefaults(values map[string]any, items []models.ManagedThemeConfigurationItem) {
|
||||
for _, item := range items {
|
||||
if item.Key == "" {
|
||||
continue
|
||||
@@ -67,38 +60,10 @@ func mergeConfigurationDefaults(values map[string]any, configuration models.Conf
|
||||
if _, exists := values[item.Key]; exists {
|
||||
continue
|
||||
}
|
||||
values[item.Key] = configurationDefault(item)
|
||||
values[item.Key] = managedconfig.DefaultValue(item)
|
||||
}
|
||||
}
|
||||
|
||||
// configurationDefault resolves one item's effective default, mirroring the
|
||||
// theme rules: select falls back to its first option; nil defaults become
|
||||
// 0 for number, false for switch and "" otherwise.
|
||||
func configurationDefault(item models.ManagedThemeConfigurationItem) any {
|
||||
def := item.Default
|
||||
if item.Type == "select" {
|
||||
if def == nil || def == "" {
|
||||
if item.Options != "" {
|
||||
opts := strings.Split(item.Options, ",")
|
||||
if len(opts) > 0 {
|
||||
return strings.TrimSpace(opts[0])
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if def == nil {
|
||||
switch item.Type {
|
||||
case "number":
|
||||
return float64(0) // JSON numbers decode as float64
|
||||
case "switch":
|
||||
return false
|
||||
default:
|
||||
return ""
|
||||
}
|
||||
}
|
||||
return def
|
||||
}
|
||||
|
||||
// SaveConfiguration upserts the configuration values of one plugin.
|
||||
func SaveConfiguration(short string, data map[string]any) error {
|
||||
if data == nil {
|
||||
|
||||
@@ -14,6 +14,8 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/gin-gonic/gin"
|
||||
"github.com/komari-monitor/komari/database/dbcore"
|
||||
"github.com/komari-monitor/komari/database/models"
|
||||
"github.com/komari-monitor/komari/pkg/rpc"
|
||||
)
|
||||
|
||||
@@ -398,6 +400,58 @@ func TestGetConfigurationMergesManifestDefaults(t *testing.T) {
|
||||
t.Errorf("mode still default = %v", values["mode"])
|
||||
}
|
||||
}
|
||||
|
||||
func TestGetConfigurationResolvesSelectorValues(t *testing.T) {
|
||||
withTempDataDir(t)
|
||||
if err := dbcore.GetDBInstance().Create(&models.Client{UUID: "node-a"}).Error; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := dbcore.GetDBInstance().Create(&models.PingTask{Id: 8}).Error; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
zipPath := writePluginZip(t, map[string]string{
|
||||
"komari-plugin.json": `{"name":"Cfg","short":"cfg","version":"1.0.0","configuration":{"type":"managed","data":[
|
||||
{"key":"nodes","name":"Nodes","type":"nodes"},
|
||||
{"key":"tasks","name":"Tasks","type":"pingtasks"}
|
||||
]}}`,
|
||||
"script.js": `function load() {}`,
|
||||
})
|
||||
if _, err := InstallZip(zipPath); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := SaveConfiguration("cfg", map[string]any{
|
||||
"nodes": `["node-a","deleted-node"]`,
|
||||
"tasks": `[8,7]`,
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
values, err := GetConfiguration("cfg")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
nodes, ok := values["nodes"].([]string)
|
||||
if !ok || len(nodes) != 1 || nodes[0] != "node-a" {
|
||||
t.Fatalf("nodes = %#v", values["nodes"])
|
||||
}
|
||||
tasks, ok := values["tasks"].([]uint)
|
||||
if !ok || len(tasks) != 1 || tasks[0] != 8 {
|
||||
t.Fatalf("tasks = %#v", values["tasks"])
|
||||
}
|
||||
|
||||
var stored models.PluginConfiguration
|
||||
if err := dbcore.GetDBInstance().Where("short = ?", "cfg").First(&stored).Error; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var raw map[string]any
|
||||
if err := json.Unmarshal([]byte(stored.Data), &raw); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if raw["nodes"] != `["node-a","deleted-node"]` || raw["tasks"] != `[8,7]` {
|
||||
t.Fatalf("stored = %#v", raw)
|
||||
}
|
||||
}
|
||||
|
||||
func TestGetConfigFromScript(t *testing.T) {
|
||||
withTempDataDir(t)
|
||||
gin.SetMode(gin.TestMode)
|
||||
|
||||
@@ -41,7 +41,7 @@ func validateManifest(info *models.Plugin) error {
|
||||
if !validShort(info.Short) {
|
||||
return fmt.Errorf("plugin short %q is invalid: only letters, digits, '_' and '-' are allowed", info.Short)
|
||||
}
|
||||
if !localizedText(info.Name) {
|
||||
if !models.IsLocalizedText(info.Name) {
|
||||
return fmt.Errorf("plugin name is required")
|
||||
}
|
||||
if info.Entry == "" {
|
||||
@@ -90,7 +90,7 @@ func validatePage(page *models.PluginPage) error {
|
||||
default:
|
||||
return fmt.Errorf("plugin page type %q is invalid: use \"iframe\" or \"redirect\"", page.Type)
|
||||
}
|
||||
if !localizedText(page.Title) {
|
||||
if !models.IsLocalizedText(page.Title) {
|
||||
return fmt.Errorf("plugin page %q requires a title", page.Title)
|
||||
}
|
||||
return nil
|
||||
@@ -113,23 +113,3 @@ func isSafeInternalPath(target string) bool {
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// localizedText reports whether v is a usable name/description/author value:
|
||||
// a non-empty string, or an i18n object with at least one non-empty value.
|
||||
func localizedText(v any) bool {
|
||||
switch t := v.(type) {
|
||||
case nil:
|
||||
return false
|
||||
case string:
|
||||
return strings.TrimSpace(t) != ""
|
||||
case map[string]any:
|
||||
for _, value := range t {
|
||||
if s, ok := value.(string); ok && strings.TrimSpace(s) != "" {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
@@ -153,6 +153,13 @@ func SetEnabled(short string, enabled, approved bool) error {
|
||||
return global.setEnabled(short, enabled, approved)
|
||||
}
|
||||
|
||||
// Reload unloads and loads an enabled plugin. The load hook completes before
|
||||
// this function returns; a failed load is persisted as the plugin's last
|
||||
// error and disables the plugin, matching the normal enable/startup path.
|
||||
func Reload(short string) error {
|
||||
return global.reload(short)
|
||||
}
|
||||
|
||||
// CloseAll unloads every loaded plugin. Used during server shutdown.
|
||||
func CloseAll() error {
|
||||
return global.closeAll()
|
||||
@@ -521,6 +528,16 @@ func (m *Manager) restartPlugin(short string) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (m *Manager) reload(short string) error {
|
||||
if !m.stateStore().get(short).Enabled {
|
||||
return nil
|
||||
}
|
||||
if err := m.unload(short); err != nil && !errors.Is(err, errNotLoaded) {
|
||||
return err
|
||||
}
|
||||
return m.restartPlugin(short)
|
||||
}
|
||||
|
||||
// disableWithError persists the auto-disable state for one plugin and wraps
|
||||
// the load failure for the startup error summary.
|
||||
func (m *Manager) disableWithError(short string, st PluginState, err error) error {
|
||||
|
||||
@@ -3,6 +3,7 @@ package plugin
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/http"
|
||||
@@ -143,7 +144,12 @@ func (m *Manager) registerServerModule(host *jsruntime.Host, registry *require.R
|
||||
_ = reject(jsRPCError(vm, resp.Error))
|
||||
return
|
||||
}
|
||||
_ = resolve(resp.Result)
|
||||
result, err := normalizeRPCValue(resp.Result)
|
||||
if err != nil {
|
||||
_ = reject(vm.NewGoError(err))
|
||||
return
|
||||
}
|
||||
_ = resolve(result)
|
||||
})
|
||||
}()
|
||||
return vm.ToValue(promise)
|
||||
@@ -266,6 +272,19 @@ func jsRPCError(vm *goja.Runtime, e *rpc.JsonRpcError) *goja.Object {
|
||||
return obj
|
||||
}
|
||||
|
||||
func normalizeRPCValue(value any) (any, error) {
|
||||
data, err := json.Marshal(value)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("serialize RPC result: %w", err)
|
||||
}
|
||||
|
||||
var normalized any
|
||||
if err := json.Unmarshal(data, &normalized); err != nil {
|
||||
return nil, fmt.Errorf("decode RPC result: %w", err)
|
||||
}
|
||||
return normalized, nil
|
||||
}
|
||||
|
||||
// registerRoute registers (or reuses) a gin route slot for one plugin. It is
|
||||
// called from the plugin's own event loop during script evaluation and takes
|
||||
// the manager lock itself. The critical section is short and never waits on
|
||||
|
||||
@@ -705,7 +705,8 @@ function runtimeInfo() {
|
||||
| ChildProcess `kill/ref/unref/disconnect/send` | `部分实现` 或调用抛错,见下文 |
|
||||
|
||||
支持的 options:`cwd`、`env`、`shell`、`timeout`、`encoding`、`maxBuffer`。
|
||||
`cwd` 受 `BaseDir` 限制;请求的 timeout 和 maxBuffer 只能收紧 Go 侧全局上限,不能放大。
|
||||
`cwd` 受 `BaseDir` 限制;子进程默认不继承 runtime 的 `Timeout`,只有显式传入 `timeout` 时才会自动结束。
|
||||
请求的 `maxBuffer` 只能收紧 Go 侧全局上限,不能放大。
|
||||
`stdio/input/detached/uid/gid/windowsHide/windowsVerbatimArguments/serialization` 等未实现。
|
||||
|
||||
- `spawn()` 的 `stdin` 是 `stream.Writable`(写入经后台写队列串行执行,`end()` 后关闭管道),
|
||||
|
||||
@@ -163,7 +163,7 @@ func (m *Module) childCommandArguments(vm *goja.Runtime, call goja.FunctionCall,
|
||||
}
|
||||
|
||||
func (m *Module) childOptions(vm *goja.Runtime, value goja.Value) childCommandOptions {
|
||||
options := childCommandOptions{cwd: m.filesystem.Cwd(), env: os.Environ(), timeout: m.runtime.Timeout(), maxBuffer: m.maxOutput}
|
||||
options := childCommandOptions{cwd: m.filesystem.Cwd(), env: os.Environ(), maxBuffer: m.maxOutput}
|
||||
if value == nil || goja.IsUndefined(value) || goja.IsNull(value) {
|
||||
return options
|
||||
}
|
||||
@@ -190,7 +190,7 @@ func (m *Module) childOptions(vm *goja.Runtime, value goja.Value) childCommandOp
|
||||
}
|
||||
if timeoutValue := object.Get("timeout"); timeoutValue != nil && timeoutValue.ToInteger() > 0 {
|
||||
timeout := timeoutValue.ToInteger()
|
||||
options.timeout = min(options.timeout, time.Duration(timeout)*time.Millisecond)
|
||||
options.timeout = time.Duration(timeout) * time.Millisecond
|
||||
}
|
||||
if encoding := object.Get("encoding"); encoding != nil && !goja.IsUndefined(encoding) && !goja.IsNull(encoding) {
|
||||
options.encoding = encoding.String()
|
||||
|
||||
@@ -5,6 +5,7 @@ package console
|
||||
// substitutions. assert(false, ...) and trace() include a stack trace.
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"strings"
|
||||
@@ -121,11 +122,27 @@ func formatConsoleValues(values []goja.Value) string {
|
||||
}
|
||||
parts := make([]string, 0, len(args))
|
||||
for _, arg := range args {
|
||||
parts = append(parts, fmt.Sprint(arg))
|
||||
parts = append(parts, formatConsoleValue(arg))
|
||||
}
|
||||
return strings.Join(parts, " ")
|
||||
}
|
||||
|
||||
func formatConsoleValue(value any) string {
|
||||
switch value := value.(type) {
|
||||
case nil:
|
||||
return "null"
|
||||
case string:
|
||||
return value
|
||||
case error:
|
||||
return value.Error()
|
||||
}
|
||||
|
||||
if data, err := json.Marshal(value); err == nil {
|
||||
return string(data)
|
||||
}
|
||||
return fmt.Sprint(value)
|
||||
}
|
||||
|
||||
func formatConsoleString(format string, args []any) string {
|
||||
var builder strings.Builder
|
||||
argIndex := 0
|
||||
@@ -154,7 +171,7 @@ func formatConsoleString(format string, args []any) string {
|
||||
argIndex++
|
||||
switch format[i] {
|
||||
case 's', 'o', 'O':
|
||||
builder.WriteString(fmt.Sprint(arg))
|
||||
builder.WriteString(formatConsoleValue(arg))
|
||||
case 'd', 'i':
|
||||
builder.WriteString(fmt.Sprintf("%d", toConsoleInteger(arg)))
|
||||
case 'f':
|
||||
@@ -168,7 +185,7 @@ func formatConsoleString(format string, args []any) string {
|
||||
if argIndex < len(args) {
|
||||
remaining := make([]string, 0, len(args)-argIndex)
|
||||
for _, arg := range args[argIndex:] {
|
||||
remaining = append(remaining, fmt.Sprint(arg))
|
||||
remaining = append(remaining, formatConsoleValue(arg))
|
||||
}
|
||||
if builder.Len() > 0 {
|
||||
builder.WriteByte(' ')
|
||||
|
||||
@@ -0,0 +1,609 @@
|
||||
package metric
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
type batchSeriesGroupKey struct {
|
||||
resolution time.Duration
|
||||
needDigest bool
|
||||
}
|
||||
|
||||
type batchSeriesGroup struct {
|
||||
key batchSeriesGroupKey
|
||||
metricNames map[string]struct{}
|
||||
fields rollupReadFields
|
||||
}
|
||||
|
||||
type seriesIdentity struct {
|
||||
metricName string
|
||||
entityID string
|
||||
tagsHash string
|
||||
}
|
||||
|
||||
type seriesReadMeta struct {
|
||||
id int64
|
||||
metricName string
|
||||
entityID string
|
||||
tagsHash string
|
||||
tagsJSON string
|
||||
tags map[string]string
|
||||
}
|
||||
|
||||
type rollupRateSample struct {
|
||||
timestamp int64
|
||||
value float64
|
||||
}
|
||||
|
||||
type rollupAggregateState struct {
|
||||
summary *rollupBucket
|
||||
rateSamples []rollupRateSample
|
||||
tags map[string]string
|
||||
}
|
||||
|
||||
type metricSeriesAccumulator struct {
|
||||
spec BatchSeriesSpec
|
||||
compression float64
|
||||
needDigest bool
|
||||
needSummary bool
|
||||
needRate bool
|
||||
groups map[rollupKey]*rollupAggregateState
|
||||
rateGroups map[rollupKey]*rollupAggregateState
|
||||
}
|
||||
|
||||
// SeriesBatch is the single rollup query implementation. It resolves metric
|
||||
// metadata once, scans each selected backing tier once, and streams rows into
|
||||
// the final output buckets.
|
||||
func (s *Store) SeriesBatch(ctx context.Context, query BatchSeriesQuery, now time.Time) (BatchSeriesResult, error) {
|
||||
if err := s.ensureOpen(); err != nil {
|
||||
return BatchSeriesResult{}, err
|
||||
}
|
||||
if err := query.Validate(); err != nil {
|
||||
return BatchSeriesResult{}, err
|
||||
}
|
||||
return s.seriesBatchAt(ctx, query.normalized(), now.UTC(), nil)
|
||||
}
|
||||
|
||||
// GetMetrics loads the requested metric definitions with one database query.
|
||||
func (s *Store) GetMetrics(ctx context.Context, metricNames []string) (map[string]Definition, error) {
|
||||
if err := s.ensureOpen(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if len(metricNames) == 0 {
|
||||
return map[string]Definition{}, nil
|
||||
}
|
||||
return s.loadBatchDefinitions(ctx, metricNames)
|
||||
}
|
||||
|
||||
func (s *Store) seriesBatchAt(ctx context.Context, query BatchSeriesQuery, now time.Time, forcedResolutions map[string]time.Duration) (BatchSeriesResult, error) {
|
||||
metricNames := make([]string, 0, len(query.Specs))
|
||||
for _, spec := range query.Specs {
|
||||
metricNames = append(metricNames, spec.MetricName)
|
||||
}
|
||||
definitions, err := s.loadBatchDefinitions(ctx, metricNames)
|
||||
if err != nil {
|
||||
return BatchSeriesResult{}, err
|
||||
}
|
||||
|
||||
result := BatchSeriesResult{
|
||||
Definitions: definitions,
|
||||
Values: make(map[string]map[Aggregation][]AggregatePoint, len(query.Specs)),
|
||||
}
|
||||
accumulators := make(map[string]*metricSeriesAccumulator, len(query.Specs))
|
||||
groups := make(map[batchSeriesGroupKey]*batchSeriesGroup)
|
||||
for _, spec := range query.Specs {
|
||||
values := make(map[Aggregation][]AggregatePoint, len(spec.Aggregations))
|
||||
accumulator := &metricSeriesAccumulator{
|
||||
spec: spec, compression: s.cfg.RollupPolicy.compression(),
|
||||
groups: make(map[rollupKey]*rollupAggregateState), rateGroups: make(map[rollupKey]*rollupAggregateState),
|
||||
}
|
||||
for _, aggregation := range spec.Aggregations {
|
||||
values[aggregation] = []AggregatePoint{}
|
||||
if aggregation == AggRate {
|
||||
accumulator.needRate = true
|
||||
} else {
|
||||
accumulator.needSummary = true
|
||||
}
|
||||
if isPercentile(aggregation) {
|
||||
accumulator.needDigest = true
|
||||
}
|
||||
}
|
||||
result.Values[spec.MetricName] = values
|
||||
accumulators[spec.MetricName] = accumulator
|
||||
|
||||
policy := s.cfg.RollupPolicy
|
||||
if definition, ok := definitions[spec.MetricName]; ok {
|
||||
if definition.RetentionDays <= 0 {
|
||||
continue
|
||||
}
|
||||
policy = policy.withMetricRetention(time.Duration(definition.RetentionDays) * 24 * time.Hour)
|
||||
}
|
||||
if len(policy.Tiers) == 0 {
|
||||
continue
|
||||
}
|
||||
resolution := seriesResolutionForPolicy(query.Start, spec.Interval, now, policy)
|
||||
if forced, ok := forcedResolutions[spec.MetricName]; ok {
|
||||
resolution = forced
|
||||
}
|
||||
key := batchSeriesGroupKey{resolution: resolution, needDigest: accumulator.needDigest}
|
||||
group := groups[key]
|
||||
if group == nil {
|
||||
group = &batchSeriesGroup{key: key, metricNames: make(map[string]struct{})}
|
||||
groups[key] = group
|
||||
}
|
||||
group.metricNames[spec.MetricName] = struct{}{}
|
||||
group.fields |= rollupFieldsForAggregations(spec.Aggregations)
|
||||
}
|
||||
|
||||
seriesByID, seriesByIdentity, err := s.loadSeriesDictionary(ctx, seriesDictionaryPlan{
|
||||
MetricNames: metricNames,
|
||||
EntityIDs: query.EntityIDs,
|
||||
Tags: query.Tags,
|
||||
})
|
||||
if err != nil {
|
||||
return BatchSeriesResult{}, err
|
||||
}
|
||||
|
||||
orderedGroups := make([]*batchSeriesGroup, 0, len(groups))
|
||||
for _, group := range groups {
|
||||
orderedGroups = append(orderedGroups, group)
|
||||
}
|
||||
sort.Slice(orderedGroups, func(i, j int) bool {
|
||||
if orderedGroups[i].key.resolution != orderedGroups[j].key.resolution {
|
||||
return orderedGroups[i].key.resolution < orderedGroups[j].key.resolution
|
||||
}
|
||||
return !orderedGroups[i].key.needDigest && orderedGroups[j].key.needDigest
|
||||
})
|
||||
|
||||
s.rollupViewMu.RLock()
|
||||
defer s.rollupViewMu.RUnlock()
|
||||
for _, group := range orderedGroups {
|
||||
if err := s.scanPersistedRollupGroup(ctx, query, group, seriesByID, accumulators); err != nil {
|
||||
return BatchSeriesResult{}, err
|
||||
}
|
||||
if err := s.scanMemoryRollupGroup(query, group, seriesByIdentity, accumulators); err != nil {
|
||||
return BatchSeriesResult{}, err
|
||||
}
|
||||
}
|
||||
|
||||
for metricName, accumulator := range accumulators {
|
||||
values, err := accumulator.points(query.Order)
|
||||
if err != nil {
|
||||
return BatchSeriesResult{}, err
|
||||
}
|
||||
result.Values[metricName] = values
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func (s *Store) loadBatchDefinitions(ctx context.Context, metricNames []string) (map[string]Definition, error) {
|
||||
args := make([]any, 0, len(metricNames))
|
||||
placeholders := make([]string, 0, len(metricNames))
|
||||
for _, metricName := range metricNames {
|
||||
args = append(args, metricName)
|
||||
placeholders = append(placeholders, s.dialect.placeholder(len(args)))
|
||||
}
|
||||
rows, err := s.reader().QueryContext(ctx, fmt.Sprintf(
|
||||
"SELECT name, type, unit, description, retention_days, metadata, created_at_milli, updated_at_milli FROM %s WHERE name IN (%s) ORDER BY name ASC",
|
||||
s.tables.definitions, strings.Join(placeholders, ", "),
|
||||
), args...)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
definitions := make(map[string]Definition, len(metricNames))
|
||||
for rows.Next() {
|
||||
definition, err := scanDefinition(rows)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
definitions[definition.Name] = definition
|
||||
}
|
||||
return definitions, rows.Err()
|
||||
}
|
||||
|
||||
func (s *Store) loadSeriesDictionary(ctx context.Context, plan seriesDictionaryPlan) (map[int64]*seriesReadMeta, map[seriesIdentity]*seriesReadMeta, error) {
|
||||
rendered := s.dialect.renderSeriesDictionary(s.tables, s.cfg.TablePrefix+"series_metric_entity_idx", plan)
|
||||
rows, err := s.reader().QueryContext(ctx, rendered.Query, rendered.Args...)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
byID := make(map[int64]*seriesReadMeta)
|
||||
byIdentity := make(map[seriesIdentity]*seriesReadMeta)
|
||||
for rows.Next() {
|
||||
var meta seriesReadMeta
|
||||
var rawTags any
|
||||
if err := rows.Scan(&meta.id, &meta.metricName, &meta.entityID, &meta.tagsHash, &rawTags); err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
meta.tagsJSON, err = rawJSONToString(rawTags)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
meta.tags, err = rollupTagsFromJSON(meta.tagsJSON)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
byID[meta.id] = &meta
|
||||
byIdentity[seriesIdentity{metricName: meta.metricName, entityID: meta.entityID, tagsHash: meta.tagsHash}] = &meta
|
||||
}
|
||||
return byID, byIdentity, rows.Err()
|
||||
}
|
||||
|
||||
func (s *Store) scanPersistedRollupGroup(ctx context.Context, query BatchSeriesQuery, group *batchSeriesGroup, seriesByID map[int64]*seriesReadMeta, accumulators map[string]*metricSeriesAccumulator) error {
|
||||
seriesIDs := make([]int64, 0)
|
||||
for seriesID, meta := range seriesByID {
|
||||
if _, ok := group.metricNames[meta.metricName]; ok {
|
||||
seriesIDs = append(seriesIDs, seriesID)
|
||||
}
|
||||
}
|
||||
if len(seriesIDs) == 0 {
|
||||
return nil
|
||||
}
|
||||
sort.Slice(seriesIDs, func(i, j int) bool { return seriesIDs[i] < seriesIDs[j] })
|
||||
resolutionID, found, err := s.resolveResolutionID(ctx, group.key.resolution)
|
||||
if err != nil || !found {
|
||||
return err
|
||||
}
|
||||
rendered := s.dialect.renderRollupRead(s.tables, s.cfg.TablePrefix+"rollups_resolution_bucket_idx", rollupReadPlan{
|
||||
SeriesIDs: seriesIDs,
|
||||
ResolutionID: resolutionID,
|
||||
StartMilli: bucketStartMillis(query.Start.UnixMilli(), group.key.resolution.Milliseconds()),
|
||||
EndMilli: query.End.UnixMilli(),
|
||||
Fields: group.fields,
|
||||
})
|
||||
rows, err := s.reader().QueryContext(ctx, rendered.Query, rendered.Args...)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer rows.Close()
|
||||
var seriesID, bucket, count, firstTS, lastTS int64
|
||||
var sum, sumSq, min, max, firstVal, lastVal float64
|
||||
var digestBlob []byte
|
||||
destinations := []any{&seriesID, &bucket, &count}
|
||||
if group.fields&rollupReadSum != 0 {
|
||||
destinations = append(destinations, &sum)
|
||||
}
|
||||
if group.fields&rollupReadSumSq != 0 {
|
||||
destinations = append(destinations, &sumSq)
|
||||
}
|
||||
if group.fields&rollupReadMin != 0 {
|
||||
destinations = append(destinations, &min)
|
||||
}
|
||||
if group.fields&rollupReadMax != 0 {
|
||||
destinations = append(destinations, &max)
|
||||
}
|
||||
if group.fields&rollupReadFirst != 0 {
|
||||
destinations = append(destinations, &firstVal, &firstTS)
|
||||
}
|
||||
if group.fields&rollupReadLast != 0 {
|
||||
destinations = append(destinations, &lastVal, &lastTS)
|
||||
}
|
||||
if group.fields&rollupReadDigest != 0 {
|
||||
destinations = append(destinations, &digestBlob)
|
||||
}
|
||||
for rows.Next() {
|
||||
err = rows.Scan(destinations...)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
meta := seriesByID[seriesID]
|
||||
accumulator := accumulators[meta.metricName]
|
||||
state := accumulator.consume(meta, bucket, count, sum, sumSq, min, max, firstVal, firstTS, lastVal, lastTS, nil)
|
||||
if accumulator.needDigest {
|
||||
if err := mergeEncodedRollupDigest(state.summary.digest, count, min, max, digestBlob); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
return rows.Err()
|
||||
}
|
||||
|
||||
func rollupFieldsForAggregations(aggregations []Aggregation) rollupReadFields {
|
||||
var fields rollupReadFields
|
||||
for _, aggregation := range aggregations {
|
||||
switch aggregation {
|
||||
case AggAvg, AggSum:
|
||||
fields |= rollupReadSum
|
||||
case AggMin:
|
||||
fields |= rollupReadMin
|
||||
case AggMax:
|
||||
fields |= rollupReadMax
|
||||
case AggFirst:
|
||||
fields |= rollupReadFirst
|
||||
case AggLast, AggRate:
|
||||
fields |= rollupReadLast
|
||||
case AggStdDev:
|
||||
fields |= rollupReadSum | rollupReadSumSq
|
||||
default:
|
||||
if isPercentile(aggregation) {
|
||||
fields |= rollupReadMin | rollupReadMax | rollupReadDigest
|
||||
}
|
||||
}
|
||||
}
|
||||
return fields
|
||||
}
|
||||
|
||||
func (s *Store) resolveResolutionID(ctx context.Context, resolution time.Duration) (int64, bool, error) {
|
||||
var resolutionID int64
|
||||
err := s.reader().QueryRowContext(ctx,
|
||||
"SELECT id FROM "+s.tables.resolutions+" WHERE resolution_milli = "+s.dialect.placeholder(1),
|
||||
resolution.Milliseconds(),
|
||||
).Scan(&resolutionID)
|
||||
if err == sql.ErrNoRows {
|
||||
return 0, false, nil
|
||||
}
|
||||
return resolutionID, err == nil, err
|
||||
}
|
||||
|
||||
func (s *Store) scanMemoryRollupGroup(query BatchSeriesQuery, group *batchSeriesGroup, seriesByIdentity map[seriesIdentity]*seriesReadMeta, accumulators map[string]*metricSeriesAccumulator) error {
|
||||
if group.key.resolution == time.Minute {
|
||||
return s.scanHotRollupGroup(query, group, seriesByIdentity, accumulators)
|
||||
}
|
||||
return s.scanCoarseRollupGroup(query, group, seriesByIdentity, accumulators)
|
||||
}
|
||||
|
||||
func (s *Store) scanHotRollupGroup(query BatchSeriesQuery, group *batchSeriesGroup, seriesByIdentity map[seriesIdentity]*seriesReadMeta, accumulators map[string]*metricSeriesAccumulator) error {
|
||||
start := bucketStartMillis(query.Start.UnixMilli(), time.Minute.Milliseconds())
|
||||
end := query.End.UnixMilli()
|
||||
entityIDs := stringSet(query.EntityIDs)
|
||||
s.hotMu.RLock()
|
||||
defer s.hotMu.RUnlock()
|
||||
for key, bucket := range s.hot {
|
||||
if _, ok := group.metricNames[key.metricName]; !ok || key.bucket < start || key.bucket > end || bucket.count == 0 {
|
||||
continue
|
||||
}
|
||||
if len(entityIDs) > 0 {
|
||||
if _, ok := entityIDs[key.entityID]; !ok {
|
||||
continue
|
||||
}
|
||||
}
|
||||
meta, matched, err := memorySeriesMeta(seriesByIdentity, key.metricName, key.entityID, key.tagsHash, bucket.tagsJSON, query.Tags)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !matched {
|
||||
continue
|
||||
}
|
||||
accumulators[key.metricName].consume(meta, key.bucket, bucket.count, bucket.sum, bucket.sumSq, bucket.min, bucket.max, bucket.firstVal, bucket.firstTS, bucket.lastVal, bucket.lastTS, bucket.digest)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *Store) scanCoarseRollupGroup(query BatchSeriesQuery, group *batchSeriesGroup, seriesByIdentity map[seriesIdentity]*seriesReadMeta, accumulators map[string]*metricSeriesAccumulator) error {
|
||||
start := bucketStartMillis(query.Start.UnixMilli(), group.key.resolution.Milliseconds())
|
||||
end := query.End.UnixMilli()
|
||||
entityIDs := stringSet(query.EntityIDs)
|
||||
s.coarseMu.RLock()
|
||||
defer s.coarseMu.RUnlock()
|
||||
for key, parent := range s.coarse {
|
||||
if key.interval != group.key.resolution || key.bucket < start || key.bucket > end {
|
||||
continue
|
||||
}
|
||||
if _, ok := group.metricNames[key.metricName]; !ok {
|
||||
continue
|
||||
}
|
||||
if len(entityIDs) > 0 {
|
||||
if _, ok := entityIDs[key.entityID]; !ok {
|
||||
continue
|
||||
}
|
||||
}
|
||||
childKeys := make([]rollupKey, 0, len(parent.children))
|
||||
for childKey := range parent.children {
|
||||
childKeys = append(childKeys, childKey)
|
||||
}
|
||||
if len(childKeys) == 0 {
|
||||
continue
|
||||
}
|
||||
sortRollupKeys(childKeys)
|
||||
firstChild := parent.children[childKeys[0]]
|
||||
meta, matched, err := memorySeriesMeta(seriesByIdentity, key.metricName, key.entityID, key.tagsHash, firstChild.tagsJSON, query.Tags)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !matched {
|
||||
continue
|
||||
}
|
||||
summary := newRollupBucketWithDigest(s.cfg.RollupPolicy.compression(), group.key.needDigest)
|
||||
for _, childKey := range childKeys {
|
||||
child := parent.children[childKey]
|
||||
mergeRollupValues(summary, child.count, child.sum, child.sumSq, child.min, child.max, child.firstVal, child.firstTS, child.lastVal, child.lastTS, child.digest, group.key.needDigest)
|
||||
}
|
||||
accumulators[key.metricName].consume(meta, key.bucket, summary.count, summary.sum, summary.sumSq, summary.min, summary.max, summary.firstVal, summary.firstTS, summary.lastVal, summary.lastTS, summary.digest)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func memorySeriesMeta(seriesByIdentity map[seriesIdentity]*seriesReadMeta, metricName, entityID, tagsHash, tagsJSON string, filter map[string]string) (*seriesReadMeta, bool, error) {
|
||||
identity := seriesIdentity{metricName: metricName, entityID: entityID, tagsHash: tagsHash}
|
||||
if meta := seriesByIdentity[identity]; meta != nil {
|
||||
return meta, true, nil
|
||||
}
|
||||
tags, err := rollupTagsFromJSON(tagsJSON)
|
||||
if err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
for key, value := range filter {
|
||||
if tags[key] != value {
|
||||
return nil, false, nil
|
||||
}
|
||||
}
|
||||
meta := &seriesReadMeta{metricName: metricName, entityID: entityID, tagsHash: tagsHash, tagsJSON: tagsJSON, tags: tags}
|
||||
seriesByIdentity[identity] = meta
|
||||
return meta, true, nil
|
||||
}
|
||||
|
||||
func stringSet(values []string) map[string]struct{} {
|
||||
set := make(map[string]struct{}, len(values))
|
||||
for _, value := range values {
|
||||
set[value] = struct{}{}
|
||||
}
|
||||
return set
|
||||
}
|
||||
|
||||
func (a *metricSeriesAccumulator) consume(meta *seriesReadMeta, sourceBucket, count int64, sum, sumSq, min, max, firstVal float64, firstTS int64, lastVal float64, lastTS int64, digest *TDigest) *rollupAggregateState {
|
||||
key := rollupKey{bucket: bucketStartMillis(sourceBucket, a.spec.Interval.Milliseconds())}
|
||||
if a.spec.PreserveSeries {
|
||||
key.entityID = meta.entityID
|
||||
key.tagsHash = meta.tagsHash
|
||||
}
|
||||
if a.needSummary {
|
||||
state := a.groups[key]
|
||||
if state == nil {
|
||||
state = &rollupAggregateState{summary: newRollupBucketWithDigest(a.compression, a.needDigest)}
|
||||
if a.spec.PreserveSeries {
|
||||
state.tags = meta.tags
|
||||
}
|
||||
a.groups[key] = state
|
||||
}
|
||||
mergeRollupValues(state.summary, count, sum, sumSq, min, max, firstVal, firstTS, lastVal, lastTS, digest, a.needDigest)
|
||||
if !a.needRate {
|
||||
return state
|
||||
}
|
||||
}
|
||||
if a.needRate {
|
||||
rateKey := rollupKey{
|
||||
bucket: bucketStartMillis(sourceBucket, a.spec.Interval.Milliseconds()),
|
||||
entityID: meta.entityID,
|
||||
tagsHash: meta.tagsHash,
|
||||
}
|
||||
state := a.rateGroups[rateKey]
|
||||
if state == nil {
|
||||
state = &rollupAggregateState{tags: meta.tags}
|
||||
a.rateGroups[rateKey] = state
|
||||
}
|
||||
state.rateSamples = append(state.rateSamples, rollupRateSample{timestamp: lastTS, value: lastVal})
|
||||
}
|
||||
return a.groups[key]
|
||||
}
|
||||
|
||||
func mergeEncodedRollupDigest(target *TDigest, count int64, min, max float64, blob []byte) error {
|
||||
if count == 0 {
|
||||
return nil
|
||||
}
|
||||
if len(blob) == 0 {
|
||||
if min != max {
|
||||
return fmt.Errorf("metric: missing t-digest for rollup with %d samples", count)
|
||||
}
|
||||
target.Add(min, float64(count))
|
||||
return nil
|
||||
}
|
||||
digest, err := DecodeTDigest(blob)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
target.Merge(digest)
|
||||
return nil
|
||||
}
|
||||
|
||||
func mergeRollupValues(target *rollupBucket, count int64, sum, sumSq, min, max, firstVal float64, firstTS int64, lastVal float64, lastTS int64, digest *TDigest, includeDigest bool) {
|
||||
if count == 0 {
|
||||
return
|
||||
}
|
||||
if target.count == 0 {
|
||||
target.min, target.max = min, max
|
||||
target.firstVal, target.firstTS = firstVal, firstTS
|
||||
target.lastVal, target.lastTS = lastVal, lastTS
|
||||
} else {
|
||||
if min < target.min {
|
||||
target.min = min
|
||||
}
|
||||
if max > target.max {
|
||||
target.max = max
|
||||
}
|
||||
if firstTS < target.firstTS {
|
||||
target.firstVal, target.firstTS = firstVal, firstTS
|
||||
}
|
||||
if lastTS > target.lastTS {
|
||||
target.lastVal, target.lastTS = lastVal, lastTS
|
||||
}
|
||||
}
|
||||
target.count += count
|
||||
target.sum += sum
|
||||
target.sumSq += sumSq
|
||||
if includeDigest && digest != nil {
|
||||
target.digest.Merge(digest)
|
||||
}
|
||||
}
|
||||
|
||||
func (a *metricSeriesAccumulator) points(order Order) (map[Aggregation][]AggregatePoint, error) {
|
||||
values := make(map[Aggregation][]AggregatePoint, len(a.spec.Aggregations))
|
||||
var summaryKeys []rollupKey
|
||||
if a.needSummary {
|
||||
summaryKeys = make([]rollupKey, 0, len(a.groups))
|
||||
for key := range a.groups {
|
||||
summaryKeys = append(summaryKeys, key)
|
||||
}
|
||||
sortRollupKeys(summaryKeys)
|
||||
}
|
||||
var rateKeys []rollupKey
|
||||
if a.needRate {
|
||||
rateKeys = make([]rollupKey, 0, len(a.rateGroups))
|
||||
for key, state := range a.rateGroups {
|
||||
rateKeys = append(rateKeys, key)
|
||||
sort.SliceStable(state.rateSamples, func(i, j int) bool { return state.rateSamples[i].timestamp < state.rateSamples[j].timestamp })
|
||||
}
|
||||
sortRollupKeys(rateKeys)
|
||||
}
|
||||
for _, aggregation := range a.spec.Aggregations {
|
||||
keys := summaryKeys
|
||||
if aggregation == AggRate {
|
||||
keys = rateKeys
|
||||
}
|
||||
start, end, step := 0, len(keys), 1
|
||||
if order == OrderDesc {
|
||||
start, end, step = len(keys)-1, -1, -1
|
||||
}
|
||||
points := make([]AggregatePoint, 0, len(keys))
|
||||
for keyIndex := start; keyIndex != end; keyIndex += step {
|
||||
key := keys[keyIndex]
|
||||
state := a.groups[key]
|
||||
count, value := 0, 0.0
|
||||
if aggregation == AggRate {
|
||||
state = a.rateGroups[key]
|
||||
value = rollupCounterRate(state.rateSamples)
|
||||
count = len(state.rateSamples)
|
||||
} else {
|
||||
var ok bool
|
||||
value, ok = state.summary.value(aggregation)
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("%w: aggregation %q requires raw samples", ErrInvalidArgument, aggregation)
|
||||
}
|
||||
count = int(state.summary.count)
|
||||
}
|
||||
points = append(points, AggregatePoint{
|
||||
MetricName: a.spec.MetricName,
|
||||
EntityID: key.entityID,
|
||||
Bucket: fromMillis(key.bucket),
|
||||
Value: value,
|
||||
Count: count,
|
||||
Tags: state.tags,
|
||||
})
|
||||
}
|
||||
values[aggregation] = points
|
||||
}
|
||||
return values, nil
|
||||
}
|
||||
|
||||
func rollupCounterRate(samples []rollupRateSample) float64 {
|
||||
if len(samples) < 2 {
|
||||
return 0
|
||||
}
|
||||
seconds := float64(samples[len(samples)-1].timestamp-samples[0].timestamp) / 1000
|
||||
if seconds <= 0 {
|
||||
return 0
|
||||
}
|
||||
increase := 0.0
|
||||
for i := 1; i < len(samples); i++ {
|
||||
if delta := samples[i].value - samples[i-1].value; delta > 0 {
|
||||
increase += delta
|
||||
}
|
||||
}
|
||||
return increase / seconds
|
||||
}
|
||||
@@ -104,6 +104,12 @@ func (s *Store) addCoarseChildren(metricName string, interval time.Duration, hig
|
||||
// late-arrival window. It flushes one tier at a time so a sealed child can feed
|
||||
// the next parent without any periodic rewrite of that parent.
|
||||
func (s *Store) flushClosedCoarseRollups(ctx context.Context, now time.Time) (int, error) {
|
||||
s.rollupViewMu.Lock()
|
||||
defer s.rollupViewMu.Unlock()
|
||||
return s.flushClosedCoarseRollupsUnderView(ctx, now)
|
||||
}
|
||||
|
||||
func (s *Store) flushClosedCoarseRollupsUnderView(ctx context.Context, now time.Time) (int, error) {
|
||||
now = now.UTC()
|
||||
written := 0
|
||||
for {
|
||||
|
||||
@@ -158,11 +158,11 @@ func DefaultConfig(driver Driver, dsn string) Config {
|
||||
SQLite: SQLiteOptions{
|
||||
PerformanceProfile: SQLiteProfileBalanced,
|
||||
BusyTimeout: 5 * time.Second,
|
||||
CacheSizeKB: 16 * 1024,
|
||||
CacheSizeKB: 4 * 1024,
|
||||
TempStoreMemory: false,
|
||||
MMapSizeBytes: 32 * 1024 * 1024,
|
||||
MMapSizeBytes: 0,
|
||||
WALAutoCheckpoint: 4000,
|
||||
JournalSizeLimitBytes: 16 * 1024 * 1024,
|
||||
JournalSizeLimitBytes: 4 * 1024 * 1024,
|
||||
},
|
||||
RollupPolicy: RollupPolicy{
|
||||
RawRetention: 10 * time.Minute,
|
||||
|
||||
+2
-62
@@ -35,6 +35,8 @@ type dialect interface {
|
||||
//
|
||||
// blobType 是 rollup t-digest sketch 的列类型。
|
||||
blobType() string
|
||||
renderSeriesDictionary(tables tables, indexName string, plan seriesDictionaryPlan) renderedSQL
|
||||
renderRollupRead(tables tables, indexName string, plan rollupReadPlan) renderedSQL
|
||||
}
|
||||
|
||||
// tables stores the physical table names used by a Store.
|
||||
@@ -287,65 +289,3 @@ func (postgresDialect) jsonExtractEquals(column, key, placeholder string) string
|
||||
// escaping.
|
||||
return fmt.Sprintf("%s->>'%s' = %s", column, sqlSingleQuote(key), placeholder)
|
||||
}
|
||||
|
||||
// sqlAggValueExpr returns the SQL value expression that computes agg over the
|
||||
// "value" column for the given driver, and whether that aggregation can be
|
||||
// pushed down to this backend. Aggregations that need the ordered raw series
|
||||
// (first/last/rate) are never pushed down here. Percentiles and population
|
||||
// standard deviation are pushed down only where the backend has a portable,
|
||||
// semantics-matching function:
|
||||
//
|
||||
// - STDDEV_POP: MySQL and PostgreSQL (matches the in-memory population stddev,
|
||||
// which divides by N). SQLite has no built-in, so it falls back to memory.
|
||||
// - percentile_cont(p) WITHIN GROUP (ORDER BY value): PostgreSQL only. This
|
||||
// matches the in-memory linear-interpolation percentile for ANY p (p50,
|
||||
// p95, p99, p99.9, ...). MySQL lacks a portable continuous-percentile
|
||||
// function and SQLite has none, so both fall back to memory.
|
||||
//
|
||||
// The simple reductions (avg/min/max/sum/count) are portable everywhere.
|
||||
//
|
||||
// sqlAggValueExpr 返回给定驱动上对 "value" 列计算 agg 的 SQL 值表达式,
|
||||
// 以及该聚合是否能下推到该后端。需要有序原始序列的聚合(first/last/rate)
|
||||
// 永远不会在这里下推。百分位和总体标准差只会在后端提供可移植且语义匹配的函数时
|
||||
// 下推:
|
||||
//
|
||||
// - STDDEV_POP:MySQL 和 PostgreSQL 支持(匹配内存中的总体标准差,
|
||||
// 即除以 N)。SQLite 没有内置函数,因此回退到内存计算。
|
||||
// - percentile_cont(p) WITHIN GROUP (ORDER BY value):仅 PostgreSQL 支持。
|
||||
// 它匹配内存中任意 p(p50、p95、p99、p99.9 等)的线性插值百分位。
|
||||
// MySQL 没有可移植的连续百分位函数,SQLite 也没有,因此两者都回退到内存计算。
|
||||
//
|
||||
// 简单归约(avg/min/max/sum/count)在所有后端都可移植。
|
||||
func sqlAggValueExpr(driver Driver, agg Aggregation) (string, bool) {
|
||||
switch agg {
|
||||
case AggAvg:
|
||||
return "AVG(value)", true
|
||||
case AggMin:
|
||||
return "MIN(value)", true
|
||||
case AggMax:
|
||||
return "MAX(value)", true
|
||||
case AggSum:
|
||||
return "SUM(value)", true
|
||||
case AggCount:
|
||||
return "COUNT(*)", true
|
||||
case AggStdDev:
|
||||
switch driver {
|
||||
case DriverMySQL, DriverPostgreSQL:
|
||||
return "STDDEV_POP(value)", true
|
||||
default:
|
||||
return "", false
|
||||
}
|
||||
default:
|
||||
// Any percentile (fixed p50/p95/p99 and arbitrary pXX / pXX.X all land
|
||||
// here). PostgreSQL has a portable continuous percentile that matches the
|
||||
// in-memory linear-interpolation percentile; other backends fall back to
|
||||
// the in-memory path.
|
||||
if frac, ok := percentileFractionString(agg); ok {
|
||||
if driver == DriverPostgreSQL {
|
||||
return fmt.Sprintf("percentile_cont(%s) WITHIN GROUP (ORDER BY value)", frac), true
|
||||
}
|
||||
return "", false
|
||||
}
|
||||
return "", false
|
||||
}
|
||||
}
|
||||
@@ -60,7 +60,11 @@ func (s *Store) writePreparedHotRollups(ctx context.Context, prepared []prepared
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
_, err = s.flushClosedCoarseRollups(ctx, now)
|
||||
if len(rebuild) > 0 {
|
||||
_, err = s.flushClosedCoarseRollupsUnderView(ctx, now)
|
||||
} else {
|
||||
_, err = s.flushClosedCoarseRollups(ctx, now)
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -78,8 +82,25 @@ func (s *Store) rebuildHotRollupsLocked(keys map[hotRollupKey]struct{}, compress
|
||||
if labelID, ok := series.labelIDs[key.labelsHash]; ok {
|
||||
bucket.labelsJSON = series.labelsJSON[labelID]
|
||||
}
|
||||
for _, sample := range series.allSamples() {
|
||||
if bucketStartMillis(sample.timestamp, minuteMillis) != key.bucket || series.labelHashes[sample.labelID] != key.labelsHash {
|
||||
bucketEnd := key.bucket + minuteMillis
|
||||
decoder := newRawSampleDecoder(series.compressed)
|
||||
for sample, more := decoder.next(); more; sample, more = decoder.next() {
|
||||
if sample.timestamp < key.bucket {
|
||||
continue
|
||||
}
|
||||
if sample.timestamp >= bucketEnd {
|
||||
break
|
||||
}
|
||||
if series.labelHashes[sample.labelID] != key.labelsHash {
|
||||
continue
|
||||
}
|
||||
bucket.labelsJSON = series.labelsJSON[sample.labelID]
|
||||
bucket.addPoint(sample.value, sample.timestamp)
|
||||
}
|
||||
directStart := sort.Search(len(series.samples), func(i int) bool { return series.samples[i].timestamp >= key.bucket })
|
||||
directEnd := sort.Search(len(series.samples), func(i int) bool { return series.samples[i].timestamp >= bucketEnd })
|
||||
for _, sample := range series.samples[directStart:directEnd] {
|
||||
if series.labelHashes[sample.labelID] != key.labelsHash {
|
||||
continue
|
||||
}
|
||||
bucket.labelsJSON = series.labelsJSON[sample.labelID]
|
||||
|
||||
@@ -634,7 +634,7 @@ func TestSeriesMergesLabelsWithoutCollapsingRawShape(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestCoveringSeriesResolutionUsesFinestTierCoveringStart(t *testing.T) {
|
||||
func TestSeriesResolutionDoesNotInspectStoredCoverage(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
policy := RollupPolicy{
|
||||
Tiers: []RollupTier{
|
||||
@@ -686,16 +686,13 @@ func TestCoveringSeriesResolutionUsesFinestTierCoveringStart(t *testing.T) {
|
||||
t.Fatalf("commit seed tiers: %v", err)
|
||||
}
|
||||
|
||||
resolution, err := s.coveringSeriesResolution(ctx, Query{MetricName: "coverage-tier", EntityID: "n1", Start: now.Add(-6 * time.Hour), End: now}, time.Minute)
|
||||
if err != nil {
|
||||
t.Fatalf("select covering tier: %v", err)
|
||||
}
|
||||
if resolution != 5*time.Minute {
|
||||
t.Fatalf("covering tier = %s, want 5m", resolution)
|
||||
resolution := seriesResolutionForPolicy(now.Add(-6*time.Hour), time.Minute, now, policy)
|
||||
if resolution != time.Minute {
|
||||
t.Fatalf("selected tier = %s, want 1m", resolution)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSeriesFallsBackWhenPreferredPolicyTierHasNoRows(t *testing.T) {
|
||||
func TestSeriesReturnsEmptyWhenSelectedTierHasNoRows(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
policy := RollupPolicy{
|
||||
Tiers: []RollupTier{
|
||||
@@ -743,8 +740,8 @@ func TestSeriesFallsBackWhenPreferredPolicyTierHasNoRows(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatalf("query series: %v", err)
|
||||
}
|
||||
if len(got) != 1 || got[0].Value != 42 || got[0].Count != 1 {
|
||||
t.Fatalf("series did not fall back to existing hourly tier: %#v", got)
|
||||
if len(got) != 0 {
|
||||
t.Fatalf("series read a non-selected hourly tier: %#v", got)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -931,15 +928,33 @@ func TestSeriesHasNoTrailingEmptyBucketAcrossDashboardRanges(t *testing.T) {
|
||||
t.Fatalf("create metric: %v", err)
|
||||
}
|
||||
now := time.Date(2026, 7, 24, 12, 0, 0, 0, time.UTC)
|
||||
points := make([]Point, 0, 60*24*60)
|
||||
for at := now.Add(-60 * 24 * time.Hour); at.Before(now); at = at.Add(time.Minute) {
|
||||
points = append(points, Point{MetricName: "coverage", EntityID: "n1", Timestamp: at, Value: 1})
|
||||
tagsHash, tagsJSON, err := tagsFingerprint(nil)
|
||||
if err != nil {
|
||||
t.Fatalf("fingerprint tags: %v", err)
|
||||
}
|
||||
if err := s.WriteBatch(ctx, points); err != nil {
|
||||
t.Fatalf("write history: %v", err)
|
||||
tx, err := s.db.BeginTx(ctx, nil)
|
||||
if err != nil {
|
||||
t.Fatalf("begin history transaction: %v", err)
|
||||
}
|
||||
if _, err := s.Compact(ctx, now); err != nil {
|
||||
t.Fatalf("compact history: %v", err)
|
||||
metricPolicy := policy.withMetricRetention(90 * 24 * time.Hour)
|
||||
for _, tierIndex := range []int{0, 2, 3} {
|
||||
tier := metricPolicy.Tiers[tierIndex]
|
||||
buckets := make(map[rollupKey]*rollupBucket)
|
||||
start := bucketStartMillis(now.Add(-tier.Retention).UnixMilli(), tier.Interval.Milliseconds())
|
||||
for bucketStart := start; bucketStart < now.UnixMilli(); bucketStart += tier.Interval.Milliseconds() {
|
||||
bucket := newRollupBucket(policy.compression())
|
||||
bucket.tagsHash, bucket.tagsJSON = tagsHash, tagsJSON
|
||||
bucket.labelsHash, bucket.labelsJSON = emptyLabelsHash, "{}"
|
||||
bucket.addPoint(1, bucketStart)
|
||||
buckets[rollupKey{entityID: "n1", tagsHash: tagsHash, labelsHash: emptyLabelsHash, bucket: bucketStart}] = bucket
|
||||
}
|
||||
if _, err := s.writeRollupBucketsTx(ctx, "coverage", tier.Interval, buckets, tx); err != nil {
|
||||
_ = tx.Rollback()
|
||||
t.Fatalf("seed %s history: %v", tier.Interval, err)
|
||||
}
|
||||
}
|
||||
if err := tx.Commit(); err != nil {
|
||||
t.Fatalf("commit history: %v", err)
|
||||
}
|
||||
for _, duration := range []time.Duration{time.Hour, 6 * time.Hour, 7 * 24 * time.Hour, 30 * 24 * time.Hour, 60 * 24 * time.Hour} {
|
||||
interval := s.CompatibleSeriesInterval(now.Add(-duration), now, time.Minute)
|
||||
|
||||
@@ -104,10 +104,11 @@ func TestTagFilterPushdownWithPaging(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// TestSQLAggregateMatchesInMemory compares SQL and in-memory aggregation.
|
||||
// TestAggregateMatchesAggregatePoints compares Store aggregation with the
|
||||
// standalone aggregation helper.
|
||||
//
|
||||
// TestSQLAggregateMatchesInMemory 对比 SQL 下推聚合和内存聚合结果。
|
||||
func TestSQLAggregateMatchesInMemory(t *testing.T) {
|
||||
// TestAggregateMatchesAggregatePoints 对比 Store 聚合和独立聚合 helper 的结果。
|
||||
func TestAggregateMatchesAggregatePoints(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
s := newMemStore(t)
|
||||
if err := s.CreateMetric(ctx, Definition{Name: "g", Type: TypeGauge, RetentionDays: 30}); err != nil {
|
||||
@@ -131,10 +132,9 @@ func TestSQLAggregateMatchesInMemory(t *testing.T) {
|
||||
Aggregation: AggAvg,
|
||||
Interval: 15 * time.Second,
|
||||
}
|
||||
// SQL pushdown path.
|
||||
sqlRes, err := s.Aggregate(ctx, q)
|
||||
storeRes, err := s.Aggregate(ctx, q)
|
||||
if err != nil {
|
||||
t.Fatalf("sql aggregate: %v", err)
|
||||
t.Fatalf("store aggregate: %v", err)
|
||||
}
|
||||
// In-memory reference over the same raw points.
|
||||
pts, err := s.Query(ctx, q.Query)
|
||||
@@ -145,12 +145,12 @@ func TestSQLAggregateMatchesInMemory(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatalf("mem aggregate: %v", err)
|
||||
}
|
||||
if len(sqlRes) != len(memRes) {
|
||||
t.Fatalf("bucket count mismatch: sql=%d mem=%d", len(sqlRes), len(memRes))
|
||||
if len(storeRes) != len(memRes) {
|
||||
t.Fatalf("bucket count mismatch: store=%d helper=%d", len(storeRes), len(memRes))
|
||||
}
|
||||
for i := range sqlRes {
|
||||
if !sqlRes[i].Bucket.Equal(memRes[i].Bucket) || sqlRes[i].Value != memRes[i].Value || sqlRes[i].Count != memRes[i].Count {
|
||||
t.Fatalf("bucket %d mismatch: sql=%#v mem=%#v", i, sqlRes[i], memRes[i])
|
||||
for i := range storeRes {
|
||||
if !storeRes[i].Bucket.Equal(memRes[i].Bucket) || storeRes[i].Value != memRes[i].Value || storeRes[i].Count != memRes[i].Count {
|
||||
t.Fatalf("bucket %d mismatch: store=%#v helper=%#v", i, storeRes[i], memRes[i])
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -260,10 +260,10 @@ func TestStdDevPopMatchesCalculateStats(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// TestAggregateStdDevSQLiteUsesMemoryPath verifies SQLite stddev fallback.
|
||||
// TestAggregateStdDev verifies raw population standard deviation aggregation.
|
||||
//
|
||||
// TestAggregateStdDevSQLiteUsesMemoryPath 验证 SQLite 标准差聚合会回退到内存路径。
|
||||
func TestAggregateStdDevSQLiteUsesMemoryPath(t *testing.T) {
|
||||
// TestAggregateStdDev 验证原始样本总体标准差聚合。
|
||||
func TestAggregateStdDev(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
s := newMemStore(t)
|
||||
if err := s.CreateMetric(ctx, Definition{Name: "sd", Type: TypeGauge, RetentionDays: 30}); err != nil {
|
||||
@@ -294,36 +294,6 @@ func TestAggregateStdDevSQLiteUsesMemoryPath(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// TestSQLAggValueExprPushdownMatrix verifies aggregation pushdown support.
|
||||
//
|
||||
// TestSQLAggValueExprPushdownMatrix 验证各后端支持的聚合下推矩阵。
|
||||
func TestSQLAggValueExprPushdownMatrix(t *testing.T) {
|
||||
cases := []struct {
|
||||
driver Driver
|
||||
agg Aggregation
|
||||
ok bool
|
||||
}{
|
||||
{DriverSQLite, AggAvg, true},
|
||||
{DriverSQLite, AggStdDev, false}, // no STDDEV_POP in sqlite -> memory
|
||||
{DriverSQLite, AggP95, false}, // no percentile in sqlite -> memory
|
||||
{DriverMySQL, AggStdDev, true}, // STDDEV_POP
|
||||
{DriverMySQL, AggP95, false}, // no portable continuous percentile
|
||||
{DriverPostgreSQL, AggStdDev, true},
|
||||
{DriverPostgreSQL, AggP50, true}, // percentile_cont
|
||||
{DriverPostgreSQL, AggP99, true},
|
||||
{DriverPostgreSQL, AggFirst, false}, // needs ordered raw series
|
||||
}
|
||||
for _, c := range cases {
|
||||
expr, ok := sqlAggValueExpr(c.driver, c.agg)
|
||||
if ok != c.ok {
|
||||
t.Fatalf("%s/%s: pushdown=%v want %v (expr=%q)", c.driver, c.agg, ok, c.ok, expr)
|
||||
}
|
||||
if ok && expr == "" {
|
||||
t.Fatalf("%s/%s: pushdown ok but empty expr", c.driver, c.agg)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestSQLiteReadPoolOpens verifies SQLite read-pool creation.
|
||||
//
|
||||
// TestSQLiteReadPoolOpens 验证 SQLite 只读连接池会按配置打开。
|
||||
@@ -403,10 +373,10 @@ func TestWriteBatchAtomicAcrossChunks(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// TestAggregateBucketPagingSQLPath verifies bucket paging in SQL aggregation.
|
||||
// TestAggregateBucketPaging verifies bucket paging.
|
||||
//
|
||||
// TestAggregateBucketPagingSQLPath 验证 SQL 聚合路径按桶分页。
|
||||
func TestAggregateBucketPagingSQLPath(t *testing.T) {
|
||||
// TestAggregateBucketPaging 验证聚合桶分页。
|
||||
func TestAggregateBucketPaging(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
s := newMemStore(t)
|
||||
if err := s.CreateMetric(ctx, Definition{Name: "bp", Type: TypeGauge, RetentionDays: 30}); err != nil {
|
||||
@@ -441,10 +411,11 @@ func TestAggregateBucketPagingSQLPath(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// TestAggregateBucketPagingMemoryPathMatchesSQL compares bucket paging paths.
|
||||
// TestAggregateBucketPagingMatchesAcrossAggregations compares bucket paging
|
||||
// across aggregation types.
|
||||
//
|
||||
// TestAggregateBucketPagingMemoryPathMatchesSQL 对比内存聚合和 SQL 聚合的桶分页语义。
|
||||
func TestAggregateBucketPagingMemoryPathMatchesSQL(t *testing.T) {
|
||||
// TestAggregateBucketPagingMatchesAcrossAggregations 对比不同聚合类型的桶分页语义。
|
||||
func TestAggregateBucketPagingMatchesAcrossAggregations(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
s := newMemStore(t)
|
||||
if err := s.CreateMetric(ctx, Definition{Name: "bp2", Type: TypeGauge, RetentionDays: 30}); err != nil {
|
||||
@@ -459,28 +430,26 @@ func TestAggregateBucketPagingMemoryPathMatchesSQL(t *testing.T) {
|
||||
t.Fatalf("write: %v", err)
|
||||
}
|
||||
q := Query{MetricName: "bp2", EntityID: "n1", Start: base, End: base.Add(time.Minute)}
|
||||
// AggMin (SQL pushdown) and AggP50 (memory fallback on sqlite) with identical
|
||||
// bucket paging must select the same bucket window (the same timestamps).
|
||||
sqlPaged, err := s.Aggregate(ctx, AggregateQuery{Query: q, Aggregation: AggMin, Interval: 10 * time.Second, BucketLimit: 3, BucketOffset: 2})
|
||||
minPaged, err := s.Aggregate(ctx, AggregateQuery{Query: q, Aggregation: AggMin, Interval: 10 * time.Second, BucketLimit: 3, BucketOffset: 2})
|
||||
if err != nil {
|
||||
t.Fatalf("sql aggregate: %v", err)
|
||||
t.Fatalf("min aggregate: %v", err)
|
||||
}
|
||||
memPaged, err := s.Aggregate(ctx, AggregateQuery{Query: q, Aggregation: AggP50, Interval: 10 * time.Second, BucketLimit: 3, BucketOffset: 2})
|
||||
p50Paged, err := s.Aggregate(ctx, AggregateQuery{Query: q, Aggregation: AggP50, Interval: 10 * time.Second, BucketLimit: 3, BucketOffset: 2})
|
||||
if err != nil {
|
||||
t.Fatalf("mem aggregate: %v", err)
|
||||
t.Fatalf("p50 aggregate: %v", err)
|
||||
}
|
||||
if len(sqlPaged) != 3 || len(memPaged) != 3 {
|
||||
t.Fatalf("expected 3 buckets each, got sql=%d mem=%d", len(sqlPaged), len(memPaged))
|
||||
if len(minPaged) != 3 || len(p50Paged) != 3 {
|
||||
t.Fatalf("expected 3 buckets each, got min=%d p50=%d", len(minPaged), len(p50Paged))
|
||||
}
|
||||
for i := range sqlPaged {
|
||||
if !sqlPaged[i].Bucket.Equal(memPaged[i].Bucket) {
|
||||
t.Fatalf("bucket window mismatch at %d: sql=%v mem=%v", i, sqlPaged[i].Bucket, memPaged[i].Bucket)
|
||||
for i := range minPaged {
|
||||
if !minPaged[i].Bucket.Equal(p50Paged[i].Bucket) {
|
||||
t.Fatalf("bucket window mismatch at %d: min=%v p50=%v", i, minPaged[i].Bucket, p50Paged[i].Bucket)
|
||||
}
|
||||
}
|
||||
// Per-bucket single point, so min == p50 == the value.
|
||||
for i := range sqlPaged {
|
||||
if sqlPaged[i].Value != memPaged[i].Value {
|
||||
t.Fatalf("value mismatch at %d: sql=%v mem=%v", i, sqlPaged[i].Value, memPaged[i].Value)
|
||||
for i := range minPaged {
|
||||
if minPaged[i].Value != p50Paged[i].Value {
|
||||
t.Fatalf("value mismatch at %d: min=%v p50=%v", i, minPaged[i].Value, p50Paged[i].Value)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -125,6 +125,9 @@ func (s *Store) ReclaimSpace(ctx context.Context) error {
|
||||
if _, err := s.db.ExecContext(ctx, sqliteVacuumSQL); err != nil {
|
||||
return fmt.Errorf("metric: vacuum sqlite database: %w", err)
|
||||
}
|
||||
if _, err := s.db.ExecContext(ctx, "PRAGMA optimize"); err != nil {
|
||||
return fmt.Errorf("metric: optimize sqlite database: %w", err)
|
||||
}
|
||||
// VACUUM itself can populate the WAL; truncate it again so the reported
|
||||
// physical size reflects the completed reclamation.
|
||||
return sqliteCheckpoint(ctx, s.db)
|
||||
|
||||
@@ -21,7 +21,14 @@ func (s *Store) Migrate(ctx context.Context) error {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return s.createNormalizedIndexes(ctx)
|
||||
if err := s.createNormalizedIndexes(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
if s.cfg.Driver == DriverSQLite {
|
||||
_, err := s.db.ExecContext(ctx, "PRAGMA optimize")
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *Store) normalizedSchemaStatements() []string {
|
||||
|
||||
@@ -18,7 +18,7 @@ func TestPostgreSQLIntegration(t *testing.T) {
|
||||
t.Skip("METRIC_POSTGRES_DSN is not set")
|
||||
}
|
||||
|
||||
runSQLIntegration(t, "postgres", PostgreSQL(dsn), true)
|
||||
runSQLIntegration(t, "postgres", PostgreSQL(dsn))
|
||||
runSQLRestructureIntegration(t, "postgres", PostgreSQL(dsn))
|
||||
}
|
||||
|
||||
@@ -31,14 +31,24 @@ func TestMySQLIntegration(t *testing.T) {
|
||||
t.Skip("METRIC_MYSQL_DSN is not set")
|
||||
}
|
||||
|
||||
runSQLIntegration(t, "mysql", MySQL(dsn), false)
|
||||
runSQLIntegration(t, "mysql", MySQL(dsn))
|
||||
runSQLRestructureIntegration(t, "mysql", MySQL(dsn))
|
||||
}
|
||||
|
||||
func TestMariaDBIntegration(t *testing.T) {
|
||||
dsn := os.Getenv("METRIC_MARIADB_DSN")
|
||||
if dsn == "" {
|
||||
t.Skip("METRIC_MARIADB_DSN is not set")
|
||||
}
|
||||
|
||||
runSQLIntegration(t, "mariadb", MySQL(dsn))
|
||||
runSQLRestructureIntegration(t, "mariadb", MySQL(dsn))
|
||||
}
|
||||
|
||||
// runSQLIntegration exercises the SQL store against an external database.
|
||||
//
|
||||
// runSQLIntegration 在外部数据库上执行通用 SQL 集成测试流程。
|
||||
func runSQLIntegration(t *testing.T, name string, cfg Config, expectSQLPercentile bool) {
|
||||
func runSQLIntegration(t *testing.T, name string, cfg Config) {
|
||||
t.Helper()
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 6*time.Minute)
|
||||
@@ -160,10 +170,6 @@ func runSQLIntegration(t *testing.T, name string, cfg Config, expectSQLPercentil
|
||||
if len(p95Buckets) != 1 || p95Buckets[0].Count != 3 || p95Buckets[0].Value <= 28 {
|
||||
t.Fatalf("unexpected p95 aggregate: %#v", p95Buckets)
|
||||
}
|
||||
if _, ok := sqlAggValueExpr(cfg.Driver, AggP95); ok != expectSQLPercentile {
|
||||
t.Fatalf("%s p95 pushdown expectation mismatch: got %v want %v", name, ok, expectSQLPercentile)
|
||||
}
|
||||
|
||||
latest, err := store.Latest(ctx, "http.latency", "api-1", 1)
|
||||
if err != nil {
|
||||
t.Fatalf("latest: %v", err)
|
||||
|
||||
+308
-173
@@ -38,6 +38,61 @@ type compressedRawSamples struct {
|
||||
lastStamp int64
|
||||
}
|
||||
|
||||
type rawSampleDecoder struct {
|
||||
data []byte
|
||||
offset int
|
||||
index int
|
||||
count int
|
||||
previous int64
|
||||
}
|
||||
|
||||
func newRawSampleDecoder(samples compressedRawSamples) rawSampleDecoder {
|
||||
return rawSampleDecoder{data: samples.data, count: samples.count}
|
||||
}
|
||||
|
||||
func (d *rawSampleDecoder) next() (rawSample, bool) {
|
||||
if d.index == d.count {
|
||||
return rawSample{}, false
|
||||
}
|
||||
var timestamp int64
|
||||
if d.index == 0 {
|
||||
value, size := binary.Varint(d.data[d.offset:])
|
||||
d.offset += size
|
||||
timestamp = value
|
||||
} else {
|
||||
delta, size := binary.Uvarint(d.data[d.offset:])
|
||||
d.offset += size
|
||||
timestamp = d.previous + int64(delta)
|
||||
}
|
||||
value := math.Float64frombits(binary.LittleEndian.Uint64(d.data[d.offset:]))
|
||||
d.offset += 8
|
||||
labelID, size := binary.Uvarint(d.data[d.offset:])
|
||||
d.offset += size
|
||||
d.index++
|
||||
d.previous = timestamp
|
||||
return rawSample{timestamp: timestamp, value: value, labelID: uint32(labelID)}, true
|
||||
}
|
||||
|
||||
func newCompressedRawSamples(capacity int) compressedRawSamples {
|
||||
return compressedRawSamples{data: make([]byte, 0, capacity)}
|
||||
}
|
||||
|
||||
func (c *compressedRawSamples) appendSample(sample rawSample) {
|
||||
if c.count == 0 {
|
||||
c.data = appendVarint(c.data, sample.timestamp)
|
||||
} else {
|
||||
c.data = appendUvarint(c.data, uint64(sample.timestamp-c.lastStamp))
|
||||
}
|
||||
c.data = appendRawSample(c.data, sample)
|
||||
c.count++
|
||||
c.lastStamp = sample.timestamp
|
||||
}
|
||||
|
||||
func (c compressedRawSamples) firstStamp() int64 {
|
||||
value, _ := binary.Varint(c.data)
|
||||
return value
|
||||
}
|
||||
|
||||
// rawSeries stores exact samples compactly for one logical series. Tags are
|
||||
// shared by the series and labels are interned into small integer ids.
|
||||
type rawSeries struct {
|
||||
@@ -170,31 +225,98 @@ func markRawRebuild(rebuild map[hotRollupKey]struct{}, point preparedMetricPoint
|
||||
rebuild[hotRollupKey{metricName: point.metricName, entityID: point.entityID, tagsHash: point.tagsHash, labelsHash: labelsHash, bucket: bucketStartMillis(point.timestamp, time.Minute.Milliseconds())}] = struct{}{}
|
||||
}
|
||||
|
||||
type rawBatchPoints struct {
|
||||
points []Point
|
||||
tagsHashes []string
|
||||
order Order
|
||||
}
|
||||
|
||||
func (p *rawBatchPoints) Len() int { return len(p.points) }
|
||||
|
||||
func (p *rawBatchPoints) Less(i, j int) bool {
|
||||
if p.order == OrderDesc {
|
||||
return rawPointLess(p.points[j], p.tagsHashes[j], p.points[i], p.tagsHashes[i])
|
||||
}
|
||||
return rawPointLess(p.points[i], p.tagsHashes[i], p.points[j], p.tagsHashes[j])
|
||||
}
|
||||
|
||||
func (p *rawBatchPoints) Swap(i, j int) {
|
||||
p.points[i], p.points[j] = p.points[j], p.points[i]
|
||||
p.tagsHashes[i], p.tagsHashes[j] = p.tagsHashes[j], p.tagsHashes[i]
|
||||
}
|
||||
|
||||
func (p *rawBatchPoints) grow(additional int) {
|
||||
if additional <= cap(p.points)-len(p.points) {
|
||||
return
|
||||
}
|
||||
pointCount := len(p.points) + additional
|
||||
points := make([]Point, len(p.points), pointCount)
|
||||
copy(points, p.points)
|
||||
p.points = points
|
||||
tagsHashes := make([]string, len(p.tagsHashes), pointCount)
|
||||
copy(tagsHashes, p.tagsHashes)
|
||||
p.tagsHashes = tagsHashes
|
||||
}
|
||||
|
||||
func (s *Store) queryRawPoints(ctx context.Context, query Query) ([]Point, error) {
|
||||
entityIDs := []string(nil)
|
||||
if query.EntityID != "" {
|
||||
entityIDs = []string{query.EntityID}
|
||||
}
|
||||
batch, err := s.queryRawPointsBatch(ctx, BatchQuery{
|
||||
MetricNames: []string{query.MetricName},
|
||||
EntityIDs: entityIDs,
|
||||
Start: query.Start,
|
||||
End: query.End,
|
||||
Tags: query.Tags,
|
||||
Order: query.Order,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
points := batch[query.MetricName]
|
||||
startIndex := query.Offset
|
||||
if startIndex > len(points) {
|
||||
startIndex = len(points)
|
||||
}
|
||||
endIndex := len(points)
|
||||
if query.Limit > 0 && startIndex+query.Limit < endIndex {
|
||||
endIndex = startIndex + query.Limit
|
||||
}
|
||||
return points[startIndex:endIndex], nil
|
||||
}
|
||||
|
||||
func (s *Store) queryRawPointsBatch(ctx context.Context, query BatchQuery) (map[string][]Point, error) {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
start, end := query.Start.UnixMilli(), query.End.UnixMilli()
|
||||
s.rawMu.Lock()
|
||||
defer s.rawMu.Unlock()
|
||||
now := time.Now().UTC()
|
||||
visibleCutoff := now.Add(-rawMemoryRetention).UnixMilli()
|
||||
visibleCutoff := time.Now().UTC().Add(-rawMemoryRetention).UnixMilli()
|
||||
if start < visibleCutoff {
|
||||
start = visibleCutoff
|
||||
}
|
||||
s.compressRawBeforeLocked("", now.Add(-directRawRetention).UnixMilli())
|
||||
s.pruneRawBeforeLocked("", rawMemoryCutoff(now))
|
||||
|
||||
type resultPoint struct {
|
||||
point Point
|
||||
entityID string
|
||||
tagsHash string
|
||||
metricNames := make(map[string]struct{}, len(query.MetricNames))
|
||||
matched := make(map[string]*rawBatchPoints, len(query.MetricNames))
|
||||
for _, metricName := range query.MetricNames {
|
||||
metricNames[metricName] = struct{}{}
|
||||
matched[metricName] = &rawBatchPoints{order: query.Order}
|
||||
}
|
||||
matched := make([]resultPoint, 0)
|
||||
entityIDs := make(map[string]struct{}, len(query.EntityIDs))
|
||||
for _, entityID := range query.EntityIDs {
|
||||
entityIDs[entityID] = struct{}{}
|
||||
}
|
||||
|
||||
s.rawMu.RLock()
|
||||
defer s.rawMu.RUnlock()
|
||||
for key, series := range s.raw {
|
||||
if key.metricName != query.MetricName || (query.EntityID != "" && key.entityID != query.EntityID) {
|
||||
if _, ok := metricNames[key.metricName]; !ok {
|
||||
continue
|
||||
}
|
||||
if len(entityIDs) > 0 {
|
||||
if _, ok := entityIDs[key.entityID]; !ok {
|
||||
continue
|
||||
}
|
||||
}
|
||||
tags, ok, err := matchRawTags(series.tagsJSON, query.Tags)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
@@ -202,68 +324,88 @@ func (s *Store) queryRawPoints(ctx context.Context, query Query) ([]Point, error
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
all := series.allSamples()
|
||||
labels := make(map[uint32]map[string]string)
|
||||
first := sort.Search(len(all), func(i int) bool { return all[i].timestamp >= start })
|
||||
for _, sample := range all[first:] {
|
||||
if sample.timestamp > end {
|
||||
break
|
||||
}
|
||||
labelMap := labels[sample.labelID]
|
||||
if labelMap == nil {
|
||||
labelMap, err = decodeMapString(series.labelsJSON[sample.labelID])
|
||||
directStart := sort.Search(len(series.samples), func(i int) bool { return series.samples[i].timestamp >= start })
|
||||
directEnd := sort.Search(len(series.samples), func(i int) bool { return series.samples[i].timestamp > end })
|
||||
additional := directEnd - directStart
|
||||
if series.compressed.count > 0 && start <= series.compressed.firstStamp() && end >= series.compressed.lastStamp {
|
||||
additional += series.compressed.count
|
||||
}
|
||||
metricPoints := matched[key.metricName]
|
||||
metricPoints.grow(additional)
|
||||
if series.compressed.count > 0 && start <= series.compressed.lastStamp && end >= series.compressed.firstStamp() {
|
||||
decoder := newRawSampleDecoder(series.compressed)
|
||||
for sample, more := decoder.next(); more; sample, more = decoder.next() {
|
||||
if sample.timestamp < start {
|
||||
continue
|
||||
}
|
||||
if sample.timestamp > end {
|
||||
break
|
||||
}
|
||||
labelMap, err := rawSampleLabels(series, labels, sample.labelID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
labels[sample.labelID] = labelMap
|
||||
appendRawQueryPoint(metricPoints, key, tags, labelMap, sample)
|
||||
}
|
||||
matched = append(matched, resultPoint{
|
||||
point: Point{MetricName: key.metricName, EntityID: key.entityID, Timestamp: fromMillis(sample.timestamp), Value: sample.value, Tags: cloneStringMap(tags), Labels: cloneStringMap(labelMap)},
|
||||
entityID: key.entityID,
|
||||
tagsHash: key.tagsHash,
|
||||
})
|
||||
}
|
||||
for _, sample := range series.samples[directStart:directEnd] {
|
||||
labelMap, err := rawSampleLabels(series, labels, sample.labelID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
appendRawQueryPoint(metricPoints, key, tags, labelMap, sample)
|
||||
}
|
||||
}
|
||||
sort.Slice(matched, func(i, j int) bool {
|
||||
left, right := matched[i], matched[j]
|
||||
less := left.point.Timestamp.Before(right.point.Timestamp)
|
||||
if left.point.Timestamp.Equal(right.point.Timestamp) {
|
||||
less = left.entityID < right.entityID || (left.entityID == right.entityID && left.tagsHash < right.tagsHash)
|
||||
}
|
||||
if query.Order == OrderDesc {
|
||||
return !less && !(left.point.Timestamp.Equal(right.point.Timestamp) && left.entityID == right.entityID && left.tagsHash == right.tagsHash)
|
||||
}
|
||||
return less
|
||||
})
|
||||
startIndex := query.Offset
|
||||
if startIndex > len(matched) {
|
||||
startIndex = len(matched)
|
||||
}
|
||||
endIndex := len(matched)
|
||||
if query.Limit > 0 && startIndex+query.Limit < endIndex {
|
||||
endIndex = startIndex + query.Limit
|
||||
}
|
||||
result := make([]Point, 0, endIndex-startIndex)
|
||||
for _, item := range matched[startIndex:endIndex] {
|
||||
result = append(result, item.point)
|
||||
result := make(map[string][]Point, len(matched))
|
||||
for metricName, metricPoints := range matched {
|
||||
sort.Sort(metricPoints)
|
||||
result[metricName] = metricPoints.points
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func rawSampleLabels(series *rawSeries, labels map[uint32]map[string]string, labelID uint32) (map[string]string, error) {
|
||||
if labelMap := labels[labelID]; labelMap != nil {
|
||||
return labelMap, nil
|
||||
}
|
||||
labelMap, err := decodeMapString(series.labelsJSON[labelID])
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
labels[labelID] = labelMap
|
||||
return labelMap, nil
|
||||
}
|
||||
|
||||
func appendRawQueryPoint(points *rawBatchPoints, key rawSeriesKey, tags, labels map[string]string, sample rawSample) {
|
||||
points.points = append(points.points, Point{
|
||||
MetricName: key.metricName, EntityID: key.entityID,
|
||||
Timestamp: fromMillis(sample.timestamp), Value: sample.value, Tags: tags, Labels: labels,
|
||||
})
|
||||
points.tagsHashes = append(points.tagsHashes, key.tagsHash)
|
||||
}
|
||||
|
||||
func rawPointLess(left Point, leftTagsHash string, right Point, rightTagsHash string) bool {
|
||||
if !left.Timestamp.Equal(right.Timestamp) {
|
||||
return left.Timestamp.Before(right.Timestamp)
|
||||
}
|
||||
if left.EntityID != right.EntityID {
|
||||
return left.EntityID < right.EntityID
|
||||
}
|
||||
return leftTagsHash < rightTagsHash
|
||||
}
|
||||
|
||||
func (s *Store) rawEntityIDs(ctx context.Context, query Query) ([]string, error) {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
start, end := query.Start.UnixMilli(), query.End.UnixMilli()
|
||||
s.rawMu.Lock()
|
||||
defer s.rawMu.Unlock()
|
||||
now := time.Now().UTC()
|
||||
visibleCutoff := now.Add(-rawMemoryRetention).UnixMilli()
|
||||
s.rawMu.RLock()
|
||||
defer s.rawMu.RUnlock()
|
||||
visibleCutoff := time.Now().UTC().Add(-rawMemoryRetention).UnixMilli()
|
||||
if start < visibleCutoff {
|
||||
start = visibleCutoff
|
||||
}
|
||||
s.compressRawBeforeLocked("", now.Add(-directRawRetention).UnixMilli())
|
||||
s.pruneRawBeforeLocked("", rawMemoryCutoff(now))
|
||||
seen := make(map[string]struct{})
|
||||
for key, series := range s.raw {
|
||||
if key.metricName != query.MetricName || (query.EntityID != "" && key.entityID != query.EntityID) {
|
||||
@@ -276,9 +418,25 @@ func (s *Store) rawEntityIDs(ctx context.Context, query Query) ([]string, error)
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
all := series.allSamples()
|
||||
index := sort.Search(len(all), func(i int) bool { return all[i].timestamp >= start })
|
||||
if index < len(all) && all[index].timestamp <= end {
|
||||
found := false
|
||||
if series.compressed.count > 0 && start <= series.compressed.lastStamp && end >= series.compressed.firstStamp() {
|
||||
decoder := newRawSampleDecoder(series.compressed)
|
||||
for sample, more := decoder.next(); more; sample, more = decoder.next() {
|
||||
if sample.timestamp < start {
|
||||
continue
|
||||
}
|
||||
if sample.timestamp > end {
|
||||
break
|
||||
}
|
||||
found = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if !found {
|
||||
index := sort.Search(len(series.samples), func(i int) bool { return series.samples[i].timestamp >= start })
|
||||
found = index < len(series.samples) && series.samples[index].timestamp <= end
|
||||
}
|
||||
if found {
|
||||
seen[key.entityID] = struct{}{}
|
||||
}
|
||||
}
|
||||
@@ -355,32 +513,48 @@ func (s *Store) pruneRawBeforeLocked(metricName string, beforeMilli int64) int64
|
||||
if metricName != "" && key.metricName != metricName {
|
||||
continue
|
||||
}
|
||||
compressed := series.decodeCompressed()
|
||||
compressedIndex := sort.Search(len(compressed), func(i int) bool { return compressed[i].timestamp >= beforeMilli })
|
||||
if compressedIndex > 0 {
|
||||
compressed = compressed[compressedIndex:]
|
||||
series.compressed.set(compressed)
|
||||
deleted += int64(compressedIndex)
|
||||
}
|
||||
if len(compressed) == 0 {
|
||||
series.compressed = compressedRawSamples{}
|
||||
}
|
||||
compressedDeleted := pruneCompressedRawSamples(&series.compressed, beforeMilli)
|
||||
deleted += int64(compressedDeleted)
|
||||
directIndex := sort.Search(len(series.samples), func(i int) bool { return series.samples[i].timestamp >= beforeMilli })
|
||||
if directIndex > 0 {
|
||||
deleted += int64(directIndex)
|
||||
series.samples = append([]rawSample(nil), series.samples[directIndex:]...)
|
||||
copy(series.samples, series.samples[directIndex:])
|
||||
series.samples = series.samples[:len(series.samples)-directIndex]
|
||||
}
|
||||
if series.compressed.count == 0 && len(series.samples) == 0 {
|
||||
delete(s.raw, key)
|
||||
continue
|
||||
}
|
||||
if compressedIndex > 0 || directIndex > 0 {
|
||||
if compressedDeleted > 0 || directIndex > 0 {
|
||||
compactRawLabels(series)
|
||||
}
|
||||
}
|
||||
return deleted
|
||||
}
|
||||
|
||||
func pruneCompressedRawSamples(samples *compressedRawSamples, beforeMilli int64) int {
|
||||
if samples.count == 0 || samples.firstStamp() >= beforeMilli {
|
||||
return 0
|
||||
}
|
||||
if samples.lastStamp < beforeMilli {
|
||||
deleted := samples.count
|
||||
*samples = compressedRawSamples{}
|
||||
return deleted
|
||||
}
|
||||
decoder := newRawSampleDecoder(*samples)
|
||||
kept := newCompressedRawSamples(len(samples.data))
|
||||
deleted := 0
|
||||
for sample, more := decoder.next(); more; sample, more = decoder.next() {
|
||||
if sample.timestamp < beforeMilli {
|
||||
deleted++
|
||||
continue
|
||||
}
|
||||
kept.appendSample(sample)
|
||||
}
|
||||
*samples = kept
|
||||
return deleted
|
||||
}
|
||||
|
||||
func (s *Store) compressRawBeforeLocked(metricName string, beforeMilli int64) {
|
||||
for key, series := range s.raw {
|
||||
if metricName != "" && key.metricName != metricName {
|
||||
@@ -391,70 +565,38 @@ func (s *Store) compressRawBeforeLocked(metricName string, beforeMilli int64) {
|
||||
continue
|
||||
}
|
||||
series.compressed.append(series.samples[:index])
|
||||
series.samples = append([]rawSample(nil), series.samples[index:]...)
|
||||
}
|
||||
}
|
||||
|
||||
func (s *rawSeries) decodeCompressed() []rawSample {
|
||||
if s.compressed.count == 0 {
|
||||
return nil
|
||||
}
|
||||
result := make([]rawSample, 0, s.compressed.count)
|
||||
for offset, previous := 0, int64(0); offset < len(s.compressed.data); {
|
||||
var timestamp int64
|
||||
if len(result) == 0 {
|
||||
first, n := binary.Varint(s.compressed.data[offset:])
|
||||
if n <= 0 {
|
||||
return result
|
||||
}
|
||||
offset += n
|
||||
timestamp = first
|
||||
} else {
|
||||
delta, dn := binary.Uvarint(s.compressed.data[offset:])
|
||||
if dn <= 0 {
|
||||
return result
|
||||
}
|
||||
offset += dn
|
||||
timestamp = previous + int64(delta)
|
||||
copy(series.samples, series.samples[index:])
|
||||
series.samples = series.samples[:len(series.samples)-index]
|
||||
if len(series.samples) == 0 {
|
||||
series.samples = nil
|
||||
}
|
||||
if offset+8 > len(s.compressed.data) {
|
||||
return result
|
||||
}
|
||||
value := math.Float64frombits(binary.LittleEndian.Uint64(s.compressed.data[offset:]))
|
||||
offset += 8
|
||||
labelID, ln := binary.Uvarint(s.compressed.data[offset:])
|
||||
if ln <= 0 {
|
||||
return result
|
||||
}
|
||||
offset += ln
|
||||
result = append(result, rawSample{timestamp: timestamp, value: value, labelID: uint32(labelID)})
|
||||
previous = timestamp
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
func (s *rawSeries) allSamples() []rawSample {
|
||||
compressed := s.decodeCompressed()
|
||||
result := make([]rawSample, 0, len(compressed)+len(s.samples))
|
||||
result = append(result, compressed...)
|
||||
result = append(result, s.samples...)
|
||||
return result
|
||||
}
|
||||
|
||||
func (s *rawSeries) replaceCompressed(sample rawSample) (rawSample, bool) {
|
||||
samples := s.decodeCompressed()
|
||||
index := sort.Search(len(samples), func(i int) bool { return samples[i].timestamp >= sample.timestamp })
|
||||
var old rawSample
|
||||
replaced := index < len(samples) && samples[index].timestamp == sample.timestamp
|
||||
if replaced {
|
||||
old = samples[index]
|
||||
samples[index] = sample
|
||||
} else {
|
||||
samples = append(samples, rawSample{})
|
||||
copy(samples[index+1:], samples[index:])
|
||||
samples[index] = sample
|
||||
replaced := false
|
||||
inserted := false
|
||||
decoder := newRawSampleDecoder(s.compressed)
|
||||
encoded := newCompressedRawSamples(len(s.compressed.data) + binary.MaxVarintLen64 + 9)
|
||||
for current, more := decoder.next(); more; current, more = decoder.next() {
|
||||
if !inserted && sample.timestamp < current.timestamp {
|
||||
encoded.appendSample(sample)
|
||||
inserted = true
|
||||
}
|
||||
if !inserted && sample.timestamp == current.timestamp {
|
||||
old = current
|
||||
replaced = true
|
||||
encoded.appendSample(sample)
|
||||
inserted = true
|
||||
continue
|
||||
}
|
||||
encoded.appendSample(current)
|
||||
}
|
||||
s.compressed.set(samples)
|
||||
if !inserted {
|
||||
encoded.appendSample(sample)
|
||||
}
|
||||
s.compressed = encoded
|
||||
return old, replaced
|
||||
}
|
||||
|
||||
@@ -463,51 +605,39 @@ func (c *compressedRawSamples) append(samples []rawSample) {
|
||||
return
|
||||
}
|
||||
if c.count > 0 && samples[0].timestamp <= c.lastStamp {
|
||||
merged := make([]rawSample, 0, c.count+len(samples))
|
||||
merged = append(merged, c.decode()...)
|
||||
for _, sample := range samples {
|
||||
index := sort.Search(len(merged), func(i int) bool { return merged[i].timestamp >= sample.timestamp })
|
||||
if index < len(merged) && merged[index].timestamp == sample.timestamp {
|
||||
merged[index] = sample
|
||||
decoder := newRawSampleDecoder(*c)
|
||||
merged := newCompressedRawSamples(len(c.data) + len(samples)*10)
|
||||
current, more := decoder.next()
|
||||
for sampleIndex := 0; more || sampleIndex < len(samples); {
|
||||
if !more {
|
||||
merged.appendSample(samples[sampleIndex])
|
||||
sampleIndex++
|
||||
continue
|
||||
}
|
||||
merged = append(merged, rawSample{})
|
||||
copy(merged[index+1:], merged[index:])
|
||||
merged[index] = sample
|
||||
if sampleIndex == len(samples) {
|
||||
merged.appendSample(current)
|
||||
current, more = decoder.next()
|
||||
continue
|
||||
}
|
||||
next := samples[sampleIndex]
|
||||
switch {
|
||||
case next.timestamp < current.timestamp:
|
||||
merged.appendSample(next)
|
||||
sampleIndex++
|
||||
case next.timestamp == current.timestamp:
|
||||
merged.appendSample(next)
|
||||
sampleIndex++
|
||||
current, more = decoder.next()
|
||||
default:
|
||||
merged.appendSample(current)
|
||||
current, more = decoder.next()
|
||||
}
|
||||
}
|
||||
c.set(merged)
|
||||
*c = merged
|
||||
return
|
||||
}
|
||||
for _, sample := range samples {
|
||||
if c.count == 0 {
|
||||
c.data = appendVarint(c.data, sample.timestamp)
|
||||
} else {
|
||||
c.data = appendUvarint(c.data, uint64(sample.timestamp-c.lastStamp))
|
||||
}
|
||||
c.data = appendRawSample(c.data, sample)
|
||||
c.count++
|
||||
c.lastStamp = sample.timestamp
|
||||
}
|
||||
}
|
||||
|
||||
func (c *compressedRawSamples) decode() []rawSample {
|
||||
series := rawSeries{compressed: *c}
|
||||
return series.decodeCompressed()
|
||||
}
|
||||
|
||||
func (c *compressedRawSamples) set(samples []rawSample) {
|
||||
c.data = nil
|
||||
c.count = 0
|
||||
c.lastStamp = 0
|
||||
for _, sample := range samples {
|
||||
if c.count == 0 {
|
||||
c.data = appendVarint(c.data, sample.timestamp)
|
||||
} else {
|
||||
c.data = appendUvarint(c.data, uint64(sample.timestamp-c.lastStamp))
|
||||
}
|
||||
c.data = appendRawSample(c.data, sample)
|
||||
c.count++
|
||||
c.lastStamp = sample.timestamp
|
||||
c.appendSample(sample)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -534,9 +664,12 @@ func compactRawLabels(series *rawSeries) {
|
||||
if len(series.labelHashes) <= 1 {
|
||||
return
|
||||
}
|
||||
all := series.allSamples()
|
||||
used := make([]bool, len(series.labelHashes))
|
||||
for _, sample := range all {
|
||||
decoder := newRawSampleDecoder(series.compressed)
|
||||
for sample, more := decoder.next(); more; sample, more = decoder.next() {
|
||||
used[sample.labelID] = true
|
||||
}
|
||||
for _, sample := range series.samples {
|
||||
used[sample.labelID] = true
|
||||
}
|
||||
usedCount := 0
|
||||
@@ -563,11 +696,13 @@ func compactRawLabels(series *rawSeries) {
|
||||
labelsJSON = append(labelsJSON, series.labelsJSON[oldID])
|
||||
labelIDs[hash] = newID
|
||||
}
|
||||
compressed := series.decodeCompressed()
|
||||
for i := range compressed {
|
||||
compressed[i].labelID = remap[compressed[i].labelID]
|
||||
decoder = newRawSampleDecoder(series.compressed)
|
||||
compressed := newCompressedRawSamples(len(series.compressed.data))
|
||||
for sample, more := decoder.next(); more; sample, more = decoder.next() {
|
||||
sample.labelID = remap[sample.labelID]
|
||||
compressed.appendSample(sample)
|
||||
}
|
||||
series.compressed.set(compressed)
|
||||
series.compressed = compressed
|
||||
for i := range series.samples {
|
||||
series.samples[i].labelID = remap[series.samples[i].labelID]
|
||||
}
|
||||
|
||||
@@ -124,14 +124,121 @@ func TestCompressedRawSamplesRoundTripExactly(t *testing.T) {
|
||||
}
|
||||
var compressed compressedRawSamples
|
||||
compressed.append(samples)
|
||||
decoded := compressed.decode()
|
||||
if len(decoded) != len(samples) {
|
||||
t.Fatalf("decoded sample count = %d, want %d", len(decoded), len(samples))
|
||||
}
|
||||
decoder := newRawSampleDecoder(compressed)
|
||||
for i := range samples {
|
||||
if decoded[i].timestamp != samples[i].timestamp || decoded[i].labelID != samples[i].labelID || math.Float64bits(decoded[i].value) != math.Float64bits(samples[i].value) {
|
||||
t.Fatalf("decoded sample %d = %#v, want %#v", i, decoded[i], samples[i])
|
||||
decoded, more := decoder.next()
|
||||
if !more {
|
||||
t.Fatalf("decoded sample count = %d, want %d", i, len(samples))
|
||||
}
|
||||
if decoded.timestamp != samples[i].timestamp || decoded.labelID != samples[i].labelID || math.Float64bits(decoded.value) != math.Float64bits(samples[i].value) {
|
||||
t.Fatalf("decoded sample %d = %#v, want %#v", i, decoded, samples[i])
|
||||
}
|
||||
}
|
||||
if _, more := decoder.next(); more {
|
||||
t.Fatal("decoder returned extra sample")
|
||||
}
|
||||
}
|
||||
|
||||
func TestCompressedRawSamplesMergeInsertionAndReplacement(t *testing.T) {
|
||||
base := time.Now().UTC().Add(-5 * time.Minute).UnixMilli()
|
||||
var compressed compressedRawSamples
|
||||
compressed.append([]rawSample{
|
||||
{timestamp: base, value: 1, labelID: 0},
|
||||
{timestamp: base + 2000, value: 3, labelID: 2},
|
||||
})
|
||||
compressed.append([]rawSample{
|
||||
{timestamp: base + 1000, value: math.Copysign(0, -1), labelID: 1},
|
||||
{timestamp: base + 2000, value: 33, labelID: 3},
|
||||
})
|
||||
want := []rawSample{
|
||||
{timestamp: base, value: 1, labelID: 0},
|
||||
{timestamp: base + 1000, value: math.Copysign(0, -1), labelID: 1},
|
||||
{timestamp: base + 2000, value: 33, labelID: 3},
|
||||
}
|
||||
decoder := newRawSampleDecoder(compressed)
|
||||
for i := range want {
|
||||
got, more := decoder.next()
|
||||
if !more || got.timestamp != want[i].timestamp || got.labelID != want[i].labelID || math.Float64bits(got.value) != math.Float64bits(want[i].value) {
|
||||
t.Fatalf("sample %d = %#v, want %#v", i, got, want[i])
|
||||
}
|
||||
}
|
||||
if _, more := decoder.next(); more {
|
||||
t.Fatal("decoder returned extra merged sample")
|
||||
}
|
||||
}
|
||||
|
||||
func TestRawQueryCompressedDirectBoundaryClosedRange(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
s := newMemStore(t)
|
||||
if err := s.CreateMetric(ctx, Definition{Name: "raw-boundary", RetentionDays: 1}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
now := time.Now().UTC().Truncate(time.Millisecond)
|
||||
compressedAt := now.Add(-time.Minute)
|
||||
directAt := compressedAt.Add(time.Millisecond)
|
||||
series := &rawSeries{
|
||||
tagsJSON: "{}", labelIDs: map[string]uint32{emptyLabelsHash: 0},
|
||||
labelHashes: []string{emptyLabelsHash}, labelsJSON: []string{"{}"},
|
||||
samples: []rawSample{{timestamp: directAt.UnixMilli(), value: 2}},
|
||||
}
|
||||
series.compressed.append([]rawSample{{timestamp: compressedAt.UnixMilli(), value: 1}})
|
||||
s.raw[rawSeriesKey{metricName: "raw-boundary", entityID: "n1", tagsHash: emptyLabelsHash}] = series
|
||||
|
||||
ascending, err := s.Query(ctx, Query{MetricName: "raw-boundary", Start: compressedAt, End: directAt, Order: OrderAsc})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(ascending) != 2 || ascending[0].Value != 1 || ascending[1].Value != 2 {
|
||||
t.Fatalf("closed ascending range = %#v", ascending)
|
||||
}
|
||||
descending, err := s.Query(ctx, Query{MetricName: "raw-boundary", Start: compressedAt, End: directAt, Order: OrderDesc, Offset: 1, Limit: 1})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(descending) != 1 || descending[0].Value != 1 {
|
||||
t.Fatalf("paged descending range = %#v", descending)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRawPruneStreamsAndReclaimsLabels(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
s := newMemStore(t)
|
||||
if err := s.CreateMetric(ctx, Definition{Name: "raw-prune", RetentionDays: 1}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
now := time.Now().UTC().Truncate(time.Millisecond)
|
||||
cutoff := now.Add(-3 * time.Minute)
|
||||
series := &rawSeries{
|
||||
tagsJSON: "{}",
|
||||
labelIDs: map[string]uint32{"old": 0, "middle": 1, "keep": 2},
|
||||
labelHashes: []string{"old", "middle", "keep"},
|
||||
labelsJSON: []string{`{"label":"old"}`, `{"label":"middle"}`, `{"label":"keep"}`},
|
||||
samples: []rawSample{{timestamp: now.Add(-30 * time.Second).UnixMilli(), value: 4, labelID: 2}},
|
||||
}
|
||||
series.compressed.append([]rawSample{
|
||||
{timestamp: now.Add(-5 * time.Minute).UnixMilli(), value: 1, labelID: 0},
|
||||
{timestamp: now.Add(-4 * time.Minute).UnixMilli(), value: 2, labelID: 1},
|
||||
{timestamp: cutoff.UnixMilli(), value: 3, labelID: 2},
|
||||
})
|
||||
s.raw[rawSeriesKey{metricName: "raw-prune", entityID: "n1", tagsHash: emptyLabelsHash}] = series
|
||||
|
||||
if deleted := s.deleteRawBefore("raw-prune", cutoff.UnixMilli()); deleted != 2 {
|
||||
t.Fatalf("deleted %d samples, want 2", deleted)
|
||||
}
|
||||
if len(series.labelsJSON) != 1 || series.labelsJSON[0] != `{"label":"keep"}` || series.compressed.count != 1 || series.samples[0].labelID != 0 {
|
||||
t.Fatalf("compacted series = %#v", series)
|
||||
}
|
||||
decoder := newRawSampleDecoder(series.compressed)
|
||||
kept, more := decoder.next()
|
||||
if !more || kept.timestamp != cutoff.UnixMilli() || kept.labelID != 0 || kept.value != 3 {
|
||||
t.Fatalf("kept compressed sample = %#v", kept)
|
||||
}
|
||||
points, err := s.Query(ctx, Query{MetricName: "raw-prune", Start: cutoff, End: now})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(points) != 2 || points[0].Labels["label"] != "keep" || points[1].Labels["label"] != "keep" {
|
||||
t.Fatalf("pruned query = %#v", points)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -832,7 +832,7 @@ func TestValidateNormalizedRestructureRejectsMissingDigest(t *testing.T) {
|
||||
if err := s.CreateMetric(ctx, Definition{Name: "invalid", Type: TypeGauge, RetentionDays: 1}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
at := time.Now().UTC().Add(-2 * time.Minute)
|
||||
at := time.Now().UTC().Truncate(time.Minute).Add(-2 * time.Minute)
|
||||
if err := s.WriteBatch(ctx, []Point{
|
||||
{MetricName: "invalid", EntityID: "n1", Timestamp: at, Value: 1},
|
||||
{MetricName: "invalid", EntityID: "n1", Timestamp: at.Add(time.Second), Value: 2},
|
||||
@@ -857,7 +857,7 @@ func TestValidateNormalizedRestructureRejectsWrongDigestCompression(t *testing.T
|
||||
if err := s.CreateMetric(ctx, Definition{Name: "invalid-compression", Type: TypeGauge, RetentionDays: 1}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
at := time.Now().UTC().Add(-2 * time.Minute)
|
||||
at := time.Now().UTC().Truncate(time.Minute).Add(-2 * time.Minute)
|
||||
if err := s.WriteBatch(ctx, []Point{
|
||||
{MetricName: "invalid-compression", EntityID: "n1", Timestamp: at, Value: 1},
|
||||
{MetricName: "invalid-compression", EntityID: "n1", Timestamp: at.Add(time.Second), Value: 2},
|
||||
@@ -929,7 +929,6 @@ func TestRestructureDump(t *testing.T) {
|
||||
if err := s.ReclaimSpace(ctx); err != nil {
|
||||
t.Fatalf("reclaim rebuilt store: %v", err)
|
||||
}
|
||||
assertRestructuredDumpQueryContinuity(t, ctx, s)
|
||||
after, err := s.StorageSize(ctx)
|
||||
if err != nil {
|
||||
t.Fatalf("measure rebuilt storage: %v", err)
|
||||
@@ -944,85 +943,3 @@ func TestRestructureDump(t *testing.T) {
|
||||
}
|
||||
t.Logf("restructured rows=%d metrics=%d before=%d after=%d saved=%d percent=%.2f", result.RowsCopied, result.Metrics, before, after, saved, percent)
|
||||
}
|
||||
|
||||
func assertRestructuredDumpQueryContinuity(t *testing.T, ctx context.Context, s *Store) {
|
||||
t.Helper()
|
||||
var metricName, entityID string
|
||||
var latestMilli int64
|
||||
err := s.db.QueryRowContext(ctx, fmt.Sprintf(`SELECT series.metric_name, series.entity_id,
|
||||
(SELECT MAX(recent.last_ts_milli) FROM %s recent
|
||||
JOIN %s recent_series ON recent_series.id = recent.series_id
|
||||
WHERE recent_series.metric_name = series.metric_name AND recent_series.entity_id = series.entity_id)
|
||||
FROM %s daily
|
||||
JOIN %s series ON series.id = daily.series_id
|
||||
JOIN %s daily_resolution ON daily_resolution.id = daily.resolution_id
|
||||
WHERE daily_resolution.resolution_milli = ?
|
||||
GROUP BY series.metric_name, series.entity_id
|
||||
ORDER BY COUNT(DISTINCT daily.bucket_milli) DESC, series.metric_name, series.entity_id LIMIT 1`,
|
||||
s.tables.rollups, s.tables.series,
|
||||
s.tables.rollups, s.tables.series, s.tables.resolutions),
|
||||
(24*time.Hour).Milliseconds()).Scan(&metricName, &entityID, &latestMilli)
|
||||
if err != nil {
|
||||
t.Fatalf("select migrated series: %v", err)
|
||||
}
|
||||
now := fromMillis(latestMilli).Add(time.Millisecond)
|
||||
for _, duration := range []time.Duration{time.Hour, 6 * time.Hour, 7 * 24 * time.Hour, 30 * 24 * time.Hour, 60 * 24 * time.Hour} {
|
||||
start := now.Add(-duration)
|
||||
interval := s.CompatibleSeriesInterval(start, now, time.Minute)
|
||||
resolution, err := s.coveringSeriesResolution(ctx, Query{MetricName: metricName, EntityID: entityID, Start: start, End: now}, interval)
|
||||
if err != nil {
|
||||
t.Fatalf("select %s backing tier: %v", duration, err)
|
||||
}
|
||||
series, err := s.Series(ctx, AggregateQuery{
|
||||
Query: Query{MetricName: metricName, EntityID: entityID, Start: start, End: now},
|
||||
Aggregation: AggAvg,
|
||||
Interval: interval,
|
||||
}, now)
|
||||
if err != nil {
|
||||
t.Fatalf("query %s: %v", duration, err)
|
||||
}
|
||||
if len(series) == 0 || series[len(series)-1].Count == 0 {
|
||||
t.Fatalf("query %s has no final data bucket: %#v", duration, series)
|
||||
}
|
||||
expectedStep := max(interval, resolution)
|
||||
maxGap := time.Duration(0)
|
||||
for i := 1; i < len(series); i++ {
|
||||
gap := series[i].Bucket.Sub(series[i-1].Bucket)
|
||||
if gap > maxGap {
|
||||
maxGap = gap
|
||||
}
|
||||
}
|
||||
if maxGap > 2*expectedStep {
|
||||
t.Fatalf("query %s has a %s discontinuity larger than two %s buckets", duration, maxGap, expectedStep)
|
||||
}
|
||||
tailGap := now.Sub(series[len(series)-1].Bucket)
|
||||
if tailGap < 0 || tailGap > 2*expectedStep {
|
||||
t.Fatalf("query %s final data is %s from the query end, expected at most %s", duration, tailGap, 2*expectedStep)
|
||||
}
|
||||
t.Logf("range=%s metric=%s entity=%s requested=%s backing=%s points=%d first=%s last=%s max_gap=%s tail_gap=%s",
|
||||
duration, metricName, entityID, interval, resolution, len(series), series[0].Bucket, series[len(series)-1].Bucket, maxGap, tailGap)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRestructuredDumpQueryContinuity(t *testing.T) {
|
||||
path := os.Getenv("KOMARI_METRIC_RESTRUCTURE_RESULT")
|
||||
if path == "" {
|
||||
t.Skip("set KOMARI_METRIC_RESTRUCTURE_RESULT to validate a rebuilt dump")
|
||||
}
|
||||
ctx := context.Background()
|
||||
s, err := Open(ctx, SQLite(path, WithAutoMigrate(false), WithMaxOpenConns(1), WithRollupPolicy(RollupPolicy{
|
||||
RawRetention: 15 * time.Minute,
|
||||
Tiers: []RollupTier{
|
||||
{Interval: time.Minute, Retention: 600 * time.Minute},
|
||||
{Interval: 5 * time.Minute, Retention: 600 * 5 * time.Minute},
|
||||
{Interval: time.Hour, Retention: 600 * time.Hour},
|
||||
{Interval: 24 * time.Hour, Retention: 100 * 365 * 24 * time.Hour},
|
||||
},
|
||||
Compression: 30,
|
||||
})))
|
||||
if err != nil {
|
||||
t.Fatalf("open rebuilt dump: %v", err)
|
||||
}
|
||||
defer s.Close()
|
||||
assertRestructuredDumpQueryContinuity(t, ctx, s)
|
||||
}
|
||||
@@ -348,12 +348,10 @@ func TestSeriesHybridIncludesPartialCoarseBucketAtRawCutoff(t *testing.T) {
|
||||
now := time.Date(2026, 6, 18, 12, 50, 0, 0, time.UTC)
|
||||
old := now.Add(-40 * time.Minute)
|
||||
recent := now.Add(-5 * time.Minute)
|
||||
if err := s.WriteBatch(ctx, []Point{
|
||||
{MetricName: "coarse-cutoff", EntityID: "n1", Timestamp: old, Value: 10},
|
||||
{MetricName: "coarse-cutoff", EntityID: "n1", Timestamp: recent, Value: 100},
|
||||
}); err != nil {
|
||||
t.Fatalf("write: %v", err)
|
||||
}
|
||||
writeRollupPointsAt(t, ctx, s, now,
|
||||
Point{MetricName: "coarse-cutoff", EntityID: "n1", Timestamp: old, Value: 10},
|
||||
Point{MetricName: "coarse-cutoff", EntityID: "n1", Timestamp: recent, Value: 100},
|
||||
)
|
||||
if _, err := s.Compact(ctx, now); err != nil {
|
||||
t.Fatalf("compact: %v", err)
|
||||
}
|
||||
@@ -367,7 +365,7 @@ func TestSeriesHybridIncludesPartialCoarseBucketAtRawCutoff(t *testing.T) {
|
||||
t.Fatalf("series: %v", err)
|
||||
}
|
||||
if len(got) != 1 || got[0].Count != 2 || math.Abs(got[0].Value-55) > 1e-9 {
|
||||
t.Fatalf("expected coarse rollup and recent raw point, got %#v", got)
|
||||
t.Fatalf("expected persisted and in-memory hourly data, got %#v", got)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -411,7 +409,9 @@ func TestSeriesAndAggregateRollupBridgeSealedCoarseTail(t *testing.T) {
|
||||
read func() ([]AggregatePoint, error)
|
||||
}{
|
||||
{name: "series", read: func() ([]AggregatePoint, error) { return s.Series(ctx, query, base.Add(16*time.Minute)) }},
|
||||
{name: "aggregate", read: func() ([]AggregatePoint, error) { return s.aggregateRollupAt(ctx, query, 5*time.Minute, base.Add(16*time.Minute)) }},
|
||||
{name: "aggregate", read: func() ([]AggregatePoint, error) {
|
||||
return s.aggregateRollupAt(ctx, query, 5*time.Minute, base.Add(16*time.Minute))
|
||||
}},
|
||||
} {
|
||||
points, err := result.read()
|
||||
if err != nil {
|
||||
|
||||
+193
-353
@@ -7,292 +7,118 @@ import (
|
||||
"time"
|
||||
)
|
||||
|
||||
// AggregateRollup reduces one materialized tier into the requested output
|
||||
// interval. The hot minute is folded in separately so recent charts never wait
|
||||
// for another report before their final real bucket becomes visible.
|
||||
// AggregateRollup reduces one explicitly selected materialized tier into the
|
||||
// requested output interval.
|
||||
func (s *Store) AggregateRollup(ctx context.Context, query AggregateQuery, resolution time.Duration) ([]AggregatePoint, error) {
|
||||
return s.aggregateRollupAt(ctx, query, resolution, time.Now().UTC())
|
||||
}
|
||||
|
||||
// aggregateRollupAt reads a sealed rollup prefix at the requested backing
|
||||
// resolution, then recursively falls back through finer tiers for the
|
||||
// ten-minute late-arrival tail. Series passes its supplied now value here so
|
||||
// callers receive a stable view while AggregateRollup keeps its public API.
|
||||
func (s *Store) aggregateRollupAt(ctx context.Context, query AggregateQuery, resolution time.Duration, now time.Time) ([]AggregatePoint, error) {
|
||||
if err := s.ensureOpen(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := query.Validate(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if resolution <= 0 {
|
||||
return nil, fmt.Errorf("%w: rollup resolution must be positive", ErrInvalidArgument)
|
||||
}
|
||||
s.rollupViewMu.RLock()
|
||||
defer s.rollupViewMu.RUnlock()
|
||||
q := query.Query.normalized()
|
||||
needDigest := isPercentile(query.Aggregation)
|
||||
policy, err := s.metricRollupPolicy(ctx, q.MetricName)
|
||||
entityIDs := []string(nil)
|
||||
if query.EntityID != "" {
|
||||
entityIDs = []string{query.EntityID}
|
||||
}
|
||||
batch := BatchSeriesQuery{
|
||||
Specs: []BatchSeriesSpec{{
|
||||
MetricName: query.MetricName,
|
||||
Aggregations: []Aggregation{query.Aggregation},
|
||||
Interval: query.Interval,
|
||||
PreserveSeries: query.PreserveSeries,
|
||||
}},
|
||||
EntityIDs: entityIDs,
|
||||
Start: query.Start,
|
||||
End: query.End,
|
||||
Tags: query.Tags,
|
||||
Order: query.Order,
|
||||
}
|
||||
if err := batch.Validate(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
result, err := s.seriesBatchAt(ctx, batch.normalized(), now.UTC(), map[string]time.Duration{query.MetricName: resolution})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
rows, err := s.hybridRollupRows(ctx, q, policy, resolution, now.UTC(), needDigest)
|
||||
return pageBuckets(result.Values[query.MetricName][query.Aggregation], query.BucketLimit, query.BucketOffset), nil
|
||||
}
|
||||
|
||||
// Series selects one deterministic backing tier and delegates to SeriesBatch.
|
||||
func (s *Store) Series(ctx context.Context, query AggregateQuery, now time.Time) ([]AggregatePoint, error) {
|
||||
if err := s.ensureOpen(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := query.Validate(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
entityIDs := []string(nil)
|
||||
if query.EntityID != "" {
|
||||
entityIDs = []string{query.EntityID}
|
||||
}
|
||||
result, err := s.SeriesBatch(ctx, BatchSeriesQuery{
|
||||
Specs: []BatchSeriesSpec{{
|
||||
MetricName: query.MetricName,
|
||||
Aggregations: []Aggregation{query.Aggregation},
|
||||
Interval: query.Interval,
|
||||
PreserveSeries: query.PreserveSeries,
|
||||
}},
|
||||
EntityIDs: entityIDs,
|
||||
Start: query.Start,
|
||||
End: query.End,
|
||||
Tags: query.Tags,
|
||||
Order: query.Order,
|
||||
}, now)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if query.Aggregation == AggRate {
|
||||
points := representativePoints(q.MetricName, rows)
|
||||
return pageBuckets(mustAggregatePoints(points, query), query.BucketLimit, query.BucketOffset), nil
|
||||
return pageBuckets(result.Values[query.MetricName][query.Aggregation], query.BucketLimit, query.BucketOffset), nil
|
||||
}
|
||||
|
||||
// SeriesAggregates derives multiple aggregations from the same streamed scan.
|
||||
func (s *Store) SeriesAggregates(ctx context.Context, query Query, aggregations []Aggregation, interval time.Duration, preserveSeries bool, now time.Time) (map[Aggregation][]AggregatePoint, error) {
|
||||
entityIDs := []string(nil)
|
||||
if query.EntityID != "" {
|
||||
entityIDs = []string{query.EntityID}
|
||||
}
|
||||
groups := foldRollupRows(nil, rows, query.Interval, s.cfg.RollupPolicy.compression(), query.PreserveSeries, false, needDigest)
|
||||
result, err := rollupGroupsToPoints(groups, query)
|
||||
result, err := s.SeriesBatch(ctx, BatchSeriesQuery{
|
||||
Specs: []BatchSeriesSpec{{
|
||||
MetricName: query.MetricName,
|
||||
Aggregations: aggregations,
|
||||
Interval: interval,
|
||||
PreserveSeries: preserveSeries,
|
||||
}},
|
||||
EntityIDs: entityIDs,
|
||||
Start: query.Start,
|
||||
End: query.End,
|
||||
Tags: query.Tags,
|
||||
Order: query.Order,
|
||||
}, now)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return pageBuckets(result, query.BucketLimit, query.BucketOffset), nil
|
||||
return result.Values[query.MetricName], nil
|
||||
}
|
||||
|
||||
// hybridRollupRows reads each part of a range once. A parent bucket becomes
|
||||
// eligible only after its end is outside coarseRollupGrace; the remainder is
|
||||
// represented by the immediate finer tier until minute buckets, where hot
|
||||
// in-memory observations complete the view.
|
||||
func (s *Store) hybridRollupRows(ctx context.Context, query Query, policy RollupPolicy, resolution time.Duration, now time.Time, needDigest bool) ([]storedRollup, error) {
|
||||
if len(policy.Tiers) == 0 {
|
||||
return nil, nil
|
||||
func seriesResolutionForPolicy(start time.Time, interval time.Duration, now time.Time, policy RollupPolicy) time.Duration {
|
||||
preferred := policy.Tiers[0].Interval
|
||||
if tier := bestRollupTier(policy, interval, start.UTC(), now.UTC()); tier != nil {
|
||||
return tier.Interval
|
||||
}
|
||||
index := -1
|
||||
for i, tier := range policy.Tiers {
|
||||
if tier.Interval <= resolution && resolution%tier.Interval == 0 {
|
||||
index = i
|
||||
for i := len(policy.Tiers) - 1; i >= 0; i-- {
|
||||
tier := policy.Tiers[i]
|
||||
if interval >= tier.Interval && interval%tier.Interval == 0 {
|
||||
preferred = tier.Interval
|
||||
break
|
||||
}
|
||||
}
|
||||
if index < 0 {
|
||||
return s.scanRollupRowsBetween(ctx, query.MetricName, query.EntityID, query.Tags, resolution.Milliseconds(), bucketStartMillis(query.Start.UnixMilli(), resolution.Milliseconds()), query.End.UnixMilli(), needDigest)
|
||||
}
|
||||
return s.hybridRollupRowsAtTier(ctx, query, policy, index, query.Start.UTC().UnixMilli(), query.End.UTC().UnixMilli(), now, needDigest)
|
||||
}
|
||||
|
||||
func (s *Store) hybridRollupRowsAtTier(ctx context.Context, query Query, policy RollupPolicy, index int, startMilli, endMilli int64, now time.Time, needDigest bool) ([]storedRollup, error) {
|
||||
if startMilli > endMilli {
|
||||
return nil, nil
|
||||
}
|
||||
tier := policy.Tiers[index]
|
||||
intervalMilli := tier.Interval.Milliseconds()
|
||||
// The greatest bucket start whose end is at least coarseRollupGrace behind
|
||||
// now. Buckets after it are still mutable in the in-memory parent map.
|
||||
sealedThrough := bucketStartMillis(now.Add(-coarseRollupGrace-tier.Interval).UnixMilli(), intervalMilli)
|
||||
rows := make([]storedRollup, 0)
|
||||
if startMilli <= sealedThrough {
|
||||
upper := sealedThrough
|
||||
if upper > endMilli {
|
||||
upper = endMilli
|
||||
}
|
||||
sealed, err := s.scanRollupRowsBetween(ctx, query.MetricName, query.EntityID, query.Tags, intervalMilli, bucketStartMillis(startMilli, intervalMilli), upper, needDigest)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
rows = append(rows, sealed...)
|
||||
}
|
||||
|
||||
tailStart := startMilli
|
||||
if tailStart <= sealedThrough {
|
||||
tailStart = sealedThrough + intervalMilli
|
||||
}
|
||||
if tailStart > endMilli {
|
||||
return rows, nil
|
||||
}
|
||||
if index == 0 {
|
||||
minute, err := s.scanRollupRowsBetween(ctx, query.MetricName, query.EntityID, query.Tags, intervalMilli, bucketStartMillis(tailStart, intervalMilli), endMilli, needDigest)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
hot, err := s.hotRollupRows(query.MetricName, query.EntityID, query.Tags, fromMillis(tailStart), fromMillis(endMilli), needDigest)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return append(rows, append(minute, hot...)...), nil
|
||||
}
|
||||
tail, err := s.hybridRollupRowsAtTier(ctx, query, policy, index-1, tailStart, endMilli, now, needDigest)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return append(rows, tail...), nil
|
||||
}
|
||||
|
||||
func mustAggregatePoints(points []Point, query AggregateQuery) []AggregatePoint {
|
||||
result, err := AggregatePoints(points, query)
|
||||
if err != nil {
|
||||
return []AggregatePoint{}
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
func representativePoints(metricName string, rows []storedRollup) []Point {
|
||||
points := make([]Point, 0, len(rows))
|
||||
for _, row := range rows {
|
||||
tags, err := rollupTagsFromJSON(row.bucketData.tagsJSON)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
labels, err := rollupTagsFromJSON(row.bucketData.labelsJSON)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
points = append(points, Point{MetricName: metricName, EntityID: row.entityID, Timestamp: fromMillis(row.bucketData.lastTS), Value: row.bucketData.lastVal, Tags: tags, Labels: labels})
|
||||
}
|
||||
return points
|
||||
}
|
||||
|
||||
func foldRollupRows(groups map[rollupKey]*rollupBucket, rows []storedRollup, interval time.Duration, compression float64, preserveSeries, preserveLabels, needDigest bool) map[rollupKey]*rollupBucket {
|
||||
if groups == nil {
|
||||
groups = make(map[rollupKey]*rollupBucket)
|
||||
}
|
||||
for _, row := range rows {
|
||||
key := rollupKey{bucket: bucketStartMillis(row.bucket, interval.Milliseconds())}
|
||||
if preserveSeries {
|
||||
key.entityID, key.tagsHash = row.entityID, row.bucketData.tagsHash
|
||||
if preserveLabels {
|
||||
key.labelsHash = row.bucketData.labelsHash
|
||||
}
|
||||
}
|
||||
bucket := groups[key]
|
||||
if bucket == nil {
|
||||
bucket = newRollupBucketWithDigest(compression, needDigest)
|
||||
bucket.tagsHash, bucket.tagsJSON = row.bucketData.tagsHash, row.bucketData.tagsJSON
|
||||
bucket.labelsHash, bucket.labelsJSON = row.bucketData.labelsHash, row.bucketData.labelsJSON
|
||||
groups[key] = bucket
|
||||
}
|
||||
bucket.mergeStored(row.bucketData)
|
||||
}
|
||||
return groups
|
||||
}
|
||||
|
||||
func rollupGroupsToPoints(groups map[rollupKey]*rollupBucket, query AggregateQuery) ([]AggregatePoint, error) {
|
||||
keys := make([]rollupKey, 0, len(groups))
|
||||
for key := range groups {
|
||||
keys = append(keys, key)
|
||||
}
|
||||
sortRollupKeys(keys)
|
||||
result := make([]AggregatePoint, 0, len(keys))
|
||||
for _, key := range keys {
|
||||
bucket := groups[key]
|
||||
value, ok := bucket.value(query.Aggregation)
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("%w: aggregation %q requires raw samples", ErrInvalidArgument, query.Aggregation)
|
||||
}
|
||||
tags, err := rollupTagsFromJSON(bucket.tagsJSON)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if !query.PreserveSeries {
|
||||
tags = map[string]string{}
|
||||
}
|
||||
result = append(result, AggregatePoint{MetricName: query.MetricName, EntityID: key.entityID, Bucket: fromMillis(key.bucket), Value: value, Count: int(bucket.count), Tags: tags})
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func (s *Store) scanRollupRowsBetween(ctx context.Context, metricName, entityID string, tags map[string]string, resolutionMilli, lowerBucket, upperBucket int64, needDigest bool) ([]storedRollup, error) {
|
||||
return s.scanRollupRowsBetweenWith(ctx, s.reader(), metricName, entityID, tags, time.Duration(resolutionMilli)*time.Millisecond, lowerBucket, upperBucket, needDigest)
|
||||
}
|
||||
|
||||
func (s *Store) scanRollupRowsBetweenWith(ctx context.Context, q querier, metricName, entityID string, tags map[string]string, resolution time.Duration, lowerBucket, upperBucket int64, needDigest bool) ([]storedRollup, error) {
|
||||
args := []any{metricName, resolution.Milliseconds(), lowerBucket, upperBucket}
|
||||
parts := []string{
|
||||
"s.metric_name = " + s.dialect.placeholder(1),
|
||||
"d.resolution_milli = " + s.dialect.placeholder(2),
|
||||
"r.bucket_milli >= " + s.dialect.placeholder(3),
|
||||
"r.bucket_milli <= " + s.dialect.placeholder(4),
|
||||
}
|
||||
if entityID != "" {
|
||||
args = append(args, entityID)
|
||||
parts = append(parts, "s.entity_id = "+s.dialect.placeholder(len(args)))
|
||||
}
|
||||
for _, key := range sortedKeys(tags) {
|
||||
args = append(args, tags[key])
|
||||
parts = append(parts, s.dialect.jsonExtractEquals("s.tags", key, s.dialect.placeholder(len(args))))
|
||||
}
|
||||
columns := "s.entity_id, s.tags_hash, s.tags, l.labels_hash, l.labels, r.bucket_milli, r.count, r.sum, r.sum_sq, r.min_val, r.max_val, r.first_val, r.first_ts_milli, r.last_val, r.last_ts_milli"
|
||||
if needDigest {
|
||||
columns += ", r.digest"
|
||||
}
|
||||
sqlText := fmt.Sprintf("SELECT %s FROM %s r JOIN %s s ON s.id = r.series_id JOIN %s d ON d.id = r.resolution_id JOIN %s l ON l.id = r.label_id WHERE %s ORDER BY r.bucket_milli ASC", columns, s.tables.rollups, s.tables.series, s.tables.resolutions, s.tables.labels, joinSQLWith(parts, " AND "))
|
||||
rows, err := q.QueryContext(ctx, sqlText, args...)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
return scanStoredRollups(rows, needDigest, s.cfg.RollupPolicy.compression())
|
||||
}
|
||||
|
||||
func joinSQLWith(parts []string, separator string) string {
|
||||
if len(parts) == 0 {
|
||||
return ""
|
||||
}
|
||||
out := parts[0]
|
||||
for _, part := range parts[1:] {
|
||||
out += separator + part
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func scanStoredRollups(rows *sql.Rows, needDigest bool, compression float64) ([]storedRollup, error) {
|
||||
result := make([]storedRollup, 0)
|
||||
for rows.Next() {
|
||||
var entityID, tagsHash, labelsHash string
|
||||
var rawTags, rawLabels any
|
||||
var bucket, count, firstTS, lastTS int64
|
||||
var sum, sumSq, min, max, firstVal, lastVal float64
|
||||
var digest []byte
|
||||
args := []any{&entityID, &tagsHash, &rawTags, &labelsHash, &rawLabels, &bucket, &count, &sum, &sumSq, &min, &max, &firstVal, &firstTS, &lastVal, &lastTS}
|
||||
if needDigest {
|
||||
args = append(args, &digest)
|
||||
}
|
||||
if err := rows.Scan(args...); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
tagsJSON, err := rawJSONToString(rawTags)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
labelsJSON, err := rawJSONToString(rawLabels)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var d *TDigest
|
||||
if needDigest {
|
||||
d, err = digestFromRollup(count, min, max, digest, compression)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
result = append(result, storedRollup{entityID: entityID, bucket: bucket, bucketData: &rollupBucket{count: count, sum: sum, sumSq: sumSq, min: min, max: max, firstVal: firstVal, firstTS: firstTS, lastVal: lastVal, lastTS: lastTS, digest: d, tagsHash: tagsHash, tagsJSON: tagsJSON, labelsHash: labelsHash, labelsJSON: labelsJSON}})
|
||||
}
|
||||
return result, rows.Err()
|
||||
}
|
||||
|
||||
func rawJSONToString(value any) (string, error) {
|
||||
switch v := value.(type) {
|
||||
case nil:
|
||||
return "{}", nil
|
||||
case string:
|
||||
if v == "" {
|
||||
return "{}", nil
|
||||
}
|
||||
return v, nil
|
||||
case []byte:
|
||||
if len(v) == 0 {
|
||||
return "{}", nil
|
||||
}
|
||||
return string(v), nil
|
||||
default:
|
||||
return "", fmt.Errorf("unsupported JSON column type %T", value)
|
||||
}
|
||||
}
|
||||
|
||||
func rollupTagsFromJSON(raw string) (map[string]string, error) {
|
||||
values, err := decodeMapString(raw)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return cloneStringMap(values), nil
|
||||
return preferred
|
||||
}
|
||||
|
||||
func (s *Store) CompatibleSeriesInterval(start, now time.Time, interval time.Duration) time.Duration {
|
||||
@@ -324,115 +150,129 @@ func bestRollupTier(policy RollupPolicy, interval time.Duration, start, now time
|
||||
for i := range policy.Tiers {
|
||||
tier := &policy.Tiers[i]
|
||||
if interval >= tier.Interval && interval%tier.Interval == 0 && !now.Add(-tier.Retention).After(start) {
|
||||
// Prefer the coarsest backing tier compatible with the requested
|
||||
// output width. hybridRollupRows fills its unsealed tail from finer
|
||||
// tiers, so this reduces reads without sacrificing recent data.
|
||||
best = tier
|
||||
}
|
||||
}
|
||||
return best
|
||||
}
|
||||
|
||||
func (s *Store) Series(ctx context.Context, query AggregateQuery, now time.Time) ([]AggregatePoint, error) {
|
||||
if err := s.ensureOpen(); err != nil {
|
||||
return nil, err
|
||||
// scanRollupRowsBetween is retained for compaction maintenance and internal
|
||||
// verification. User-facing aggregation goes through SeriesBatch.
|
||||
func (s *Store) scanRollupRowsBetween(ctx context.Context, metricName, entityID string, tags map[string]string, resolutionMilli, lowerBucket, upperBucket int64, needDigest bool) ([]storedRollup, error) {
|
||||
return s.scanRollupRowsBetweenWith(ctx, s.reader(), metricName, entityID, tags, time.Duration(resolutionMilli)*time.Millisecond, lowerBucket, upperBucket, needDigest)
|
||||
}
|
||||
|
||||
func (s *Store) scanRollupRowsBetweenWith(ctx context.Context, q querier, metricName, entityID string, tags map[string]string, resolution time.Duration, lowerBucket, upperBucket int64, needDigest bool) ([]storedRollup, error) {
|
||||
args := []any{metricName, resolution.Milliseconds(), lowerBucket, upperBucket}
|
||||
parts := []string{
|
||||
"s.metric_name = " + s.dialect.placeholder(1),
|
||||
"d.resolution_milli = " + s.dialect.placeholder(2),
|
||||
"r.bucket_milli >= " + s.dialect.placeholder(3),
|
||||
"r.bucket_milli <= " + s.dialect.placeholder(4),
|
||||
}
|
||||
if err := query.Validate(); err != nil {
|
||||
return nil, err
|
||||
if entityID != "" {
|
||||
args = append(args, entityID)
|
||||
parts = append(parts, "s.entity_id = "+s.dialect.placeholder(len(args)))
|
||||
}
|
||||
policy, err := s.metricRollupPolicy(ctx, query.MetricName)
|
||||
for _, key := range sortedKeys(tags) {
|
||||
args = append(args, tags[key])
|
||||
parts = append(parts, s.dialect.jsonExtractEquals("s.tags", key, s.dialect.placeholder(len(args))))
|
||||
}
|
||||
columns := "s.entity_id, s.tags_hash, s.tags, l.labels_hash, l.labels, r.bucket_milli, r.count, r.sum, r.sum_sq, r.min_val, r.max_val, r.first_val, r.first_ts_milli, r.last_val, r.last_ts_milli"
|
||||
if needDigest {
|
||||
columns += ", r.digest"
|
||||
}
|
||||
sqlText := fmt.Sprintf("SELECT %s FROM %s r JOIN %s s ON s.id = r.series_id JOIN %s d ON d.id = r.resolution_id JOIN %s l ON l.id = r.label_id WHERE %s ORDER BY r.bucket_milli ASC", columns, s.tables.rollups, s.tables.series, s.tables.resolutions, s.tables.labels, joinSQLWith(parts, " AND "))
|
||||
rows, err := q.QueryContext(ctx, sqlText, args...)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if len(policy.Tiers) == 0 {
|
||||
return []AggregatePoint{}, nil
|
||||
defer rows.Close()
|
||||
return scanStoredRollupsForMaintenance(rows, needDigest, s.cfg.RollupPolicy.compression())
|
||||
}
|
||||
|
||||
func joinSQLWith(parts []string, separator string) string {
|
||||
if len(parts) == 0 {
|
||||
return ""
|
||||
}
|
||||
preferred := policy.Tiers[0].Interval
|
||||
if tier := bestRollupTier(policy, query.Interval, query.Start.UTC(), now.UTC()); tier != nil {
|
||||
preferred = tier.Interval
|
||||
} else {
|
||||
for i := len(policy.Tiers) - 1; i >= 0; i-- {
|
||||
tier := policy.Tiers[i]
|
||||
if query.Interval >= tier.Interval && query.Interval%tier.Interval == 0 {
|
||||
preferred = tier.Interval
|
||||
break
|
||||
out := parts[0]
|
||||
for _, part := range parts[1:] {
|
||||
out += separator + part
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func scanStoredRollupsForMaintenance(rows *sql.Rows, needDigest bool, compression float64) ([]storedRollup, error) {
|
||||
result := make([]storedRollup, 0)
|
||||
for rows.Next() {
|
||||
var entityID, tagsHash, tagsJSON, labelsHash, labelsJSON string
|
||||
var bucket, count, firstTS, lastTS int64
|
||||
var sum, sumSq, min, max, firstVal, lastVal float64
|
||||
var digest []byte
|
||||
var err error
|
||||
if needDigest {
|
||||
err = rows.Scan(&entityID, &tagsHash, &tagsJSON, &labelsHash, &labelsJSON, &bucket, &count, &sum, &sumSq, &min, &max, &firstVal, &firstTS, &lastVal, &lastTS, &digest)
|
||||
} else {
|
||||
err = rows.Scan(&entityID, &tagsHash, &tagsJSON, &labelsHash, &labelsJSON, &bucket, &count, &sum, &sumSq, &min, &max, &firstVal, &firstTS, &lastVal, &lastTS)
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var decoded *TDigest
|
||||
if needDigest {
|
||||
decoded, err = digestFromRollup(count, min, max, digest, compression)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
result = append(result, storedRollup{entityID: entityID, bucket: bucket, bucketData: &rollupBucket{
|
||||
count: count, sum: sum, sumSq: sumSq, min: min, max: max,
|
||||
firstVal: firstVal, firstTS: firstTS, lastVal: lastVal, lastTS: lastTS,
|
||||
digest: decoded, tagsHash: tagsHash, tagsJSON: tagsJSON, labelsHash: labelsHash, labelsJSON: labelsJSON,
|
||||
}})
|
||||
}
|
||||
resolution, err := s.coveringSeriesResolutionForPolicy(ctx, query.Query.normalized(), preferred, policy)
|
||||
return result, rows.Err()
|
||||
}
|
||||
|
||||
func representativePoints(metricName string, rows []storedRollup) []Point {
|
||||
points := make([]Point, 0, len(rows))
|
||||
for _, row := range rows {
|
||||
tags, err := rollupTagsFromJSON(row.bucketData.tagsJSON)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
labels, err := rollupTagsFromJSON(row.bucketData.labelsJSON)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
points = append(points, Point{MetricName: metricName, EntityID: row.entityID, Timestamp: fromMillis(row.bucketData.lastTS), Value: row.bucketData.lastVal, Tags: tags, Labels: labels})
|
||||
}
|
||||
return points
|
||||
}
|
||||
|
||||
func rawJSONToString(value any) (string, error) {
|
||||
switch v := value.(type) {
|
||||
case nil:
|
||||
return "{}", nil
|
||||
case string:
|
||||
if v == "" {
|
||||
return "{}", nil
|
||||
}
|
||||
return v, nil
|
||||
case []byte:
|
||||
if len(v) == 0 {
|
||||
return "{}", nil
|
||||
}
|
||||
return string(v), nil
|
||||
default:
|
||||
return "", fmt.Errorf("unsupported JSON column type %T", value)
|
||||
}
|
||||
}
|
||||
|
||||
func rollupTagsFromJSON(raw string) (map[string]string, error) {
|
||||
values, err := decodeMapString(raw)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return s.aggregateRollupAt(ctx, query, resolution, now.UTC())
|
||||
}
|
||||
|
||||
// coveringSeriesResolution falls back to a coarser retained tier only when
|
||||
// the preferred tier starts after the requested range. This happens directly
|
||||
// after a legacy migration: historical 5m/hour rollups predate the newly
|
||||
// written 1m tier. Returning the coarser summary preserves the entire query
|
||||
// rather than presenting a false gap at that boundary.
|
||||
func (s *Store) coveringSeriesResolution(ctx context.Context, query Query, preferred time.Duration) (time.Duration, error) {
|
||||
return s.coveringSeriesResolutionForPolicy(ctx, query, preferred, s.cfg.RollupPolicy)
|
||||
}
|
||||
|
||||
func (s *Store) coveringSeriesResolutionForPolicy(ctx context.Context, query Query, preferred time.Duration, policy RollupPolicy) (time.Duration, error) {
|
||||
selected := preferred
|
||||
earliest, latest, found, err := s.rollupTimeBounds(ctx, query, preferred)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
startMilli := query.Start.UTC().UnixMilli()
|
||||
endMilli := query.End.UTC().UnixMilli()
|
||||
if found && earliest <= startMilli && latest >= startMilli {
|
||||
return selected, nil
|
||||
}
|
||||
selectedFound := found && earliest <= endMilli && latest >= startMilli
|
||||
for _, tier := range policy.Tiers {
|
||||
if tier.Interval <= preferred {
|
||||
continue
|
||||
}
|
||||
candidateEarliest, candidateLatest, candidateFound, err := s.rollupTimeBounds(ctx, query, tier.Interval)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
if !candidateFound || candidateEarliest > endMilli || candidateLatest < startMilli {
|
||||
continue
|
||||
}
|
||||
if candidateEarliest <= startMilli && candidateLatest >= startMilli {
|
||||
return tier.Interval, nil
|
||||
}
|
||||
if !selectedFound || candidateEarliest < earliest {
|
||||
selected, earliest, selectedFound = tier.Interval, candidateEarliest, true
|
||||
}
|
||||
}
|
||||
return selected, nil
|
||||
}
|
||||
|
||||
func (s *Store) rollupTimeBounds(ctx context.Context, query Query, interval time.Duration) (int64, int64, bool, error) {
|
||||
args := []any{query.MetricName}
|
||||
seriesParts := []string{"s.metric_name = " + s.dialect.placeholder(1)}
|
||||
if query.EntityID != "" {
|
||||
args = append(args, query.EntityID)
|
||||
seriesParts = append(seriesParts, "s.entity_id = "+s.dialect.placeholder(len(args)))
|
||||
}
|
||||
for _, key := range sortedKeys(query.Tags) {
|
||||
args = append(args, query.Tags[key])
|
||||
seriesParts = append(seriesParts, s.dialect.jsonExtractEquals("s.tags", key, s.dialect.placeholder(len(args))))
|
||||
}
|
||||
args = append(args, interval.Milliseconds())
|
||||
resolutionPlaceholder := s.dialect.placeholder(len(args))
|
||||
args = append(args, query.End.UTC().UnixMilli())
|
||||
endPlaceholder := s.dialect.placeholder(len(args))
|
||||
|
||||
var earliest, latest sql.NullInt64
|
||||
err := s.reader().QueryRowContext(ctx, fmt.Sprintf(`SELECT MIN(r.first_ts_milli), MAX(r.last_ts_milli) FROM %s r
|
||||
WHERE r.series_id IN (SELECT s.id FROM %s s WHERE %s)
|
||||
AND r.resolution_id IN (SELECT d.id FROM %s d WHERE d.resolution_milli = %s)
|
||||
AND r.first_ts_milli <= %s`, s.tables.rollups, s.tables.series, joinSQLWith(seriesParts, " AND "),
|
||||
s.tables.resolutions, resolutionPlaceholder, endPlaceholder),
|
||||
args...).Scan(&earliest, &latest)
|
||||
if err != nil {
|
||||
return 0, 0, false, err
|
||||
}
|
||||
return earliest.Int64, latest.Int64, earliest.Valid && latest.Valid, nil
|
||||
return cloneStringMap(values), nil
|
||||
}
|
||||
@@ -0,0 +1,152 @@
|
||||
package metric
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"strings"
|
||||
)
|
||||
|
||||
type seriesDictionaryPlan struct {
|
||||
MetricNames []string
|
||||
EntityIDs []string
|
||||
Tags map[string]string
|
||||
}
|
||||
|
||||
type rollupReadPlan struct {
|
||||
SeriesIDs []int64
|
||||
ResolutionID int64
|
||||
StartMilli int64
|
||||
EndMilli int64
|
||||
Fields rollupReadFields
|
||||
}
|
||||
|
||||
type rollupReadFields uint16
|
||||
|
||||
const (
|
||||
rollupReadSum rollupReadFields = 1 << iota
|
||||
rollupReadSumSq
|
||||
rollupReadMin
|
||||
rollupReadMax
|
||||
rollupReadFirst
|
||||
rollupReadLast
|
||||
rollupReadDigest
|
||||
)
|
||||
|
||||
type renderedSQL struct {
|
||||
Query string
|
||||
Args []any
|
||||
}
|
||||
|
||||
func (d sqliteDialect) renderSeriesDictionary(tables tables, indexName string, plan seriesDictionaryPlan) renderedSQL {
|
||||
return renderExpandedSeriesDictionary(d, tables, " INDEXED BY "+indexName, plan)
|
||||
}
|
||||
|
||||
func (d mysqlDialect) renderSeriesDictionary(tables tables, indexName string, plan seriesDictionaryPlan) renderedSQL {
|
||||
return renderExpandedSeriesDictionary(d, tables, " FORCE INDEX ("+indexName+")", plan)
|
||||
}
|
||||
|
||||
func (d postgresDialect) renderSeriesDictionary(tables tables, _ string, plan seriesDictionaryPlan) renderedSQL {
|
||||
args := []any{plan.MetricNames}
|
||||
parts := []string{"s.metric_name = ANY(" + d.placeholder(1) + "::text[])"}
|
||||
if len(plan.EntityIDs) > 0 {
|
||||
args = append(args, plan.EntityIDs)
|
||||
parts = append(parts, "s.entity_id = ANY("+d.placeholder(len(args))+"::text[])")
|
||||
}
|
||||
for _, key := range sortedKeys(plan.Tags) {
|
||||
args = append(args, plan.Tags[key])
|
||||
parts = append(parts, d.jsonExtractEquals("s.tags", key, d.placeholder(len(args))))
|
||||
}
|
||||
return renderedSQL{
|
||||
Query: fmt.Sprintf("SELECT s.id, s.metric_name, s.entity_id, s.tags_hash, s.tags FROM %s s WHERE %s ORDER BY s.id ASC", tables.series, strings.Join(parts, " AND ")),
|
||||
Args: args,
|
||||
}
|
||||
}
|
||||
|
||||
func renderExpandedSeriesDictionary(d dialect, tables tables, indexHint string, plan seriesDictionaryPlan) renderedSQL {
|
||||
args := make([]any, 0, len(plan.MetricNames)+len(plan.EntityIDs)+len(plan.Tags))
|
||||
metricPlaceholders := appendPlaceholders(d, &args, plan.MetricNames)
|
||||
parts := []string{"s.metric_name IN (" + strings.Join(metricPlaceholders, ", ") + ")"}
|
||||
if len(plan.EntityIDs) > 0 {
|
||||
entityPlaceholders := appendPlaceholders(d, &args, plan.EntityIDs)
|
||||
parts = append(parts, "s.entity_id IN ("+strings.Join(entityPlaceholders, ", ")+")")
|
||||
}
|
||||
for _, key := range sortedKeys(plan.Tags) {
|
||||
args = append(args, plan.Tags[key])
|
||||
parts = append(parts, d.jsonExtractEquals("s.tags", key, d.placeholder(len(args))))
|
||||
}
|
||||
return renderedSQL{
|
||||
Query: fmt.Sprintf("SELECT s.id, s.metric_name, s.entity_id, s.tags_hash, s.tags FROM %s s%s WHERE %s ORDER BY s.id ASC", tables.series, indexHint, strings.Join(parts, " AND ")),
|
||||
Args: args,
|
||||
}
|
||||
}
|
||||
|
||||
func appendPlaceholders[T ~string](d dialect, args *[]any, values []T) []string {
|
||||
placeholders := make([]string, 0, len(values))
|
||||
for _, value := range values {
|
||||
*args = append(*args, string(value))
|
||||
placeholders = append(placeholders, d.placeholder(len(*args)))
|
||||
}
|
||||
return placeholders
|
||||
}
|
||||
|
||||
func (d sqliteDialect) renderRollupRead(tables tables, indexName string, plan rollupReadPlan) renderedSQL {
|
||||
seriesJSON, _ := json.Marshal(plan.SeriesIDs)
|
||||
args := []any{plan.ResolutionID, plan.StartMilli, plan.EndMilli, string(seriesJSON)}
|
||||
return renderedSQL{
|
||||
Query: fmt.Sprintf("SELECT %s FROM %s r INDEXED BY %s WHERE r.resolution_id = %s AND r.bucket_milli >= %s AND r.bucket_milli <= %s AND r.series_id IN (SELECT CAST(value AS INTEGER) FROM json_each(%s)) ORDER BY r.bucket_milli ASC, r.series_id ASC, r.label_id ASC",
|
||||
rollupReadColumns(plan.Fields), tables.rollups, indexName,
|
||||
d.placeholder(1), d.placeholder(2), d.placeholder(3), d.placeholder(4)),
|
||||
Args: args,
|
||||
}
|
||||
}
|
||||
|
||||
func (d mysqlDialect) renderRollupRead(tables tables, indexName string, plan rollupReadPlan) renderedSQL {
|
||||
args := []any{plan.ResolutionID, plan.StartMilli, plan.EndMilli}
|
||||
seriesPlaceholders := make([]string, 0, len(plan.SeriesIDs))
|
||||
for _, seriesID := range plan.SeriesIDs {
|
||||
args = append(args, seriesID)
|
||||
seriesPlaceholders = append(seriesPlaceholders, d.placeholder(len(args)))
|
||||
}
|
||||
return renderedSQL{
|
||||
Query: fmt.Sprintf("SELECT %s FROM %s r FORCE INDEX (%s) WHERE r.resolution_id = %s AND r.bucket_milli >= %s AND r.bucket_milli <= %s AND r.series_id IN (%s) ORDER BY r.bucket_milli ASC, r.series_id ASC, r.label_id ASC",
|
||||
rollupReadColumns(plan.Fields), tables.rollups, indexName,
|
||||
d.placeholder(1), d.placeholder(2), d.placeholder(3), strings.Join(seriesPlaceholders, ", ")),
|
||||
Args: args,
|
||||
}
|
||||
}
|
||||
|
||||
func (d postgresDialect) renderRollupRead(tables tables, _ string, plan rollupReadPlan) renderedSQL {
|
||||
args := []any{plan.ResolutionID, plan.StartMilli, plan.EndMilli, plan.SeriesIDs}
|
||||
return renderedSQL{
|
||||
Query: fmt.Sprintf("SELECT %s FROM %s r WHERE r.resolution_id = %s AND r.bucket_milli >= %s AND r.bucket_milli <= %s AND r.series_id = ANY(%s::bigint[]) ORDER BY r.bucket_milli ASC, r.series_id ASC, r.label_id ASC",
|
||||
rollupReadColumns(plan.Fields), tables.rollups,
|
||||
d.placeholder(1), d.placeholder(2), d.placeholder(3), d.placeholder(4)),
|
||||
Args: args,
|
||||
}
|
||||
}
|
||||
|
||||
func rollupReadColumns(fields rollupReadFields) string {
|
||||
columns := []string{"r.series_id", "r.bucket_milli", "r.count"}
|
||||
if fields&rollupReadSum != 0 {
|
||||
columns = append(columns, "r.sum")
|
||||
}
|
||||
if fields&rollupReadSumSq != 0 {
|
||||
columns = append(columns, "r.sum_sq")
|
||||
}
|
||||
if fields&rollupReadMin != 0 {
|
||||
columns = append(columns, "r.min_val")
|
||||
}
|
||||
if fields&rollupReadMax != 0 {
|
||||
columns = append(columns, "r.max_val")
|
||||
}
|
||||
if fields&rollupReadFirst != 0 {
|
||||
columns = append(columns, "r.first_val", "r.first_ts_milli")
|
||||
}
|
||||
if fields&rollupReadLast != 0 {
|
||||
columns = append(columns, "r.last_val", "r.last_ts_milli")
|
||||
}
|
||||
if fields&rollupReadDigest != 0 {
|
||||
columns = append(columns, "r.digest")
|
||||
}
|
||||
return strings.Join(columns, ", ")
|
||||
}
|
||||
@@ -146,7 +146,7 @@ func (s *Store) rebuildRollupBucketTx(ctx context.Context, metricName string, fi
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
stored, err := scanStoredRollups(rows, true, s.cfg.RollupPolicy.compression())
|
||||
stored, err := scanStoredRollupsForMaintenance(rows, true, s.cfg.RollupPolicy.compression())
|
||||
_ = rows.Close()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
|
||||
@@ -1104,8 +1104,8 @@ func TestCompactWithRawRetentionOnlyWritesChangedBuckets(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatalf("rollup: %v", err)
|
||||
}
|
||||
if len(got) != 1 || got[0].Count != 2 || got[0].Value != 3 {
|
||||
t.Fatalf("late delta was not merged exactly once: %#v", got)
|
||||
if len(got) != 1 || got[0].Count != 1 || got[0].Value != 1 {
|
||||
t.Fatalf("explicit 5m query read a finer tier: %#v", got)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
+36
-5
@@ -74,7 +74,7 @@ type Store struct {
|
||||
// coarseMu protects in-memory parent buckets. Coarser tiers are only
|
||||
// materialized once their source window has remained closed long enough for
|
||||
// the raw late-arrival window to pass.
|
||||
coarseMu sync.Mutex
|
||||
coarseMu sync.RWMutex
|
||||
coarse map[coarseRollupKey]*coarseRollup
|
||||
// mu protects closed state.
|
||||
//
|
||||
@@ -247,16 +247,19 @@ func prepareSQLiteConfig(cfg Config) (Config, error) {
|
||||
cfg.SQLite.BusyTimeout = 5 * time.Second
|
||||
}
|
||||
if cfg.SQLite.CacheSizeKB == 0 {
|
||||
cfg.SQLite.CacheSizeKB = 16 * 1024
|
||||
cfg.SQLite.CacheSizeKB = 4 * 1024
|
||||
}
|
||||
if cfg.SQLite.MMapSizeBytes == 0 {
|
||||
cfg.SQLite.MMapSizeBytes = 32 * 1024 * 1024
|
||||
// MMapSizeBytes == 0 表示显式禁用 mmap,而不是"未设置":DefaultConfig 始终
|
||||
// 填充 SQLiteOptions,因此这里的 0 是"跳过 mmap"的明确选择,不能被覆盖为
|
||||
// 32MB。负数视为非法输入,归一为 0(禁用)。
|
||||
if cfg.SQLite.MMapSizeBytes < 0 {
|
||||
cfg.SQLite.MMapSizeBytes = 0
|
||||
}
|
||||
if cfg.SQLite.WALAutoCheckpoint == 0 {
|
||||
cfg.SQLite.WALAutoCheckpoint = 4000
|
||||
}
|
||||
if cfg.SQLite.JournalSizeLimitBytes == 0 {
|
||||
cfg.SQLite.JournalSizeLimitBytes = 16 * 1024 * 1024
|
||||
cfg.SQLite.JournalSizeLimitBytes = 4 * 1024 * 1024
|
||||
}
|
||||
|
||||
if cfg.DB == nil {
|
||||
@@ -410,6 +413,22 @@ func (s *Store) Ping(ctx context.Context) error {
|
||||
return s.db.PingContext(ctx)
|
||||
}
|
||||
|
||||
// QueryContext executes a raw read query against the Store's read pool.
|
||||
func (s *Store) QueryContext(ctx context.Context, query string, args ...any) (*sql.Rows, error) {
|
||||
if err := s.ensureOpen(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return s.reader().QueryContext(ctx, query, args...)
|
||||
}
|
||||
|
||||
// ExecContext executes a raw statement against the Store's primary write pool.
|
||||
func (s *Store) ExecContext(ctx context.Context, query string, args ...any) (sql.Result, error) {
|
||||
if err := s.ensureOpen(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return s.db.ExecContext(ctx, query, args...)
|
||||
}
|
||||
|
||||
// ensureOpen verifies that the Store is not closed.
|
||||
//
|
||||
// ensureOpen 检查 Store 是否仍处于打开状态。
|
||||
@@ -931,6 +950,18 @@ func (s *Store) Query(ctx context.Context, query Query) ([]Point, error) {
|
||||
return s.queryRawPoints(ctx, query)
|
||||
}
|
||||
|
||||
// QueryBatch loads exact raw samples for multiple metrics with one scan of the
|
||||
// in-memory raw store.
|
||||
func (s *Store) QueryBatch(ctx context.Context, query BatchQuery) (map[string][]Point, error) {
|
||||
if err := s.ensureOpen(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := query.Validate(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return s.queryRawPointsBatch(ctx, query.normalized())
|
||||
}
|
||||
|
||||
// EntityIDs returns distinct entity ids that have exact in-memory samples or
|
||||
// persisted/hot rollups.
|
||||
func (s *Store) EntityIDs(ctx context.Context, query Query) ([]string, error) {
|
||||
|
||||
@@ -213,8 +213,8 @@ func TestSQLiteInDirCreatesDirectoryAndAppliesPragmas(t *testing.T) {
|
||||
if err := store.db.QueryRowContext(ctx, "PRAGMA journal_size_limit").Scan(&journalSizeLimit); err != nil {
|
||||
t.Fatalf("query journal size limit: %v", err)
|
||||
}
|
||||
if journalSizeLimit != 16*1024*1024 {
|
||||
t.Fatalf("journal size limit = %d, want %d", journalSizeLimit, 16*1024*1024)
|
||||
if journalSizeLimit != 4*1024*1024 {
|
||||
t.Fatalf("journal size limit = %d, want %d", journalSizeLimit, 4*1024*1024)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
+10
-7
@@ -139,11 +139,15 @@ func (t *TDigest) Add(x, w float64) {
|
||||
t.processed = false
|
||||
// Bound the unprocessed buffer so a long stream cannot grow memory without
|
||||
// limit; process() collapses it back to ~compression centroids.
|
||||
if len(t.centroids) > int(8*t.compression)+16 {
|
||||
if len(t.centroids) > t.processThreshold() {
|
||||
t.process()
|
||||
}
|
||||
}
|
||||
|
||||
func (t *TDigest) processThreshold() int {
|
||||
return int(8*t.compression) + 16
|
||||
}
|
||||
|
||||
// Merge folds every centroid of other into t. This is the operation rollup
|
||||
// composition relies on: a coarse bucket merges the digests of the finer
|
||||
// buckets it spans.
|
||||
@@ -154,10 +158,7 @@ func (t *TDigest) Merge(other *TDigest) {
|
||||
if other == nil || other.count == 0 {
|
||||
return
|
||||
}
|
||||
other.process()
|
||||
for _, c := range other.centroids {
|
||||
t.centroids = append(t.centroids, c)
|
||||
}
|
||||
t.centroids = append(t.centroids, other.centroids...)
|
||||
t.count += other.count
|
||||
if other.min < t.min {
|
||||
t.min = other.min
|
||||
@@ -166,7 +167,9 @@ func (t *TDigest) Merge(other *TDigest) {
|
||||
t.max = other.max
|
||||
}
|
||||
t.processed = false
|
||||
t.process()
|
||||
if len(t.centroids) > t.processThreshold() {
|
||||
t.process()
|
||||
}
|
||||
}
|
||||
|
||||
// Count returns the total weight observed.
|
||||
@@ -194,7 +197,7 @@ func (t *TDigest) process() {
|
||||
return t.centroids[i].mean < t.centroids[j].mean
|
||||
})
|
||||
total := t.count
|
||||
merged := t.centroids[:0:0] // fresh backing array; don't alias input mid-merge
|
||||
merged := t.centroids[:0]
|
||||
cur := t.centroids[0]
|
||||
weightBefore := 0.0
|
||||
for i := 1; i < len(t.centroids); i++ {
|
||||
|
||||
@@ -73,6 +73,95 @@ func TestTDigestMergeMatchesCombined(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestTDigestMergeGroupingAccuracy(t *testing.T) {
|
||||
rng := rand.New(rand.NewSource(17))
|
||||
all := make([]float64, 0, 24*2000)
|
||||
fine := make([]*TDigest, 24)
|
||||
for bucket := range fine {
|
||||
fine[bucket] = NewTDigest(100)
|
||||
for i := 0; i < 2000; i++ {
|
||||
value := math.Abs(rng.NormFloat64())*70 + rng.ExpFloat64()*30
|
||||
fine[bucket].Add(value, 1)
|
||||
all = append(all, value)
|
||||
}
|
||||
}
|
||||
|
||||
flat := NewTDigest(100)
|
||||
grouped := NewTDigest(100)
|
||||
for _, digest := range fine {
|
||||
flat.Merge(digest)
|
||||
}
|
||||
for groupStart := 0; groupStart < len(fine); groupStart += 4 {
|
||||
group := NewTDigest(100)
|
||||
for _, digest := range fine[groupStart : groupStart+4] {
|
||||
group.Merge(digest)
|
||||
}
|
||||
grouped.Merge(group)
|
||||
}
|
||||
|
||||
for name, digest := range map[string]*TDigest{"flat": flat, "grouped": grouped} {
|
||||
if digest.Count() != float64(len(all)) || digest.min != exactQuantile(all, 0) || digest.max != exactQuantile(all, 1) {
|
||||
t.Fatalf("%s exact aggregates changed: count=%v min=%v max=%v", name, digest.Count(), digest.min, digest.max)
|
||||
}
|
||||
for _, q := range []float64{0.5, 0.95, 0.99, 0.999} {
|
||||
exact := exactQuantile(all, q)
|
||||
relErr := math.Abs(digest.Quantile(q)-exact) / math.Abs(exact)
|
||||
if relErr > 0.02 {
|
||||
t.Fatalf("%s q=%v relative error %v exceeds 2%%", name, q, relErr)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestTDigestMergeDefersProcessingUntilThreshold(t *testing.T) {
|
||||
source := NewTDigest(30)
|
||||
source.Add(42, 1)
|
||||
if source.processed {
|
||||
t.Fatal("source should have buffered centroid before merge")
|
||||
}
|
||||
|
||||
merged := NewTDigest(30)
|
||||
for i := 0; i <= merged.processThreshold(); i++ {
|
||||
merged.Merge(source)
|
||||
}
|
||||
if !merged.processed {
|
||||
t.Fatal("merge did not process after exceeding threshold")
|
||||
}
|
||||
if source.processed {
|
||||
t.Fatal("merge processed the source digest")
|
||||
}
|
||||
|
||||
merged.Merge(source)
|
||||
if merged.processed {
|
||||
t.Fatal("merge below threshold should remain buffered")
|
||||
}
|
||||
if merged.Count() != float64(merged.processThreshold()+2) {
|
||||
t.Fatalf("count = %v", merged.Count())
|
||||
}
|
||||
if merged.processed {
|
||||
t.Fatal("Count processed buffered centroids")
|
||||
}
|
||||
if value := merged.Quantile(0.5); value != 42 || !merged.processed {
|
||||
t.Fatalf("Quantile did not finalize digest: value=%v processed=%v", value, merged.processed)
|
||||
}
|
||||
|
||||
merged.Merge(source)
|
||||
if merged.processed {
|
||||
t.Fatal("merge below threshold should remain buffered before Encode")
|
||||
}
|
||||
encoded := merged.Encode()
|
||||
if len(encoded) == 0 || !merged.processed {
|
||||
t.Fatal("Encode did not finalize digest")
|
||||
}
|
||||
decoded, err := DecodeTDigest(encoded)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if decoded.Count() != merged.Count() || decoded.min != merged.min || decoded.max != merged.max {
|
||||
t.Fatalf("round trip changed exact aggregates: %#v vs %#v", decoded, merged)
|
||||
}
|
||||
}
|
||||
|
||||
// TestTDigestEncodeRoundTrip verifies t-digest serialization.
|
||||
//
|
||||
// TestTDigestEncodeRoundTrip 验证 t-digest 编码和解码往返。
|
||||
|
||||
@@ -291,6 +291,43 @@ type Query struct {
|
||||
Order Order `json:"order,omitempty"`
|
||||
}
|
||||
|
||||
// BatchQuery loads exact raw samples for multiple metrics and entities from one
|
||||
// shared time window.
|
||||
type BatchQuery struct {
|
||||
MetricNames []string `json:"metric_names"`
|
||||
EntityIDs []string `json:"entity_ids,omitempty"`
|
||||
Start time.Time `json:"start"`
|
||||
End time.Time `json:"end"`
|
||||
Tags map[string]string `json:"tags,omitempty"`
|
||||
Order Order `json:"order,omitempty"`
|
||||
}
|
||||
|
||||
// BatchSeriesSpec describes the rollup output requested for one metric.
|
||||
type BatchSeriesSpec struct {
|
||||
MetricName string
|
||||
Aggregations []Aggregation
|
||||
Interval time.Duration
|
||||
PreserveSeries bool
|
||||
}
|
||||
|
||||
// BatchSeriesQuery combines per-metric rollup specifications that share the
|
||||
// same time, entity, tag, and ordering filters.
|
||||
type BatchSeriesQuery struct {
|
||||
Specs []BatchSeriesSpec
|
||||
EntityIDs []string
|
||||
Start time.Time
|
||||
End time.Time
|
||||
Tags map[string]string
|
||||
Order Order
|
||||
}
|
||||
|
||||
// BatchSeriesResult contains definitions and aggregate values keyed by metric
|
||||
// and aggregation.
|
||||
type BatchSeriesResult struct {
|
||||
Definitions map[string]Definition
|
||||
Values map[string]map[Aggregation][]AggregatePoint
|
||||
}
|
||||
|
||||
// Validate checks whether the value is well formed.
|
||||
//
|
||||
// Validate 检查原始查询条件是否合法。
|
||||
@@ -327,6 +364,76 @@ func (q Query) normalized() Query {
|
||||
return q
|
||||
}
|
||||
|
||||
// Validate checks whether a batch raw query is well formed.
|
||||
func (q BatchQuery) Validate() error {
|
||||
if len(q.MetricNames) == 0 {
|
||||
return fmt.Errorf("%w: at least one metric name is required", ErrInvalidArgument)
|
||||
}
|
||||
for _, name := range q.MetricNames {
|
||||
if strings.TrimSpace(name) == "" {
|
||||
return fmt.Errorf("%w: metric name is required", ErrInvalidArgument)
|
||||
}
|
||||
}
|
||||
if q.Start.IsZero() || q.End.IsZero() {
|
||||
return fmt.Errorf("%w: start and end time are required", ErrInvalidArgument)
|
||||
}
|
||||
if q.End.Before(q.Start) {
|
||||
return fmt.Errorf("%w: end time cannot be before start time", ErrInvalidArgument)
|
||||
}
|
||||
switch q.Order {
|
||||
case "", OrderAsc, OrderDesc:
|
||||
default:
|
||||
return fmt.Errorf("%w: unsupported order %q", ErrInvalidArgument, q.Order)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (q BatchQuery) normalized() BatchQuery {
|
||||
q.Start = q.Start.UTC()
|
||||
q.End = q.End.UTC()
|
||||
if q.Order == "" {
|
||||
q.Order = OrderAsc
|
||||
}
|
||||
return q
|
||||
}
|
||||
|
||||
// Validate checks whether a batch rollup query is well formed.
|
||||
func (q BatchSeriesQuery) Validate() error {
|
||||
if len(q.Specs) == 0 {
|
||||
return fmt.Errorf("%w: at least one series specification is required", ErrInvalidArgument)
|
||||
}
|
||||
seen := make(map[string]struct{}, len(q.Specs))
|
||||
for _, spec := range q.Specs {
|
||||
if _, ok := seen[spec.MetricName]; ok {
|
||||
return fmt.Errorf("%w: duplicate series specification for metric %q", ErrInvalidArgument, spec.MetricName)
|
||||
}
|
||||
seen[spec.MetricName] = struct{}{}
|
||||
if len(spec.Aggregations) == 0 {
|
||||
return fmt.Errorf("%w: at least one aggregation is required for metric %q", ErrInvalidArgument, spec.MetricName)
|
||||
}
|
||||
for _, aggregation := range spec.Aggregations {
|
||||
if err := (AggregateQuery{
|
||||
Query: Query{MetricName: spec.MetricName, Start: q.Start, End: q.End, Tags: q.Tags, Order: q.Order},
|
||||
Aggregation: aggregation,
|
||||
Interval: spec.Interval,
|
||||
PreserveSeries: spec.PreserveSeries,
|
||||
}).Validate(); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (q BatchSeriesQuery) normalized() BatchSeriesQuery {
|
||||
q.Start = q.Start.UTC()
|
||||
q.End = q.End.UTC()
|
||||
if q.Order == "" {
|
||||
q.Order = OrderAsc
|
||||
}
|
||||
return q
|
||||
}
|
||||
|
||||
// AggregateQuery describes a bucketed aggregate query.
|
||||
//
|
||||
// AggregateQuery 描述按固定时间间隔分桶后的聚合查询。
|
||||
|
||||
@@ -130,7 +130,7 @@ func sendTrafficReport(daily, weekly, monthly bool) {
|
||||
return
|
||||
}
|
||||
|
||||
message := strings.Join(lines, "\n")
|
||||
message := strings.Join(lines, "\nKomari 内置的流量报告即将在 1.5.0 版本移除,请尽快迁移至插件“流量定期报告”。\nKomari's built-in traffic reports will be removed in version 1.5.0. Please migrate to the Scheduled Traffic Reports plugin as soon as possible.")
|
||||
var emoji string
|
||||
switch {
|
||||
case daily:
|
||||
|
||||
@@ -0,0 +1,70 @@
|
||||
package admin
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"time"
|
||||
|
||||
"github.com/komari-monitor/komari/internal/plugin"
|
||||
logger "github.com/komari-monitor/komari/utils/log"
|
||||
"github.com/komari-monitor/komari/web/backup"
|
||||
"github.com/komari-monitor/komari/web/upload"
|
||||
)
|
||||
|
||||
func NewArchiveUploadHandler() *upload.Handler {
|
||||
return upload.NewHandler(upload.DefaultStore, map[upload.Purpose]upload.Finalizer{
|
||||
upload.PurposeBackup: finalizeBackupUpload,
|
||||
upload.PurposePlugin: finalizePluginUpload,
|
||||
upload.PurposeTheme: finalizeThemeUpload,
|
||||
})
|
||||
}
|
||||
|
||||
func finalizeBackupUpload(session upload.Session) (upload.Result, error) {
|
||||
restoreLock, err := backup.AcquireRestoreLock()
|
||||
if err != nil {
|
||||
return upload.Result{}, err
|
||||
}
|
||||
archive, err := os.Open(session.ArchivePath)
|
||||
if err != nil {
|
||||
restoreLock.Release()
|
||||
return upload.Result{}, fmt.Errorf("open merged backup: %w", err)
|
||||
}
|
||||
if err := restoreLock.SaveUploadedBackup(archive, session.Metadata.Filename); err != nil {
|
||||
_ = archive.Close()
|
||||
restoreLock.Release()
|
||||
return upload.Result{}, err
|
||||
}
|
||||
if err := archive.Close(); err != nil {
|
||||
restoreLock.Release()
|
||||
return upload.Result{}, fmt.Errorf("close merged backup: %w", err)
|
||||
}
|
||||
|
||||
go func() {
|
||||
logger.InfoArgs("admin-api", "Backup uploaded, restarting service in 2 seconds to apply on startup...")
|
||||
time.Sleep(2 * time.Second)
|
||||
restoreLock.Release()
|
||||
os.Exit(0)
|
||||
}()
|
||||
|
||||
return upload.Result{
|
||||
Message: "Backup uploaded successfully. The service will restart and apply the backup.",
|
||||
Data: map[string]string{"path": filepath.Join(".", "data", "backup.zip")},
|
||||
}, nil
|
||||
}
|
||||
|
||||
func finalizePluginUpload(session upload.Session) (upload.Result, error) {
|
||||
info, err := plugin.InstallZip(session.ArchivePath)
|
||||
if err != nil {
|
||||
return upload.Result{}, err
|
||||
}
|
||||
return upload.Result{Message: "插件上传成功", Data: info}, nil
|
||||
}
|
||||
|
||||
func finalizeThemeUpload(session upload.Session) (upload.Result, error) {
|
||||
info, err := extractAndValidateTheme(session.ArchivePath)
|
||||
if err != nil {
|
||||
return upload.Result{}, err
|
||||
}
|
||||
return upload.Result{Message: "主题上传成功", Data: info}, nil
|
||||
}
|
||||
@@ -1,111 +0,0 @@
|
||||
package admin
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestIsValidUploadID(t *testing.T) {
|
||||
for _, uploadID := range []string{chunkUploadID, "../backup", "", "another-upload"} {
|
||||
want := uploadID == chunkUploadID
|
||||
if got := isValidUploadID(uploadID); got != want {
|
||||
t.Errorf("isValidUploadID(%q) = %v, want %v", uploadID, got, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestChunkUploadDirRejectsTraversal(t *testing.T) {
|
||||
if _, err := chunkUploadDir(".."); err == nil {
|
||||
t.Fatal("chunkUploadDir accepted traversal upload_id")
|
||||
}
|
||||
}
|
||||
|
||||
func TestClearChunkUploadCache(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
if err := os.WriteFile(filepath.Join(root, "interrupted.part"), []byte("stale"), 0600); err != nil {
|
||||
t.Fatalf("write cache: %v", err)
|
||||
}
|
||||
if err := clearChunkUploadCache(root); err != nil {
|
||||
t.Fatalf("clearChunkUploadCache: %v", err)
|
||||
}
|
||||
entries, err := os.ReadDir(root)
|
||||
if err != nil {
|
||||
t.Fatalf("read cache: %v", err)
|
||||
}
|
||||
if len(entries) != 0 {
|
||||
t.Fatalf("cache entries = %d, want 0", len(entries))
|
||||
}
|
||||
}
|
||||
|
||||
func TestCopyWhitelistedFilesIncludesFont(t *testing.T) {
|
||||
dataDir := t.TempDir()
|
||||
backupDir := t.TempDir()
|
||||
fontPath := filepath.Join(dataDir, "font.ttf")
|
||||
if err := os.WriteFile(fontPath, []byte("font-data"), 0644); err != nil {
|
||||
t.Fatalf("write font: %v", err)
|
||||
}
|
||||
|
||||
if err := copyWhitelistedFilesFrom(dataDir, backupDir); err != nil {
|
||||
t.Fatalf("copyWhitelistedFilesFrom: %v", err)
|
||||
}
|
||||
content, err := os.ReadFile(filepath.Join(backupDir, "font.ttf"))
|
||||
if err != nil {
|
||||
t.Fatalf("read copied font: %v", err)
|
||||
}
|
||||
if string(content) != "font-data" {
|
||||
t.Fatalf("font content = %q, want %q", content, "font-data")
|
||||
}
|
||||
}
|
||||
|
||||
func TestChunkUploadMetadataAndSize(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
metadata, err := json.Marshal(chunkUploadMetadata{Size: 7})
|
||||
if err != nil {
|
||||
t.Fatalf("marshal metadata: %v", err)
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(dir, uploadMetadataName), metadata, 0600); err != nil {
|
||||
t.Fatalf("write metadata: %v", err)
|
||||
}
|
||||
for name, content := range map[string]string{"0.part": "abc", "1.part": "defg", ".1-staging.part": "staging", "ignored.txt": "ignored"} {
|
||||
if err := os.WriteFile(filepath.Join(dir, name), []byte(content), 0600); err != nil {
|
||||
t.Fatalf("write %s: %v", name, err)
|
||||
}
|
||||
}
|
||||
|
||||
gotMetadata, err := readChunkUploadMetadata(dir)
|
||||
if err != nil {
|
||||
t.Fatalf("read metadata: %v", err)
|
||||
}
|
||||
if gotMetadata.Size != 7 {
|
||||
t.Fatalf("metadata size = %d, want 7", gotMetadata.Size)
|
||||
}
|
||||
gotSize, err := chunkUploadSize(dir)
|
||||
if err != nil {
|
||||
t.Fatalf("chunkUploadSize: %v", err)
|
||||
}
|
||||
if gotSize != 7 {
|
||||
t.Fatalf("chunk upload size = %d, want 7", gotSize)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMergeChunksIgnoresTemporaryParts(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
for name, content := range map[string]string{"0.part": "first", "1.part": "second", ".1-staging.part": "temporary"} {
|
||||
if err := os.WriteFile(filepath.Join(dir, name), []byte(content), 0600); err != nil {
|
||||
t.Fatalf("write %s: %v", name, err)
|
||||
}
|
||||
}
|
||||
destination := filepath.Join(dir, "merged.zip")
|
||||
if err := mergeChunks(dir, destination); err != nil {
|
||||
t.Fatalf("mergeChunks: %v", err)
|
||||
}
|
||||
content, err := os.ReadFile(destination)
|
||||
if err != nil {
|
||||
t.Fatalf("read merged chunks: %v", err)
|
||||
}
|
||||
if string(content) != "firstsecond" {
|
||||
t.Fatalf("merged chunks = %q, want %q", content, "firstsecond")
|
||||
}
|
||||
}
|
||||
@@ -15,6 +15,8 @@ var backupWhitelist = []string{
|
||||
"favicon.ico",
|
||||
"font.ttf",
|
||||
"theme/",
|
||||
"plugin/",
|
||||
"plguin-data/",
|
||||
"metrics.db",
|
||||
}
|
||||
|
||||
|
||||
@@ -1,381 +0,0 @@
|
||||
package admin
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"log"
|
||||
"net/http"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/gin-gonic/gin"
|
||||
"github.com/komari-monitor/komari/web/api"
|
||||
"github.com/komari-monitor/komari/web/backup"
|
||||
)
|
||||
|
||||
const (
|
||||
defaultChunkSize = 5 * 1024 * 1024 // 5 MiB
|
||||
maxChunkRequestSize = defaultChunkSize + 1*1024*1024
|
||||
chunkUploadID = "backup"
|
||||
maxChunkIndex = int(backup.MaxArchiveSize / defaultChunkSize)
|
||||
)
|
||||
|
||||
var chunkUploadRoot = filepath.Join(".", "data", "backup", ".uploading")
|
||||
|
||||
const uploadMetadataName = "upload.json"
|
||||
|
||||
type chunkUploadMetadata struct {
|
||||
Size int64 `json:"size"`
|
||||
}
|
||||
|
||||
func isValidUploadID(uploadID string) bool {
|
||||
return uploadID == chunkUploadID
|
||||
}
|
||||
|
||||
func chunkUploadDir(uploadID string) (string, error) {
|
||||
if !isValidUploadID(uploadID) {
|
||||
return "", fmt.Errorf("invalid upload_id")
|
||||
}
|
||||
return filepath.Join(chunkUploadRoot, uploadID), nil
|
||||
}
|
||||
|
||||
func readChunkUploadMetadata(chunkDir string) (chunkUploadMetadata, error) {
|
||||
var metadata chunkUploadMetadata
|
||||
data, err := os.ReadFile(filepath.Join(chunkDir, uploadMetadataName))
|
||||
if err != nil {
|
||||
return metadata, err
|
||||
}
|
||||
if err := json.Unmarshal(data, &metadata); err != nil {
|
||||
return metadata, err
|
||||
}
|
||||
if metadata.Size <= 0 || metadata.Size > backup.MaxArchiveSize {
|
||||
return metadata, fmt.Errorf("invalid upload size")
|
||||
}
|
||||
return metadata, nil
|
||||
}
|
||||
|
||||
func chunkPartIndex(name string) (int, bool) {
|
||||
index, err := strconv.Atoi(strings.TrimSuffix(name, ".part"))
|
||||
return index, err == nil && index >= 0 && strings.HasSuffix(name, ".part")
|
||||
}
|
||||
|
||||
func chunkUploadSize(chunkDir string) (int64, error) {
|
||||
entries, err := os.ReadDir(chunkDir)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
var total int64
|
||||
for _, entry := range entries {
|
||||
if entry.IsDir() {
|
||||
continue
|
||||
}
|
||||
if _, ok := chunkPartIndex(entry.Name()); !ok {
|
||||
continue
|
||||
}
|
||||
info, err := entry.Info()
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
if total > backup.MaxArchiveSize-info.Size() {
|
||||
return 0, fmt.Errorf("uploaded chunks exceed the archive limit")
|
||||
}
|
||||
total += info.Size()
|
||||
}
|
||||
return total, nil
|
||||
}
|
||||
|
||||
func clearChunkUploadCache(root string) error {
|
||||
if err := os.RemoveAll(root); err != nil {
|
||||
return err
|
||||
}
|
||||
return os.MkdirAll(root, 0755)
|
||||
}
|
||||
|
||||
// InitChunkUpload 初始化分块上传,返回 upload_id 和 chunk_size。
|
||||
// Komari 使用单一上传槽位,初始化时会清除上次中断留下的全部缓存。
|
||||
func InitChunkUpload(c *gin.Context) {
|
||||
var req struct {
|
||||
Size int64 `json:"size"`
|
||||
}
|
||||
if err := c.ShouldBindJSON(&req); err != nil || req.Size <= 0 || req.Size > backup.MaxArchiveSize {
|
||||
api.RespondError(c, http.StatusBadRequest, fmt.Sprintf("size must be between 1 and %d bytes", backup.MaxArchiveSize))
|
||||
return
|
||||
}
|
||||
|
||||
if err := clearChunkUploadCache(chunkUploadRoot); err != nil {
|
||||
api.RespondError(c, http.StatusInternalServerError, fmt.Sprintf("Error clearing upload cache: %v", err))
|
||||
return
|
||||
}
|
||||
chunkDir, err := chunkUploadDir(chunkUploadID)
|
||||
if err != nil {
|
||||
api.RespondError(c, http.StatusInternalServerError, "Error preparing upload directory")
|
||||
return
|
||||
}
|
||||
if err := os.MkdirAll(chunkDir, 0755); err != nil {
|
||||
api.RespondError(c, http.StatusInternalServerError, fmt.Sprintf("Error creating upload directory: %v", err))
|
||||
return
|
||||
}
|
||||
metadata, err := json.Marshal(chunkUploadMetadata{Size: req.Size})
|
||||
if err != nil {
|
||||
clearChunkUploadCache(chunkUploadRoot)
|
||||
api.RespondError(c, http.StatusInternalServerError, fmt.Sprintf("Error encoding upload metadata: %v", err))
|
||||
return
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(chunkDir, uploadMetadataName), metadata, 0600); err != nil {
|
||||
clearChunkUploadCache(chunkUploadRoot)
|
||||
api.RespondError(c, http.StatusInternalServerError, fmt.Sprintf("Error writing upload metadata: %v", err))
|
||||
return
|
||||
}
|
||||
|
||||
c.JSON(http.StatusOK, gin.H{
|
||||
"upload_id": chunkUploadID,
|
||||
"chunk_size": defaultChunkSize,
|
||||
})
|
||||
}
|
||||
|
||||
// UploadChunk 接收单个分块,保存到临时目录。
|
||||
func UploadChunk(c *gin.Context) {
|
||||
defer func() {
|
||||
if c.Request.Context().Err() != nil {
|
||||
_ = clearChunkUploadCache(chunkUploadRoot)
|
||||
}
|
||||
}()
|
||||
c.Request.Body = http.MaxBytesReader(c.Writer, c.Request.Body, maxChunkRequestSize)
|
||||
uploadID := c.PostForm("upload_id")
|
||||
chunkDir, err := chunkUploadDir(uploadID)
|
||||
if err != nil {
|
||||
api.RespondError(c, http.StatusBadRequest, "invalid upload_id")
|
||||
return
|
||||
}
|
||||
|
||||
chunkIndexStr := c.PostForm("chunk_index")
|
||||
if chunkIndexStr == "" {
|
||||
api.RespondError(c, http.StatusBadRequest, "chunk_index is required")
|
||||
return
|
||||
}
|
||||
chunkIndex, err := strconv.Atoi(chunkIndexStr)
|
||||
if err != nil || chunkIndex < 0 || chunkIndex > maxChunkIndex {
|
||||
api.RespondError(c, http.StatusBadRequest, "chunk_index must be a non-negative integer")
|
||||
return
|
||||
}
|
||||
|
||||
if info, err := os.Stat(chunkDir); err != nil || !info.IsDir() {
|
||||
api.RespondError(c, http.StatusNotFound, "upload_id not found or expired")
|
||||
return
|
||||
}
|
||||
|
||||
file, _, err := c.Request.FormFile("chunk_data")
|
||||
if err != nil {
|
||||
api.RespondError(c, http.StatusBadRequest, fmt.Sprintf("Error getting chunk data: %v", err))
|
||||
return
|
||||
}
|
||||
defer file.Close()
|
||||
|
||||
metadata, err := readChunkUploadMetadata(chunkDir)
|
||||
if err != nil {
|
||||
api.RespondError(c, http.StatusNotFound, "upload_id not found or expired")
|
||||
return
|
||||
}
|
||||
|
||||
chunkPath := filepath.Join(chunkDir, fmt.Sprintf("%d.part", chunkIndex))
|
||||
tempChunk, err := os.CreateTemp(chunkDir, fmt.Sprintf(".%d-*.part", chunkIndex))
|
||||
if err != nil {
|
||||
api.RespondError(c, http.StatusInternalServerError, fmt.Sprintf("Error saving chunk: %v", err))
|
||||
return
|
||||
}
|
||||
tempChunkPath := tempChunk.Name()
|
||||
defer os.Remove(tempChunkPath)
|
||||
|
||||
written, err := io.Copy(tempChunk, io.LimitReader(file, defaultChunkSize+1))
|
||||
if closeErr := tempChunk.Close(); err == nil {
|
||||
err = closeErr
|
||||
}
|
||||
if err != nil || written > defaultChunkSize {
|
||||
if err != nil {
|
||||
api.RespondError(c, http.StatusInternalServerError, fmt.Sprintf("Error writing chunk: %v", err))
|
||||
} else {
|
||||
api.RespondError(c, http.StatusBadRequest, fmt.Sprintf("Chunk too large: %d bytes (max %d)", written, defaultChunkSize))
|
||||
}
|
||||
return
|
||||
}
|
||||
currentSize, err := chunkUploadSize(chunkDir)
|
||||
if err != nil {
|
||||
api.RespondError(c, http.StatusInternalServerError, fmt.Sprintf("Error checking uploaded chunks: %v", err))
|
||||
return
|
||||
}
|
||||
oldSize := int64(0)
|
||||
if info, err := os.Stat(chunkPath); err == nil {
|
||||
oldSize = info.Size()
|
||||
} else if !os.IsNotExist(err) {
|
||||
api.RespondError(c, http.StatusInternalServerError, fmt.Sprintf("Error checking existing chunk: %v", err))
|
||||
return
|
||||
}
|
||||
if currentSize-oldSize+written > metadata.Size {
|
||||
api.RespondError(c, http.StatusBadRequest, "uploaded chunks exceed the declared backup size")
|
||||
return
|
||||
}
|
||||
if err := os.Remove(chunkPath); err != nil && !os.IsNotExist(err) {
|
||||
api.RespondError(c, http.StatusInternalServerError, fmt.Sprintf("Error replacing chunk: %v", err))
|
||||
return
|
||||
}
|
||||
if err := os.Rename(tempChunkPath, chunkPath); err != nil {
|
||||
api.RespondError(c, http.StatusInternalServerError, fmt.Sprintf("Error publishing chunk: %v", err))
|
||||
return
|
||||
}
|
||||
|
||||
c.JSON(http.StatusOK, gin.H{
|
||||
"received": true,
|
||||
"chunk_index": chunkIndex,
|
||||
})
|
||||
}
|
||||
|
||||
// MergeChunkUpload 合并分块到 backup.zip 后触发恢复。
|
||||
func MergeChunkUpload(c *gin.Context) {
|
||||
restoreLock, err := backup.AcquireRestoreLock()
|
||||
if err != nil {
|
||||
api.RespondError(c, http.StatusConflict, err.Error())
|
||||
return
|
||||
}
|
||||
keepRestoreLock := false
|
||||
defer func() {
|
||||
if !keepRestoreLock {
|
||||
restoreLock.Release()
|
||||
}
|
||||
}()
|
||||
|
||||
var req struct {
|
||||
UploadID string `json:"upload_id" binding:"required"`
|
||||
}
|
||||
if err := c.ShouldBindJSON(&req); err != nil {
|
||||
api.RespondError(c, http.StatusBadRequest, fmt.Sprintf("Invalid request: %v", err))
|
||||
return
|
||||
}
|
||||
|
||||
chunkDir, err := chunkUploadDir(req.UploadID)
|
||||
if err != nil {
|
||||
api.RespondError(c, http.StatusBadRequest, "invalid upload_id")
|
||||
return
|
||||
}
|
||||
if info, err := os.Stat(chunkDir); err != nil || !info.IsDir() {
|
||||
api.RespondError(c, http.StatusNotFound, "upload_id not found or expired")
|
||||
return
|
||||
}
|
||||
metadata, err := readChunkUploadMetadata(chunkDir)
|
||||
if err != nil {
|
||||
api.RespondError(c, http.StatusNotFound, "upload_id not found or expired")
|
||||
return
|
||||
}
|
||||
if uploadedSize, err := chunkUploadSize(chunkDir); err != nil || uploadedSize != metadata.Size {
|
||||
_ = clearChunkUploadCache(chunkUploadRoot)
|
||||
if err != nil {
|
||||
api.RespondError(c, http.StatusInternalServerError, fmt.Sprintf("Error checking uploaded chunks: %v", err))
|
||||
} else {
|
||||
api.RespondError(c, http.StatusBadRequest, "uploaded chunk size does not match the declared backup size")
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
mergedPath := filepath.Join(chunkDir, "merged.zip")
|
||||
if err := mergeChunks(chunkDir, mergedPath); err != nil {
|
||||
clearChunkUploadCache(chunkUploadRoot)
|
||||
api.RespondError(c, http.StatusInternalServerError, fmt.Sprintf("Error merging chunks: %v", err))
|
||||
return
|
||||
}
|
||||
|
||||
if err := validateBackupZip(mergedPath); err != nil {
|
||||
clearChunkUploadCache(chunkUploadRoot)
|
||||
api.RespondError(c, http.StatusBadRequest, fmt.Sprintf("Invalid backup: %v", err))
|
||||
return
|
||||
}
|
||||
|
||||
archive, err := os.Open(mergedPath)
|
||||
if err != nil {
|
||||
clearChunkUploadCache(chunkUploadRoot)
|
||||
api.RespondError(c, http.StatusInternalServerError, fmt.Sprintf("Error opening merged backup: %v", err))
|
||||
return
|
||||
}
|
||||
if err := restoreLock.SaveUploadedBackup(archive, "backup.zip"); err != nil {
|
||||
archive.Close()
|
||||
clearChunkUploadCache(chunkUploadRoot)
|
||||
api.RespondError(c, http.StatusBadRequest, fmt.Sprintf("Error preparing restore: %v", err))
|
||||
return
|
||||
}
|
||||
if err := archive.Close(); err != nil {
|
||||
clearChunkUploadCache(chunkUploadRoot)
|
||||
api.RespondError(c, http.StatusInternalServerError, fmt.Sprintf("Error closing merged backup: %v", err))
|
||||
return
|
||||
}
|
||||
|
||||
clearChunkUploadCache(chunkUploadRoot)
|
||||
|
||||
c.JSON(http.StatusOK, gin.H{
|
||||
"status": "success",
|
||||
"message": "Backup uploaded successfully. The service will restart and apply the backup.",
|
||||
"path": filepath.Join(".", "data", "backup.zip"),
|
||||
})
|
||||
|
||||
keepRestoreLock = true
|
||||
go func() {
|
||||
log.Println("Backup uploaded (chunk), restarting service in 2 seconds to apply on startup...")
|
||||
time.Sleep(2 * time.Second)
|
||||
restoreLock.Release()
|
||||
os.Exit(0)
|
||||
}()
|
||||
}
|
||||
|
||||
// mergeChunks 按分块索引数值排序后顺序合并所有 .part 文件。
|
||||
func mergeChunks(chunkDir, destPath string) error {
|
||||
parts, err := filepath.Glob(filepath.Join(chunkDir, "*.part"))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
numericParts := parts[:0]
|
||||
for _, partPath := range parts {
|
||||
if _, ok := chunkPartIndex(filepath.Base(partPath)); ok {
|
||||
numericParts = append(numericParts, partPath)
|
||||
}
|
||||
}
|
||||
parts = numericParts
|
||||
if len(parts) == 0 {
|
||||
return fmt.Errorf("no chunks found")
|
||||
}
|
||||
|
||||
// 按分块索引数值排序(非字典序),避免 10.part 排在 2.part 前面
|
||||
sort.Slice(parts, func(i, j int) bool {
|
||||
idxI, _ := chunkPartIndex(filepath.Base(parts[i]))
|
||||
idxJ, _ := chunkPartIndex(filepath.Base(parts[j]))
|
||||
return idxI < idxJ
|
||||
})
|
||||
|
||||
out, err := os.Create(destPath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer out.Close()
|
||||
|
||||
for _, partPath := range parts {
|
||||
f, err := os.Open(partPath)
|
||||
if err != nil {
|
||||
return fmt.Errorf("error opening chunk %s: %v", filepath.Base(partPath), err)
|
||||
}
|
||||
if _, err := io.Copy(out, f); err != nil {
|
||||
f.Close()
|
||||
return fmt.Errorf("error reading chunk %s: %v", filepath.Base(partPath), err)
|
||||
}
|
||||
f.Close()
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// validateBackupZip 校验 ZIP 结构完整性及 komari-backup-markup 标记文件。
|
||||
func validateBackupZip(zipPath string) error {
|
||||
if err := backup.ValidateArchive(zipPath); err != nil {
|
||||
return fmt.Errorf("not a valid backup archive: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -1,50 +1,13 @@
|
||||
package admin
|
||||
|
||||
import (
|
||||
"io"
|
||||
"net/http"
|
||||
"os"
|
||||
"strings"
|
||||
|
||||
"github.com/gin-gonic/gin"
|
||||
"github.com/komari-monitor/komari/internal/plugin"
|
||||
"github.com/komari-monitor/komari/web/api"
|
||||
)
|
||||
|
||||
// UploadPlugin installs a plugin ZIP package. The request body is the raw
|
||||
// ZIP, mirroring UploadTheme.
|
||||
func UploadPlugin(c *gin.Context) {
|
||||
data, err := io.ReadAll(c.Request.Body)
|
||||
if err != nil || len(data) == 0 {
|
||||
api.RespondError(c, http.StatusBadRequest, "请选择要上传的插件文件")
|
||||
return
|
||||
}
|
||||
|
||||
tempFile, err := os.CreateTemp("", "komari-plugin-*.zip")
|
||||
if err != nil {
|
||||
api.RespondError(c, http.StatusInternalServerError, "保存文件失败: "+err.Error())
|
||||
return
|
||||
}
|
||||
tempPath := tempFile.Name()
|
||||
defer func() { _ = os.Remove(tempPath) }()
|
||||
if _, err := tempFile.Write(data); err != nil {
|
||||
_ = tempFile.Close()
|
||||
api.RespondError(c, http.StatusInternalServerError, "保存文件失败: "+err.Error())
|
||||
return
|
||||
}
|
||||
if err := tempFile.Close(); err != nil {
|
||||
api.RespondError(c, http.StatusInternalServerError, "保存文件失败: "+err.Error())
|
||||
return
|
||||
}
|
||||
|
||||
info, err := plugin.InstallZip(tempPath)
|
||||
if err != nil {
|
||||
api.RespondError(c, http.StatusBadRequest, err.Error())
|
||||
return
|
||||
}
|
||||
api.RespondSuccessMessage(c, "插件上传成功", info)
|
||||
}
|
||||
|
||||
// ServePluginFile serves a static file from an installed plugin directory,
|
||||
// used by injected plugin admin pages.
|
||||
func ServePluginFile(c *gin.Context) {
|
||||
|
||||
+12
-54
@@ -28,39 +28,6 @@ const (
|
||||
maxThemeManifestSize = 1 << 20
|
||||
)
|
||||
|
||||
// UploadTheme 上传主题
|
||||
func UploadTheme(c *gin.Context) {
|
||||
// 读取上传的文件内容
|
||||
data, err := io.ReadAll(c.Request.Body)
|
||||
if err != nil || len(data) == 0 {
|
||||
api.RespondError(c, http.StatusBadRequest, "请选择要上传的主题文件")
|
||||
return
|
||||
}
|
||||
|
||||
// 临时文件名
|
||||
tempFile := filepath.Join(os.TempDir(), "uploaded_theme.zip")
|
||||
if err := os.WriteFile(tempFile, data, 0644); err != nil {
|
||||
api.RespondError(c, http.StatusInternalServerError, "保存文件失败: "+err.Error())
|
||||
return
|
||||
}
|
||||
defer os.Remove(tempFile)
|
||||
|
||||
// 检查文件扩展名(这里假定上传的就是zip)
|
||||
if !strings.HasSuffix(strings.ToLower(tempFile), ".zip") {
|
||||
api.RespondError(c, http.StatusBadRequest, "只支持ZIP格式的主题文件")
|
||||
return
|
||||
}
|
||||
|
||||
// 解压ZIP文件并验证
|
||||
themeInfo, err := extractAndValidateTheme(tempFile)
|
||||
if err != nil {
|
||||
api.RespondError(c, http.StatusBadRequest, err.Error())
|
||||
return
|
||||
}
|
||||
|
||||
api.RespondSuccessMessage(c, "主题上传成功", themeInfo)
|
||||
}
|
||||
|
||||
// ListThemes 列出所有主题
|
||||
func ListThemes(c *gin.Context) {
|
||||
dataDir := "./data/theme"
|
||||
@@ -217,17 +184,7 @@ func extractAndValidateTheme(zipPath string) (models.Theme, error) {
|
||||
return themeInfo, fmt.Errorf("主题配置格式错误: %v", err)
|
||||
}
|
||||
|
||||
// 验证必填字段
|
||||
if themeInfo.Name == "" || themeInfo.Short == "" {
|
||||
return themeInfo, fmt.Errorf("主题配置缺少必填字段(name、short)")
|
||||
}
|
||||
|
||||
// 验证Short字段格式(只允许字母、数字、下划线、连字符)
|
||||
if !isValidMarketShort(themeInfo.Short) {
|
||||
return themeInfo, fmt.Errorf("主题short字段格式无效,只允许字母、数字、下划线和连字符")
|
||||
}
|
||||
|
||||
if err := themeInfo.ValidateConfiguration(); err != nil {
|
||||
if err := validateThemeManifest(themeInfo); err != nil {
|
||||
return themeInfo, err
|
||||
}
|
||||
|
||||
@@ -324,6 +281,16 @@ func loadThemeConfig(configPath string) (models.Theme, error) {
|
||||
return themeInfo, nil
|
||||
}
|
||||
|
||||
func validateThemeManifest(themeInfo models.Theme) error {
|
||||
if !models.IsLocalizedText(themeInfo.Name) || themeInfo.Short == "" {
|
||||
return fmt.Errorf("主题配置缺少必填字段(name、short)")
|
||||
}
|
||||
if !isValidMarketShort(themeInfo.Short) {
|
||||
return fmt.Errorf("主题short字段格式无效,只允许字母、数字、下划线和连字符")
|
||||
}
|
||||
return themeInfo.ValidateConfiguration()
|
||||
}
|
||||
|
||||
// isValidMarketShort validates a market entry short name (shared by the
|
||||
// theme and plugin markets).
|
||||
func isValidMarketShort(short string) bool {
|
||||
@@ -726,15 +693,7 @@ func peekThemeFromZip(zipPath string) (models.Theme, error) {
|
||||
return themeInfo, fmt.Errorf("主题配置格式错误: %v", err)
|
||||
}
|
||||
|
||||
if themeInfo.Name == "" || themeInfo.Short == "" {
|
||||
return themeInfo, fmt.Errorf("主题配置缺少必填字段(name、short)")
|
||||
}
|
||||
|
||||
if !isValidMarketShort(themeInfo.Short) {
|
||||
return themeInfo, fmt.Errorf("主题short字段格式无效,只允许字母、数字、下划线和连字符")
|
||||
}
|
||||
|
||||
if err := themeInfo.ValidateConfiguration(); err != nil {
|
||||
if err := validateThemeManifest(themeInfo); err != nil {
|
||||
return themeInfo, err
|
||||
}
|
||||
|
||||
@@ -840,7 +799,6 @@ func UpdateThemeSettings(c *gin.Context) {
|
||||
api.RespondError(c, http.StatusBadRequest, "主题名称不能为空")
|
||||
return
|
||||
}
|
||||
|
||||
var req map[string]any
|
||||
|
||||
err := c.ShouldBindJSON(&req)
|
||||
|
||||
@@ -16,6 +16,7 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/gin-gonic/gin"
|
||||
"github.com/komari-monitor/komari/database/models"
|
||||
"github.com/komari-monitor/komari/internal/config"
|
||||
"github.com/komari-monitor/komari/web/api"
|
||||
)
|
||||
@@ -445,17 +446,7 @@ func validateThemeMarketTheme(theme ThemeMarketTheme) error {
|
||||
// isMarketText reports whether a market entry field is usable: a non-empty
|
||||
// string or an i18n object with at least one non-empty value.
|
||||
func isMarketText(value any) bool {
|
||||
switch text := value.(type) {
|
||||
case string:
|
||||
return strings.TrimSpace(text) != ""
|
||||
case map[string]any:
|
||||
for _, item := range text {
|
||||
if itemText, ok := item.(string); ok && strings.TrimSpace(itemText) != "" {
|
||||
return true
|
||||
}
|
||||
}
|
||||
}
|
||||
return false
|
||||
return models.IsLocalizedText(value)
|
||||
}
|
||||
|
||||
func validateMarketURLSyntax(rawURL string) error {
|
||||
|
||||
@@ -1,6 +1,13 @@
|
||||
package admin
|
||||
|
||||
import "testing"
|
||||
import (
|
||||
"archive/zip"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"github.com/komari-monitor/komari/database/models"
|
||||
)
|
||||
|
||||
// TestIsValidThemeShort_PathTraversal 防止 DeleteTheme/UpdateTheme/SetTheme
|
||||
// 的路径穿越漏洞:req.Short 直接进入 filepath.Join("./data/theme", short),
|
||||
@@ -45,3 +52,82 @@ func TestIsValidThemeShort_PathTraversal(t *testing.T) {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateThemeManifestAcceptsLocalizedMetadata(t *testing.T) {
|
||||
theme := models.Theme{
|
||||
Name: map[string]any{
|
||||
"zh-CN": "配置字段演示主题",
|
||||
"en": "Managed Configuration Demo Theme",
|
||||
},
|
||||
Short: "managed-config-demo",
|
||||
Description: map[string]any{
|
||||
"zh-CN": "用于验证全部托管配置字段的测试主题。",
|
||||
"en": "A test theme covering every managed configuration field type.",
|
||||
},
|
||||
Author: map[string]any{
|
||||
"zh-CN": "Komari 团队",
|
||||
"en": "Komari",
|
||||
},
|
||||
}
|
||||
|
||||
if err := validateThemeManifest(theme); err != nil {
|
||||
t.Fatalf("localized theme metadata was rejected: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPeekThemeFromZipAcceptsLocalizedMetadata(t *testing.T) {
|
||||
zipPath := filepath.Join(t.TempDir(), "localized-theme.zip")
|
||||
archive, err := os.Create(zipPath)
|
||||
if err != nil {
|
||||
t.Fatalf("create zip: %v", err)
|
||||
}
|
||||
writer := zip.NewWriter(archive)
|
||||
manifest, err := writer.Create("komari-theme.json")
|
||||
if err != nil {
|
||||
t.Fatalf("create manifest: %v", err)
|
||||
}
|
||||
_, err = manifest.Write([]byte(`{
|
||||
"name": {"zh-CN": "本地化主题", "en": "Localized Theme"},
|
||||
"short": "localized-theme",
|
||||
"description": {"zh-CN": "描述", "en": "Description"},
|
||||
"author": {"zh-CN": "作者", "en": "Author"},
|
||||
"version": "1.0.0"
|
||||
}`))
|
||||
if err != nil {
|
||||
t.Fatalf("write manifest: %v", err)
|
||||
}
|
||||
if err := writer.Close(); err != nil {
|
||||
t.Fatalf("close zip writer: %v", err)
|
||||
}
|
||||
if err := archive.Close(); err != nil {
|
||||
t.Fatalf("close zip file: %v", err)
|
||||
}
|
||||
|
||||
theme, err := peekThemeFromZip(zipPath)
|
||||
if err != nil {
|
||||
t.Fatalf("localized theme package was rejected: %v", err)
|
||||
}
|
||||
if theme.Short != "localized-theme" {
|
||||
t.Fatalf("short = %q, want localized-theme", theme.Short)
|
||||
}
|
||||
|
||||
workDir, err := os.Getwd()
|
||||
if err != nil {
|
||||
t.Fatalf("get working directory: %v", err)
|
||||
}
|
||||
tempDir := t.TempDir()
|
||||
if err := os.Chdir(tempDir); err != nil {
|
||||
t.Fatalf("change working directory: %v", err)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
_ = os.Chdir(workDir)
|
||||
})
|
||||
|
||||
installed, err := extractAndValidateTheme(zipPath)
|
||||
if err != nil {
|
||||
t.Fatalf("localized theme upload package was rejected: %v", err)
|
||||
}
|
||||
if installed.Short != "localized-theme" {
|
||||
t.Fatalf("installed short = %q, want localized-theme", installed.Short)
|
||||
}
|
||||
}
|
||||
@@ -1,45 +0,0 @@
|
||||
package admin
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"net/http"
|
||||
"os"
|
||||
"time"
|
||||
|
||||
"github.com/gin-gonic/gin"
|
||||
logger "github.com/komari-monitor/komari/utils/log"
|
||||
"github.com/komari-monitor/komari/web/api"
|
||||
"github.com/komari-monitor/komari/web/backup"
|
||||
)
|
||||
|
||||
// UploadBackup receives a backup and applies it after the service restarts.
|
||||
func UploadBackup(c *gin.Context) {
|
||||
file, header, err := c.Request.FormFile("backup")
|
||||
if err != nil {
|
||||
api.RespondError(c, http.StatusBadRequest, fmt.Sprintf("get uploaded backup: %v", err))
|
||||
return
|
||||
}
|
||||
defer file.Close()
|
||||
restoreLock, err := backup.AcquireRestoreLock()
|
||||
if err != nil {
|
||||
api.RespondError(c, http.StatusConflict, err.Error())
|
||||
return
|
||||
}
|
||||
if err := restoreLock.SaveUploadedBackup(file, header.Filename); err != nil {
|
||||
restoreLock.Release()
|
||||
api.RespondError(c, http.StatusBadRequest, err.Error())
|
||||
return
|
||||
}
|
||||
|
||||
c.JSON(http.StatusOK, gin.H{
|
||||
"status": "success",
|
||||
"message": "Backup uploaded successfully. The service will restart and apply the backup.",
|
||||
"path": "./data/backup.zip",
|
||||
})
|
||||
go func() {
|
||||
logger.InfoArgs("admin-api", "Backup uploaded, restarting service in 2 seconds to apply on startup...")
|
||||
time.Sleep(2 * time.Second)
|
||||
restoreLock.Release()
|
||||
os.Exit(0)
|
||||
}()
|
||||
}
|
||||
+19
-13
@@ -15,12 +15,13 @@ import (
|
||||
|
||||
"github.com/gin-gonic/gin"
|
||||
"github.com/komari-monitor/komari/database/accounts"
|
||||
"github.com/komari-monitor/komari/internal/metricstore"
|
||||
"github.com/komari-monitor/komari/database/models"
|
||||
appconfig "github.com/komari-monitor/komari/internal/config"
|
||||
"github.com/komari-monitor/komari/internal/metricstore"
|
||||
logger "github.com/komari-monitor/komari/utils/log"
|
||||
"github.com/komari-monitor/komari/web/api"
|
||||
"github.com/komari-monitor/komari/web/backup"
|
||||
"github.com/komari-monitor/komari/web/upload"
|
||||
"gorm.io/gorm"
|
||||
)
|
||||
|
||||
@@ -64,7 +65,16 @@ func (c *Controller) Register(r *gin.Engine) {
|
||||
g := r.Group(APIPath, c.requireActive)
|
||||
g.GET("/status", c.status)
|
||||
g.POST("/complete", c.complete)
|
||||
g.POST("/restore", c.restore)
|
||||
uploadHandler := upload.NewHandler(upload.DefaultStore, map[upload.Purpose]upload.Finalizer{
|
||||
upload.PurposeBackup: c.finalizeBackupUpload,
|
||||
})
|
||||
uploadGroup := g.Group("/upload")
|
||||
{
|
||||
uploadGroup.POST("/init", uploadHandler.Init)
|
||||
uploadGroup.POST("/chunk", uploadHandler.Chunk)
|
||||
uploadGroup.POST("/merge", uploadHandler.Merge)
|
||||
uploadGroup.POST("/cancel", uploadHandler.Cancel)
|
||||
}
|
||||
}
|
||||
|
||||
func (c *Controller) requireActive(ctx *gin.Context) {
|
||||
@@ -82,25 +92,21 @@ func (c *Controller) status(ctx *gin.Context) {
|
||||
api.RespondSuccess(ctx, Status{State: state, Required: state != "completed"})
|
||||
}
|
||||
|
||||
// restore stages a verified backup before the guide creates any new data.
|
||||
func (c *Controller) restore(ctx *gin.Context) {
|
||||
file, header, err := ctx.Request.FormFile("backup")
|
||||
func (c *Controller) finalizeBackupUpload(session upload.Session) (upload.Result, error) {
|
||||
archive, err := os.Open(session.ArchivePath)
|
||||
if err != nil {
|
||||
api.RespondError(ctx, http.StatusBadRequest, fmt.Sprintf("get uploaded backup: %v", err))
|
||||
return
|
||||
return upload.Result{}, fmt.Errorf("open merged backup: %w", err)
|
||||
}
|
||||
defer file.Close()
|
||||
if err := backup.SaveUploadedBackup(file, header.Filename); err != nil {
|
||||
api.RespondError(ctx, http.StatusBadRequest, err.Error())
|
||||
return
|
||||
defer archive.Close()
|
||||
if err := backup.SaveUploadedBackup(archive, session.Metadata.Filename); err != nil {
|
||||
return upload.Result{}, err
|
||||
}
|
||||
|
||||
api.RespondSuccessMessage(ctx, "backup uploaded; restarting to restore", gin.H{})
|
||||
go func() {
|
||||
logger.InfoArgs("install", "Backup uploaded, restarting service to restore it on startup...")
|
||||
time.Sleep(2 * time.Second)
|
||||
os.Exit(0)
|
||||
}()
|
||||
return upload.Result{Message: "backup uploaded; restarting to restore", Data: gin.H{}}, nil
|
||||
}
|
||||
|
||||
func (c *Controller) complete(ctx *gin.Context) {
|
||||
|
||||
@@ -30,6 +30,7 @@ const (
|
||||
)
|
||||
|
||||
const largeDatasetThreshold int64 = 300_000
|
||||
const structureProgressCeiling = 80.0
|
||||
|
||||
type Mode string
|
||||
|
||||
@@ -471,7 +472,7 @@ func (c *Controller) failTarget(err error, dsn, phase string) {
|
||||
|
||||
func structureProgressPercent(progress metricstore.RestructureProgress) float64 {
|
||||
if progress.Phase == "reclaiming" {
|
||||
return 95
|
||||
return structureProgressCeiling
|
||||
}
|
||||
if progress.Phase == "discarding" && progress.RowsTotal == 0 {
|
||||
return 1
|
||||
@@ -479,12 +480,12 @@ func structureProgressPercent(progress metricstore.RestructureProgress) float64
|
||||
if progress.RowsTotal <= 0 {
|
||||
return 0
|
||||
}
|
||||
value := float64(progress.RowsDone) / float64(progress.RowsTotal) * 95
|
||||
value := float64(progress.RowsDone) / float64(progress.RowsTotal) * structureProgressCeiling
|
||||
if value < 0 {
|
||||
return 0
|
||||
}
|
||||
if value > 95 {
|
||||
return 95
|
||||
if value > structureProgressCeiling {
|
||||
return structureProgressCeiling
|
||||
}
|
||||
return value
|
||||
}
|
||||
|
||||
@@ -126,7 +126,13 @@ func TestUnifiedAuthIncludesMode(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestStructureProgressPercent(t *testing.T) {
|
||||
if got := structureProgressPercent(metricstore.RestructureProgress{RowsDone: 50, RowsTotal: 100}); got != 47.5 {
|
||||
t.Fatalf("copying percent = %v, want 47.5", got)
|
||||
if got := structureProgressPercent(metricstore.RestructureProgress{RowsDone: 50, RowsTotal: 100}); got != 40 {
|
||||
t.Fatalf("copying percent = %v, want 40", got)
|
||||
}
|
||||
if got := structureProgressPercent(metricstore.RestructureProgress{Phase: "reclaiming"}); got != 80 {
|
||||
t.Fatalf("reclaiming percent = %v, want 80", got)
|
||||
}
|
||||
if got := structureProgressPercent(metricstore.RestructureProgress{RowsDone: 100, RowsTotal: 100}); got != 80 {
|
||||
t.Fatalf("capped percent = %v, want 80", got)
|
||||
}
|
||||
}
|
||||
+10
-9
@@ -86,11 +86,14 @@ func registerAdminRoutes(r *gin.Engine) {
|
||||
|
||||
// --- 二进制/流/重定向类,保留 REST handler ---
|
||||
g.GET("/download/backup", admin.DownloadBackup)
|
||||
g.POST("/upload/backup", admin.UploadBackup)
|
||||
// chunk upload 用于大备份文件分块上传,提高稳定性
|
||||
g.POST("/upload/backup/init", admin.InitChunkUpload)
|
||||
g.POST("/upload/backup/chunk", admin.UploadChunk)
|
||||
g.POST("/upload/backup/merge", admin.MergeChunkUpload)
|
||||
uploadHandler := admin.NewArchiveUploadHandler()
|
||||
uploadGroup := g.Group("/upload")
|
||||
{
|
||||
uploadGroup.POST("/init", uploadHandler.Init)
|
||||
uploadGroup.POST("/chunk", uploadHandler.Chunk)
|
||||
uploadGroup.POST("/merge", uploadHandler.Merge)
|
||||
uploadGroup.POST("/cancel", uploadHandler.Cancel)
|
||||
}
|
||||
g.GET("/test/geoip", jsonRpc.Bind("admin:testGeoip", jsonRpc.WithQuery("ip")))
|
||||
g.POST("/test/sendMessage", jsonRpc.Bind("admin:testSendMessage"))
|
||||
g.POST("/update/mmdb", admin.UpdateMmdbGeoIP)
|
||||
@@ -98,10 +101,9 @@ func registerAdminRoutes(r *gin.Engine) {
|
||||
g.PUT("/update/favicon", admin.UploadFavicon)
|
||||
g.POST("/update/favicon", admin.DeleteFavicon)
|
||||
|
||||
// theme 含文件上传,保留 REST handler。
|
||||
// theme 的安装流程通过统一的分片上传接口;其余主题接口保留 REST handler。
|
||||
theme := g.Group("/theme")
|
||||
{
|
||||
theme.PUT("/upload", admin.UploadTheme)
|
||||
theme.GET("/list", admin.ListThemes)
|
||||
theme.POST("/delete", admin.DeleteTheme)
|
||||
theme.GET("/set", admin.SetTheme)
|
||||
@@ -205,13 +207,12 @@ func registerAdminRoutes(r *gin.Engine) {
|
||||
clipboardGroup.POST("/:id/remove", jsonRpc.Bind("admin:deleteClipboard", jsonRpc.WithPath("id")))
|
||||
}
|
||||
|
||||
// plugins: 上传走 REST(zip 二进制),启停/列表/日志走 RPC2,市场对齐主题市场。
|
||||
// plugins: 安装流程通过统一的分片上传接口,启停/列表/日志走 RPC2,市场对齐主题市场。
|
||||
pluginGroup := g.Group("/plugin")
|
||||
{
|
||||
pluginGroup.GET("/list", jsonRpc.Bind("admin:listPlugins"))
|
||||
pluginGroup.POST("/enabled", jsonRpc.Bind("admin:setPluginEnabled"))
|
||||
pluginGroup.GET("/logs", jsonRpc.Bind("admin:getPluginLogs", jsonRpc.WithQuery("short")))
|
||||
pluginGroup.POST("/install", admin.UploadPlugin)
|
||||
pluginGroup.GET("/market/sources", admin.ListPluginMarketSources)
|
||||
pluginGroup.POST("/market/sources", admin.CreatePluginMarketSource)
|
||||
pluginGroup.PUT("/market/sources/:id", admin.UpdatePluginMarketSource)
|
||||
|
||||
@@ -0,0 +1,349 @@
|
||||
package jsonrpc
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/komari-monitor/komari/database/auditlog"
|
||||
"github.com/komari-monitor/komari/database/dbcore"
|
||||
"github.com/komari-monitor/komari/internal/metricstore"
|
||||
"github.com/komari-monitor/komari/pkg/metric"
|
||||
"github.com/komari-monitor/komari/pkg/rpc"
|
||||
)
|
||||
|
||||
const (
|
||||
databaseTargetMain = "main"
|
||||
databaseTargetMetrics = "metrics"
|
||||
defaultDBQueryLimit = 1000
|
||||
maxDBQueryLimit = 10000
|
||||
)
|
||||
|
||||
type databaseQueryParams struct {
|
||||
Database *string `json:"database"`
|
||||
SQL string `json:"sql"`
|
||||
Args []any `json:"args"`
|
||||
Limit *int `json:"limit"`
|
||||
}
|
||||
|
||||
type databaseExecParams struct {
|
||||
Database *string `json:"database"`
|
||||
SQL string `json:"sql"`
|
||||
Args []any `json:"args"`
|
||||
}
|
||||
|
||||
type databaseTablesParams struct {
|
||||
Database *string `json:"database"`
|
||||
}
|
||||
|
||||
type databaseQueryResponse struct {
|
||||
Database string `json:"database"`
|
||||
Driver string `json:"driver"`
|
||||
Columns []string `json:"columns"`
|
||||
Rows [][]any `json:"rows"`
|
||||
RowCount int `json:"row_count"`
|
||||
Truncated bool `json:"truncated"`
|
||||
}
|
||||
|
||||
type databaseExecResponse struct {
|
||||
Database string `json:"database"`
|
||||
Driver string `json:"driver"`
|
||||
RowsAffected int64 `json:"rows_affected"`
|
||||
LastInsertID *int64 `json:"last_insert_id"`
|
||||
}
|
||||
|
||||
type databaseTablesResponse struct {
|
||||
Database string `json:"database"`
|
||||
Driver string `json:"driver"`
|
||||
Tables []string `json:"tables"`
|
||||
}
|
||||
|
||||
func init() {
|
||||
RegisterWithGroupAndMeta("dbQuery", rpc.RoleAdmin, adminDBQuery, &rpc.MethodMeta{
|
||||
Name: "admin:dbQuery",
|
||||
Summary: "Execute a SQL query against the main or metrics database",
|
||||
Params: []rpc.ParamMeta{
|
||||
{Name: "database", Type: "main | metrics", Description: "Database target (default main)"},
|
||||
{Name: "sql", Type: "string", Required: true, Description: "SQL query to execute"},
|
||||
{Name: "args", Type: "any[]", Description: "Positional SQL parameters"},
|
||||
{Name: "limit", Type: "number", Description: "Maximum rows to return (default 1000, max 10000)"},
|
||||
},
|
||||
Returns: "{ database: string, driver: string, columns: string[], rows: any[][], row_count: number, truncated: boolean }",
|
||||
})
|
||||
RegisterWithGroupAndMeta("dbExec", rpc.RoleAdmin, adminDBExec, &rpc.MethodMeta{
|
||||
Name: "admin:dbExec",
|
||||
Summary: "Execute a SQL statement against the main or metrics database",
|
||||
Params: []rpc.ParamMeta{
|
||||
{Name: "database", Type: "main | metrics", Description: "Database target (default main)"},
|
||||
{Name: "sql", Type: "string", Required: true, Description: "SQL statement to execute"},
|
||||
{Name: "args", Type: "any[]", Description: "Positional SQL parameters"},
|
||||
},
|
||||
Returns: "{ database: string, driver: string, rows_affected: number, last_insert_id: number | null }",
|
||||
})
|
||||
RegisterWithGroupAndMeta("dbTables", rpc.RoleAdmin, adminDBTables, &rpc.MethodMeta{
|
||||
Name: "admin:dbTables",
|
||||
Summary: "List tables in the main or metrics database",
|
||||
Params: []rpc.ParamMeta{
|
||||
{Name: "database", Type: "main | metrics", Description: "Database target (default main)"},
|
||||
},
|
||||
Returns: "{ database: string, driver: string, tables: string[] }",
|
||||
})
|
||||
}
|
||||
|
||||
func adminDBQuery(ctx context.Context, req *rpc.JsonRpcRequest) (any, *rpc.JsonRpcError) {
|
||||
params, target, statement, limit, rpcErr := parseDatabaseQueryParams(req)
|
||||
if rpcErr != nil {
|
||||
return nil, rpcErr
|
||||
}
|
||||
|
||||
response, err := queryDatabase(ctx, target, statement, params.Args, limit)
|
||||
if err != nil {
|
||||
return nil, rpc.MakeError(rpc.InternalError, "Failed to execute database query: "+err.Error(), nil)
|
||||
}
|
||||
return response, nil
|
||||
}
|
||||
|
||||
func adminDBExec(ctx context.Context, req *rpc.JsonRpcRequest) (any, *rpc.JsonRpcError) {
|
||||
var params databaseExecParams
|
||||
if err := req.BindParams(¶ms); err != nil {
|
||||
return nil, rpc.MakeError(rpc.InvalidParams, "Invalid request body: "+err.Error(), nil)
|
||||
}
|
||||
target, rpcErr := parseDatabaseTarget(params.Database)
|
||||
if rpcErr != nil {
|
||||
return nil, rpcErr
|
||||
}
|
||||
statement := strings.TrimSpace(params.SQL)
|
||||
if statement == "" {
|
||||
return nil, rpc.MakeError(rpc.InvalidParams, "sql is required", nil)
|
||||
}
|
||||
|
||||
actor, ip := auditActor(ctx)
|
||||
auditlog.Log(ip, actor, "execute database SQL on "+target, "warn")
|
||||
|
||||
result, driver, err := executeDatabase(ctx, target, statement, params.Args)
|
||||
if err != nil {
|
||||
return nil, rpc.MakeError(rpc.InternalError, "Failed to execute database statement: "+err.Error(), nil)
|
||||
}
|
||||
rowsAffected, err := result.RowsAffected()
|
||||
if err != nil {
|
||||
return nil, rpc.MakeError(rpc.InternalError, "Failed to read affected row count: "+err.Error(), nil)
|
||||
}
|
||||
|
||||
var lastInsertID *int64
|
||||
if id, err := result.LastInsertId(); err == nil {
|
||||
lastInsertID = &id
|
||||
}
|
||||
return databaseExecResponse{
|
||||
Database: target,
|
||||
Driver: string(driver),
|
||||
RowsAffected: rowsAffected,
|
||||
LastInsertID: lastInsertID,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func adminDBTables(ctx context.Context, req *rpc.JsonRpcRequest) (any, *rpc.JsonRpcError) {
|
||||
var params databaseTablesParams
|
||||
if err := req.BindParams(¶ms); err != nil {
|
||||
return nil, rpc.MakeError(rpc.InvalidParams, "Invalid request body: "+err.Error(), nil)
|
||||
}
|
||||
target, rpcErr := parseDatabaseTarget(params.Database)
|
||||
if rpcErr != nil {
|
||||
return nil, rpcErr
|
||||
}
|
||||
|
||||
response, err := listDatabaseTables(ctx, target)
|
||||
if err != nil {
|
||||
return nil, rpc.MakeError(rpc.InternalError, "Failed to list database tables: "+err.Error(), nil)
|
||||
}
|
||||
return response, nil
|
||||
}
|
||||
|
||||
func parseDatabaseQueryParams(req *rpc.JsonRpcRequest) (databaseQueryParams, string, string, int, *rpc.JsonRpcError) {
|
||||
var params databaseQueryParams
|
||||
if err := req.BindParams(¶ms); err != nil {
|
||||
return databaseQueryParams{}, "", "", 0, rpc.MakeError(rpc.InvalidParams, "Invalid request body: "+err.Error(), nil)
|
||||
}
|
||||
target, rpcErr := parseDatabaseTarget(params.Database)
|
||||
if rpcErr != nil {
|
||||
return databaseQueryParams{}, "", "", 0, rpcErr
|
||||
}
|
||||
statement := strings.TrimSpace(params.SQL)
|
||||
if statement == "" {
|
||||
return databaseQueryParams{}, "", "", 0, rpc.MakeError(rpc.InvalidParams, "sql is required", nil)
|
||||
}
|
||||
limit := defaultDBQueryLimit
|
||||
if params.Limit != nil {
|
||||
limit = *params.Limit
|
||||
if limit < 1 || limit > maxDBQueryLimit {
|
||||
return databaseQueryParams{}, "", "", 0, rpc.MakeError(rpc.InvalidParams, fmt.Sprintf("limit must be between 1 and %d", maxDBQueryLimit), nil)
|
||||
}
|
||||
}
|
||||
return params, target, statement, limit, nil
|
||||
}
|
||||
|
||||
func parseDatabaseTarget(value *string) (string, *rpc.JsonRpcError) {
|
||||
if value == nil {
|
||||
return databaseTargetMain, nil
|
||||
}
|
||||
target := strings.TrimSpace(*value)
|
||||
if target != databaseTargetMain && target != databaseTargetMetrics {
|
||||
return "", rpc.MakeError(rpc.InvalidParams, "database must be main or metrics", nil)
|
||||
}
|
||||
return target, nil
|
||||
}
|
||||
|
||||
func queryDatabase(ctx context.Context, target, statement string, args []any, limit int) (databaseQueryResponse, error) {
|
||||
rows, driver, release, err := openDatabaseRows(ctx, target, statement, args...)
|
||||
if err != nil {
|
||||
return databaseQueryResponse{}, err
|
||||
}
|
||||
defer func() {
|
||||
_ = rows.Close()
|
||||
release()
|
||||
}()
|
||||
|
||||
columns, resultRows, truncated, err := collectDatabaseRows(rows, limit)
|
||||
if err != nil {
|
||||
return databaseQueryResponse{}, err
|
||||
}
|
||||
return databaseQueryResponse{
|
||||
Database: target,
|
||||
Driver: string(driver),
|
||||
Columns: columns,
|
||||
Rows: resultRows,
|
||||
RowCount: len(resultRows),
|
||||
Truncated: truncated,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func executeDatabase(ctx context.Context, target, statement string, args []any) (sql.Result, metric.Driver, error) {
|
||||
switch target {
|
||||
case databaseTargetMain:
|
||||
db, err := dbcore.GetDBInstance().DB()
|
||||
if err != nil {
|
||||
return nil, "", err
|
||||
}
|
||||
result, err := db.ExecContext(ctx, statement, args...)
|
||||
return result, metric.DriverSQLite, err
|
||||
case databaseTargetMetrics:
|
||||
return metricstore.ExecContext(ctx, statement, args...)
|
||||
default:
|
||||
return nil, "", fmt.Errorf("unsupported database target: %s", target)
|
||||
}
|
||||
}
|
||||
|
||||
func listDatabaseTables(ctx context.Context, target string) (databaseTablesResponse, error) {
|
||||
var (
|
||||
rows *sql.Rows
|
||||
actualDriver metric.Driver
|
||||
release func()
|
||||
err error
|
||||
)
|
||||
switch target {
|
||||
case databaseTargetMain:
|
||||
statement, err := tableListSQL(metric.DriverSQLite)
|
||||
if err != nil {
|
||||
return databaseTablesResponse{}, err
|
||||
}
|
||||
rows, actualDriver, release, err = openDatabaseRows(ctx, target, statement)
|
||||
case databaseTargetMetrics:
|
||||
rows, actualDriver, release, err = metricstore.QueryForDriver(ctx, tableListSQL)
|
||||
default:
|
||||
return databaseTablesResponse{}, fmt.Errorf("unsupported database target: %s", target)
|
||||
}
|
||||
if err != nil {
|
||||
return databaseTablesResponse{}, err
|
||||
}
|
||||
defer func() {
|
||||
_ = rows.Close()
|
||||
release()
|
||||
}()
|
||||
|
||||
tables := make([]string, 0)
|
||||
for rows.Next() {
|
||||
var name string
|
||||
if err := rows.Scan(&name); err != nil {
|
||||
return databaseTablesResponse{}, err
|
||||
}
|
||||
tables = append(tables, name)
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return databaseTablesResponse{}, err
|
||||
}
|
||||
sort.Strings(tables)
|
||||
return databaseTablesResponse{Database: target, Driver: string(actualDriver), Tables: tables}, nil
|
||||
}
|
||||
|
||||
func openDatabaseRows(ctx context.Context, target, statement string, args ...any) (*sql.Rows, metric.Driver, func(), error) {
|
||||
switch target {
|
||||
case databaseTargetMain:
|
||||
db, err := dbcore.GetDBInstance().DB()
|
||||
if err != nil {
|
||||
return nil, "", nil, err
|
||||
}
|
||||
rows, err := db.QueryContext(ctx, statement, args...)
|
||||
return rows, metric.DriverSQLite, func() {}, err
|
||||
case databaseTargetMetrics:
|
||||
return metricstore.QueryContext(ctx, statement, args...)
|
||||
default:
|
||||
return nil, "", nil, fmt.Errorf("unsupported database target: %s", target)
|
||||
}
|
||||
}
|
||||
|
||||
func collectDatabaseRows(rows *sql.Rows, limit int) ([]string, [][]any, bool, error) {
|
||||
columns, err := rows.Columns()
|
||||
if err != nil {
|
||||
return nil, nil, false, err
|
||||
}
|
||||
resultRows := make([][]any, 0, limit)
|
||||
values := make([]any, len(columns))
|
||||
scans := make([]any, len(columns))
|
||||
for i := range scans {
|
||||
scans[i] = &values[i]
|
||||
}
|
||||
|
||||
for rows.Next() {
|
||||
if len(resultRows) == limit {
|
||||
return columns, resultRows, true, nil
|
||||
}
|
||||
if err := rows.Scan(scans...); err != nil {
|
||||
return nil, nil, false, err
|
||||
}
|
||||
row := make([]any, len(values))
|
||||
for i, value := range values {
|
||||
row[i] = normalizeDatabaseValue(value)
|
||||
}
|
||||
resultRows = append(resultRows, row)
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return nil, nil, false, err
|
||||
}
|
||||
return columns, resultRows, false, nil
|
||||
}
|
||||
|
||||
func normalizeDatabaseValue(value any) any {
|
||||
switch value := value.(type) {
|
||||
case []byte:
|
||||
return string(value)
|
||||
case time.Time:
|
||||
return value.Format(time.RFC3339Nano)
|
||||
default:
|
||||
return value
|
||||
}
|
||||
}
|
||||
|
||||
func tableListSQL(driver metric.Driver) (string, error) {
|
||||
switch driver {
|
||||
case metric.DriverSQLite:
|
||||
return "SELECT name FROM sqlite_master WHERE type = 'table' AND name NOT LIKE 'sqlite_%'", nil
|
||||
case metric.DriverMySQL:
|
||||
return "SELECT table_name FROM information_schema.tables WHERE table_schema = DATABASE() AND table_type = 'BASE TABLE'", nil
|
||||
case metric.DriverPostgreSQL:
|
||||
return "SELECT tablename FROM pg_catalog.pg_tables WHERE schemaname = current_schema()", nil
|
||||
default:
|
||||
return "", fmt.Errorf("unsupported database driver: %s", driver)
|
||||
}
|
||||
}
|
||||
@@ -113,5 +113,8 @@ func adminSetPluginConfiguration(_ context.Context, req *rpc.JsonRpcRequest) (an
|
||||
if err := plugin.SaveConfiguration(short, data); err != nil {
|
||||
return nil, rpc.MakeError(rpc.InternalError, err.Error(), nil)
|
||||
}
|
||||
if err := plugin.Reload(short); err != nil {
|
||||
return nil, rpc.MakeError(rpc.InternalError, "plugin configuration saved but reload failed: "+err.Error(), nil)
|
||||
}
|
||||
return nil, nil
|
||||
}
|
||||
@@ -4,6 +4,7 @@ import (
|
||||
"archive/zip"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/komari-monitor/komari/cmd/flags"
|
||||
@@ -93,6 +94,35 @@ func TestAdminPluginConfigurationRoundTrip(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestAdminPluginConfigurationKeepsSavedDataWhenReloadFails(t *testing.T) {
|
||||
withTempPluginDir(t)
|
||||
installTestPlugin(t, `{"name":"Cfg","short":"cfg","version":"1.0.0","configuration":{"type":"managed","data":[{"key":"greeting","name":"Greeting","type":"string","default":"hi"}]}}`, "function load() {}")
|
||||
|
||||
if err := plugin.SetEnabled("cfg", true, false); err != nil {
|
||||
t.Fatalf("enable failed: %v", err)
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(plugin.DataDir, "cfg", "script.js"), []byte(`function load() { throw new Error("reload-boom"); }`), 0644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
_, jerr := adminSetPluginConfiguration(nil, &rpc.JsonRpcRequest{
|
||||
Version: rpc.RPC_VERSION,
|
||||
Method: "admin:setPluginConfiguration",
|
||||
Params: map[string]any{"short": "cfg", "data": map[string]any{"greeting": "saved-before-reload-error"}},
|
||||
})
|
||||
if jerr == nil || !strings.Contains(jerr.Message, "plugin configuration saved but reload failed") {
|
||||
t.Fatalf("expected reload error after saving, got %v", jerr)
|
||||
}
|
||||
|
||||
values, err := plugin.GetConfiguration("cfg")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if values["greeting"] != "saved-before-reload-error" {
|
||||
t.Fatalf("saved configuration was rolled back: %#v", values)
|
||||
}
|
||||
}
|
||||
|
||||
// withTempPluginDir redirects plugin.DataDir for the duration of a test.
|
||||
func withTempPluginDir(t *testing.T) {
|
||||
t.Helper()
|
||||
|
||||
+231
-176
@@ -10,9 +10,9 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/komari-monitor/komari/database/clients"
|
||||
"github.com/komari-monitor/komari/internal/metricstore"
|
||||
"github.com/komari-monitor/komari/database/models"
|
||||
"github.com/komari-monitor/komari/database/tasks"
|
||||
"github.com/komari-monitor/komari/internal/metricstore"
|
||||
"github.com/komari-monitor/komari/pkg/metric"
|
||||
"github.com/komari-monitor/komari/pkg/rpc"
|
||||
)
|
||||
@@ -54,11 +54,12 @@ type publicMetricQueryParams struct {
|
||||
}
|
||||
|
||||
type publicMetricPoint struct {
|
||||
Time time.Time `json:"time"`
|
||||
Value *float64 `json:"value"`
|
||||
Count int `json:"count,omitempty"`
|
||||
Tags map[string]string `json:"tags,omitempty"`
|
||||
Labels map[string]string `json:"labels,omitempty"`
|
||||
entityID string
|
||||
Time time.Time `json:"time"`
|
||||
Value *float64 `json:"value"`
|
||||
Count int `json:"count,omitempty"`
|
||||
Tags map[string]string `json:"tags,omitempty"`
|
||||
Labels map[string]string `json:"labels,omitempty"`
|
||||
}
|
||||
|
||||
type publicMetricSeries struct {
|
||||
@@ -167,7 +168,8 @@ func publicQueryMetrics(ctx context.Context, req *rpc.JsonRpcRequest) (any, *rpc
|
||||
return nil, rpc.MakeError(rpc.InvalidParams, "end must be after start", nil)
|
||||
}
|
||||
|
||||
entityIDs, rpcErr := publicMetricEntityIDs(ctx, normalizeStringList(params.EntityIDs, []string{params.EntityID}))
|
||||
requestedEntityIDs := normalizeStringList(params.EntityIDs, []string{params.EntityID})
|
||||
entityIDs, rpcErr := publicMetricEntityIDs(ctx, requestedEntityIDs)
|
||||
if rpcErr != nil {
|
||||
return nil, rpcErr
|
||||
}
|
||||
@@ -176,59 +178,154 @@ func publicQueryMetrics(ctx context.Context, req *rpc.JsonRpcRequest) (any, *rpc
|
||||
if store == nil {
|
||||
return nil, rpc.MakeError(rpc.InternalError, "metric store not initialized", nil)
|
||||
}
|
||||
defs, err := store.ListMetrics(ctx)
|
||||
if err != nil {
|
||||
return nil, rpc.MakeError(rpc.InternalError, "Failed to list metric definitions: "+err.Error(), nil)
|
||||
}
|
||||
defMap := make(map[string]metric.Definition, len(defs))
|
||||
for _, def := range defs {
|
||||
defMap[def.Name] = def
|
||||
}
|
||||
|
||||
series := make([]publicMetricSeries, 0, len(metricKeys)*maxInt(1, len(entityIDs)))
|
||||
type metricLoadSpec struct {
|
||||
metricKey string
|
||||
algorithm metric.Aggregation
|
||||
maxPoints int
|
||||
interval time.Duration
|
||||
}
|
||||
loadSpecs := make([]metricLoadSpec, 0, len(metricKeys))
|
||||
for _, metricKey := range metricKeys {
|
||||
def, ok := defMap[metricKey]
|
||||
if !ok {
|
||||
return nil, rpc.MakeError(rpc.InvalidParams, "unknown metric key: "+metricKey, nil)
|
||||
}
|
||||
maxPoints, err := resolveMetricMaxPoints(metricKey, params)
|
||||
if err != nil {
|
||||
return nil, rpc.MakeError(rpc.InvalidParams, err.Error(), nil)
|
||||
}
|
||||
algorithm := resolveMetricAggregation(metricKey, params)
|
||||
metricFillEmpty := resolveMetricFillEmpty(params)
|
||||
loadSpecs = append(loadSpecs, metricLoadSpec{
|
||||
metricKey: metricKey,
|
||||
algorithm: resolveMetricAggregation(metricKey, params),
|
||||
maxPoints: maxPoints,
|
||||
})
|
||||
}
|
||||
|
||||
metricFillEmpty := resolveMetricFillEmpty(params)
|
||||
useRaw := publicMetricUsesRawWindow(start, end, queryNow)
|
||||
definitions := make(map[string]metric.Definition, len(metricKeys))
|
||||
rawValues := make(map[string][]metric.Point)
|
||||
rollupValues := make(map[string]map[metric.Aggregation][]metric.AggregatePoint)
|
||||
if len(entityIDs) > 0 && useRaw {
|
||||
var err error
|
||||
definitions, err = store.GetMetrics(ctx, metricKeys)
|
||||
if err != nil {
|
||||
return nil, rpc.MakeError(rpc.InternalError, "Failed to query metric definitions: "+err.Error(), nil)
|
||||
}
|
||||
for _, spec := range loadSpecs {
|
||||
if _, ok := definitions[spec.metricKey]; !ok {
|
||||
return nil, rpc.MakeError(rpc.InvalidParams, "unknown metric key: "+spec.metricKey, nil)
|
||||
}
|
||||
}
|
||||
rawValues, err = store.QueryBatch(ctx, metric.BatchQuery{
|
||||
MetricNames: metricKeys,
|
||||
EntityIDs: entityIDs,
|
||||
Start: start,
|
||||
End: end,
|
||||
Tags: params.Tags,
|
||||
Order: metric.OrderAsc,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, rpc.MakeError(rpc.InvalidParams, "Failed to query metrics: "+err.Error(), nil)
|
||||
}
|
||||
} else if len(entityIDs) > 0 {
|
||||
batchSpecs := make([]metric.BatchSeriesSpec, 0, len(loadSpecs))
|
||||
for i := range loadSpecs {
|
||||
loadSpecs[i].interval = metricDownsampleInterval(end.Sub(start), loadSpecs[i].maxPoints)
|
||||
loadSpecs[i].interval = store.CompatibleSeriesInterval(start, queryNow, loadSpecs[i].interval)
|
||||
batchSpecs = append(batchSpecs, metric.BatchSeriesSpec{
|
||||
MetricName: loadSpecs[i].metricKey,
|
||||
Aggregations: []metric.Aggregation{loadSpecs[i].algorithm},
|
||||
Interval: loadSpecs[i].interval,
|
||||
PreserveSeries: true,
|
||||
})
|
||||
}
|
||||
loaded, err := store.SeriesBatch(ctx, metric.BatchSeriesQuery{
|
||||
Specs: batchSpecs,
|
||||
EntityIDs: entityIDs,
|
||||
Start: start,
|
||||
End: end,
|
||||
Tags: params.Tags,
|
||||
Order: metric.OrderAsc,
|
||||
}, queryNow)
|
||||
if err != nil {
|
||||
return nil, rpc.MakeError(rpc.InvalidParams, "Failed to query metrics: "+err.Error(), nil)
|
||||
}
|
||||
definitions = loaded.Definitions
|
||||
rollupValues = loaded.Values
|
||||
for _, spec := range loadSpecs {
|
||||
if _, ok := definitions[spec.metricKey]; !ok {
|
||||
return nil, rpc.MakeError(rpc.InvalidParams, "unknown metric key: "+spec.metricKey, nil)
|
||||
}
|
||||
}
|
||||
} else {
|
||||
var err error
|
||||
definitions, err = store.GetMetrics(ctx, metricKeys)
|
||||
if err != nil {
|
||||
return nil, rpc.MakeError(rpc.InternalError, "Failed to query metric definitions: "+err.Error(), nil)
|
||||
}
|
||||
for _, spec := range loadSpecs {
|
||||
if _, ok := definitions[spec.metricKey]; !ok {
|
||||
return nil, rpc.MakeError(rpc.InvalidParams, "unknown metric key: "+spec.metricKey, nil)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
series := make([]publicMetricSeries, 0, len(metricKeys)*maxInt(1, len(entityIDs)))
|
||||
for _, spec := range loadSpecs {
|
||||
def := definitions[spec.metricKey]
|
||||
item := publicMetricSeries{
|
||||
MetricKey: spec.metricKey,
|
||||
Type: string(def.Type),
|
||||
Unit: def.Unit,
|
||||
RetentionDays: def.RetentionDays,
|
||||
Tags: params.Tags,
|
||||
FillEmpty: metricFillEmpty,
|
||||
MaxPoints: spec.maxPoints,
|
||||
Downsampled: !useRaw,
|
||||
}
|
||||
if useRaw {
|
||||
points := rawValues[spec.metricKey]
|
||||
item.Points = make([]publicMetricPoint, 0, len(points))
|
||||
for _, point := range points {
|
||||
item.Points = append(item.Points, publicMetricPoint{
|
||||
entityID: point.EntityID,
|
||||
Time: point.Timestamp.UTC(),
|
||||
Value: publicRawMetricValue(point.MetricName, point.Value, metricFillEmpty),
|
||||
Count: 1,
|
||||
Tags: point.Tags,
|
||||
Labels: point.Labels,
|
||||
})
|
||||
}
|
||||
} else {
|
||||
item.DownsampleAlgorithm = string(spec.algorithm)
|
||||
item.IntervalSeconds = spec.interval.Seconds()
|
||||
points := rollupValues[spec.metricKey][spec.algorithm]
|
||||
item.Points = make([]publicMetricPoint, 0, len(points))
|
||||
for _, point := range points {
|
||||
item.Points = append(item.Points, publicMetricPoint{
|
||||
entityID: point.EntityID,
|
||||
Time: point.Bucket.UTC(),
|
||||
Value: publicRawMetricValue(point.MetricName, point.Value, metricFillEmpty),
|
||||
Count: point.Count,
|
||||
Tags: point.Tags,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
byEntity := make(map[string][]publicMetricSeries, len(entityIDs))
|
||||
for _, split := range splitPublicMetricSeries(item) {
|
||||
if split.EntityID != "" {
|
||||
byEntity[split.EntityID] = append(byEntity[split.EntityID], split)
|
||||
}
|
||||
}
|
||||
for _, entityID := range entityIDs {
|
||||
query := metric.Query{
|
||||
MetricName: metricKey,
|
||||
EntityID: entityID,
|
||||
Start: start,
|
||||
End: end,
|
||||
Tags: params.Tags,
|
||||
Order: metric.OrderAsc,
|
||||
entitySeries := byEntity[entityID]
|
||||
if len(entitySeries) == 0 {
|
||||
empty := item
|
||||
empty.EntityID = entityID
|
||||
empty.Count = 0
|
||||
empty.Points = nil
|
||||
entitySeries = []publicMetricSeries{empty}
|
||||
}
|
||||
item := publicMetricSeries{
|
||||
MetricKey: metricKey,
|
||||
EntityID: entityID,
|
||||
Type: string(def.Type),
|
||||
Unit: def.Unit,
|
||||
RetentionDays: def.RetentionDays,
|
||||
Tags: params.Tags,
|
||||
FillEmpty: metricFillEmpty,
|
||||
MaxPoints: maxPoints,
|
||||
}
|
||||
|
||||
loaded, err := loadPublicMetricPoints(ctx, store, query, algorithm, maxPoints, metricFillEmpty, queryNow)
|
||||
if err != nil {
|
||||
return nil, rpc.MakeError(rpc.InvalidParams, "Failed to query metric "+metricKey+": "+err.Error(), nil)
|
||||
}
|
||||
item.Downsampled = loaded.downsampled
|
||||
item.Points = loaded.points
|
||||
if loaded.downsampled {
|
||||
item.DownsampleAlgorithm = string(algorithm)
|
||||
item.IntervalSeconds = loaded.interval.Seconds()
|
||||
}
|
||||
for _, split := range splitPublicMetricSeries(item) {
|
||||
for _, split := range entitySeries {
|
||||
if metricFillEmpty {
|
||||
split = adaptiveFillPublicMetricSeries(split, start, end)
|
||||
}
|
||||
@@ -263,62 +360,21 @@ func loadPublicMetricPoints(
|
||||
now time.Time,
|
||||
) (publicMetricPointResult, error) {
|
||||
if publicMetricUsesRawWindow(query.Start, query.End, now) {
|
||||
fallbackInterval := store.CompatibleSeriesInterval(query.Start, now, time.Minute)
|
||||
// Read rollups first so a concurrent write is observed by the later raw
|
||||
// read and suppresses the older summary for that bucket.
|
||||
fallback, err := store.Series(ctx, metric.AggregateQuery{
|
||||
Query: query,
|
||||
Aggregation: algorithm,
|
||||
Interval: fallbackInterval,
|
||||
PreserveSeries: true,
|
||||
}, now)
|
||||
if err != nil {
|
||||
return publicMetricPointResult{}, err
|
||||
}
|
||||
points, err := store.Query(ctx, query)
|
||||
if err != nil {
|
||||
return publicMetricPointResult{}, err
|
||||
}
|
||||
result := publicMetricPointResult{points: make([]publicMetricPoint, 0, len(points))}
|
||||
rawBuckets := make(map[string]struct{}, len(points))
|
||||
for _, point := range points {
|
||||
publicPoint := publicMetricPoint{
|
||||
Time: point.Timestamp.UTC(),
|
||||
Value: publicRawMetricValue(point.MetricName, point.Value, fillEmpty),
|
||||
Count: 1,
|
||||
Tags: point.Tags,
|
||||
Labels: point.Labels,
|
||||
}
|
||||
result.points = append(result.points, publicPoint)
|
||||
rawBuckets[publicMetricBucketKey(publicPoint, fallbackInterval)] = struct{}{}
|
||||
result.points = append(result.points, publicMetricPoint{
|
||||
entityID: point.EntityID,
|
||||
Time: point.Timestamp.UTC(),
|
||||
Value: publicRawMetricValue(point.MetricName, point.Value, fillEmpty),
|
||||
Count: 1,
|
||||
Tags: point.Tags,
|
||||
Labels: point.Labels,
|
||||
})
|
||||
}
|
||||
|
||||
// Exact samples disappear on restart, while the minute summaries remain
|
||||
// persisted. Fill only buckets that have no exact sample so a restart does
|
||||
// not create a false gap or duplicate a bucket that is still in memory.
|
||||
for _, point := range fallback {
|
||||
publicPoint := publicMetricPoint{
|
||||
Time: point.Bucket.UTC(),
|
||||
Value: publicRawMetricValue(point.MetricName, point.Value, fillEmpty),
|
||||
Count: point.Count,
|
||||
Tags: point.Tags,
|
||||
}
|
||||
if publicPoint.Time.Before(query.Start) || publicPoint.Time.Add(fallbackInterval).After(query.End) {
|
||||
continue
|
||||
}
|
||||
if _, exact := rawBuckets[publicMetricBucketKey(publicPoint, fallbackInterval)]; exact {
|
||||
continue
|
||||
}
|
||||
result.points = append(result.points, publicPoint)
|
||||
result.downsampled = true
|
||||
result.interval = fallbackInterval
|
||||
}
|
||||
sort.SliceStable(result.points, func(i, j int) bool {
|
||||
if result.points[i].Time.Equal(result.points[j].Time) {
|
||||
return publicMetricTagsKey(result.points[i].Tags) < publicMetricTagsKey(result.points[j].Tags)
|
||||
}
|
||||
return result.points[i].Time.Before(result.points[j].Time)
|
||||
})
|
||||
return result, nil
|
||||
}
|
||||
|
||||
@@ -340,10 +396,11 @@ func loadPublicMetricPoints(
|
||||
}
|
||||
for _, point := range points {
|
||||
result.points = append(result.points, publicMetricPoint{
|
||||
Time: point.Bucket.UTC(),
|
||||
Value: publicRawMetricValue(point.MetricName, point.Value, fillEmpty),
|
||||
Count: point.Count,
|
||||
Tags: point.Tags,
|
||||
entityID: point.EntityID,
|
||||
Time: point.Bucket.UTC(),
|
||||
Value: publicRawMetricValue(point.MetricName, point.Value, fillEmpty),
|
||||
Count: point.Count,
|
||||
Tags: point.Tags,
|
||||
})
|
||||
}
|
||||
return result, nil
|
||||
@@ -354,11 +411,6 @@ func publicMetricUsesRawWindow(start, end, now time.Time) bool {
|
||||
return end.Sub(start) <= retention && end.After(now.UTC().Add(-retention))
|
||||
}
|
||||
|
||||
func publicMetricBucketKey(point publicMetricPoint, interval time.Duration) string {
|
||||
bucket := point.Time.UTC().Truncate(interval).UnixMilli()
|
||||
return strconv.FormatInt(bucket, 10) + "\x00" + publicMetricTagsKey(point.Tags)
|
||||
}
|
||||
|
||||
func publicGetPingMetricStats(ctx context.Context, req *rpc.JsonRpcRequest) (any, *rpc.JsonRpcError) {
|
||||
var params publicPingMetricStatsParams
|
||||
if err := req.BindParams(¶ms); err != nil {
|
||||
@@ -409,14 +461,13 @@ func publicGetPingMetricStats(ctx context.Context, req *rpc.JsonRpcRequest) (any
|
||||
interval := metricDownsampleInterval(end.Sub(start), maxPoints)
|
||||
interval = store.CompatibleSeriesInterval(start, now, interval)
|
||||
|
||||
groupsByEntity, err := loadPublicPingMetricAggregateGroups(ctx, store, entityIDs, start, end, interval, now)
|
||||
if err != nil {
|
||||
return nil, rpc.MakeError(rpc.InternalError, "Failed to query ping stats: "+err.Error(), nil)
|
||||
}
|
||||
stats := make([]publicPingMetricTaskStats, 0)
|
||||
for _, entityID := range entityIDs {
|
||||
groups, err := loadPublicPingMetricAggregateGroups(ctx, store, entityID, start, end, interval, now)
|
||||
if err != nil {
|
||||
return nil, rpc.MakeError(rpc.InternalError, "Failed to query ping stats: "+err.Error(), nil)
|
||||
}
|
||||
entityStats := publicPingStatsFromAggregateGroups(entityID, groups, taskMap, taskFilter)
|
||||
stats = append(stats, entityStats...)
|
||||
stats = append(stats, publicPingStatsFromAggregateGroups(entityID, groupsByEntity[entityID], taskMap, taskFilter)...)
|
||||
}
|
||||
|
||||
sort.Slice(stats, func(i, j int) bool {
|
||||
@@ -452,6 +503,9 @@ func splitPublicMetricSeries(base publicMetricSeries) []publicMetricSeries {
|
||||
order := make([]string, 0)
|
||||
for _, point := range base.Points {
|
||||
entityID := base.EntityID
|
||||
if point.entityID != "" {
|
||||
entityID = point.entityID
|
||||
}
|
||||
tags := point.Tags
|
||||
point.Tags = tags
|
||||
tagsKey := publicMetricTagsKey(tags)
|
||||
@@ -724,77 +778,78 @@ type publicPingMetricAggregateGroups struct {
|
||||
LossAvailable bool
|
||||
}
|
||||
|
||||
func loadPublicPingMetricAggregateGroups(ctx context.Context, store *metric.Store, entityID string, start, end time.Time, interval time.Duration, now time.Time) (publicPingMetricAggregateGroups, error) {
|
||||
query := func(metricName string, aggregation metric.Aggregation) (map[string][]metric.AggregatePoint, error) {
|
||||
points, err := store.Series(ctx, metric.AggregateQuery{
|
||||
Query: metric.Query{
|
||||
MetricName: metricName,
|
||||
EntityID: entityID,
|
||||
Start: start,
|
||||
End: end,
|
||||
Order: metric.OrderAsc,
|
||||
},
|
||||
Aggregation: aggregation,
|
||||
Interval: interval,
|
||||
PreserveSeries: true,
|
||||
}, now)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
func loadPublicPingMetricAggregateGroups(ctx context.Context, store *metric.Store, entityIDs []string, start, end time.Time, interval time.Duration, now time.Time) (map[string]publicPingMetricAggregateGroups, error) {
|
||||
latencyAggregations := []metric.Aggregation{
|
||||
metric.AggAvg,
|
||||
metric.AggMin,
|
||||
metric.AggMax,
|
||||
metric.AggLast,
|
||||
metric.AggP50,
|
||||
metric.AggP99,
|
||||
metric.AggStdDev,
|
||||
}
|
||||
loaded, err := store.SeriesBatch(ctx, metric.BatchSeriesQuery{
|
||||
Specs: []metric.BatchSeriesSpec{
|
||||
{MetricName: metricstore.MetricPingLatency, Aggregations: latencyAggregations, Interval: interval, PreserveSeries: true},
|
||||
{MetricName: metricstore.MetricPingLoss, Aggregations: []metric.Aggregation{metric.AggAvg}, Interval: interval, PreserveSeries: true},
|
||||
},
|
||||
EntityIDs: entityIDs,
|
||||
Start: start,
|
||||
End: end,
|
||||
Order: metric.OrderAsc,
|
||||
}, now)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
latency := loaded.Values[metricstore.MetricPingLatency]
|
||||
lossPoints := loaded.Values[metricstore.MetricPingLoss][metric.AggAvg]
|
||||
|
||||
avg := groupPingMetricAggregatePointsByEntity(latency[metric.AggAvg])
|
||||
minimum := groupPingMetricAggregatePointsByEntity(latency[metric.AggMin])
|
||||
maximum := groupPingMetricAggregatePointsByEntity(latency[metric.AggMax])
|
||||
last := groupPingMetricAggregatePointsByEntity(latency[metric.AggLast])
|
||||
p50 := groupPingMetricAggregatePointsByEntity(latency[metric.AggP50])
|
||||
p99 := groupPingMetricAggregatePointsByEntity(latency[metric.AggP99])
|
||||
stddev := groupPingMetricAggregatePointsByEntity(latency[metric.AggStdDev])
|
||||
loss := groupPingMetricAggregatePointsByEntity(lossPoints)
|
||||
|
||||
entitySet := make(map[string]struct{})
|
||||
for _, groups := range []map[string]map[string][]metric.AggregatePoint{avg, minimum, maximum, last, p50, p99, stddev, loss} {
|
||||
for currentEntityID := range groups {
|
||||
entitySet[currentEntityID] = struct{}{}
|
||||
}
|
||||
return groupPingMetricAggregatePointsByTask(points), nil
|
||||
}
|
||||
|
||||
avg, err := query(metricstore.MetricPingLatency, metric.AggAvg)
|
||||
if err != nil {
|
||||
return publicPingMetricAggregateGroups{}, err
|
||||
result := make(map[string]publicPingMetricAggregateGroups, len(entitySet))
|
||||
for currentEntityID := range entitySet {
|
||||
entityLoss := loss[currentEntityID]
|
||||
result[currentEntityID] = publicPingMetricAggregateGroups{
|
||||
Avg: avg[currentEntityID],
|
||||
Min: minimum[currentEntityID],
|
||||
Max: maximum[currentEntityID],
|
||||
Last: last[currentEntityID],
|
||||
P50: p50[currentEntityID],
|
||||
P99: p99[currentEntityID],
|
||||
StdDev: stddev[currentEntityID],
|
||||
Loss: entityLoss,
|
||||
LossAvailable: pingMetricGroupsHaveData(entityLoss),
|
||||
}
|
||||
}
|
||||
minimum, err := query(metricstore.MetricPingLatency, metric.AggMin)
|
||||
if err != nil {
|
||||
return publicPingMetricAggregateGroups{}, err
|
||||
}
|
||||
maximum, err := query(metricstore.MetricPingLatency, metric.AggMax)
|
||||
if err != nil {
|
||||
return publicPingMetricAggregateGroups{}, err
|
||||
}
|
||||
last, err := query(metricstore.MetricPingLatency, metric.AggLast)
|
||||
if err != nil {
|
||||
return publicPingMetricAggregateGroups{}, err
|
||||
}
|
||||
p50, err := query(metricstore.MetricPingLatency, metric.AggP50)
|
||||
if err != nil {
|
||||
return publicPingMetricAggregateGroups{}, err
|
||||
}
|
||||
p99, err := query(metricstore.MetricPingLatency, metric.AggP99)
|
||||
if err != nil {
|
||||
return publicPingMetricAggregateGroups{}, err
|
||||
}
|
||||
stddev, err := query(metricstore.MetricPingLatency, metric.AggStdDev)
|
||||
if err != nil {
|
||||
return publicPingMetricAggregateGroups{}, err
|
||||
}
|
||||
|
||||
loss, lossErr := query(metricstore.MetricPingLoss, metric.AggAvg)
|
||||
return publicPingMetricAggregateGroups{
|
||||
Avg: avg,
|
||||
Min: minimum,
|
||||
Max: maximum,
|
||||
Last: last,
|
||||
P50: p50,
|
||||
P99: p99,
|
||||
StdDev: stddev,
|
||||
Loss: loss,
|
||||
LossAvailable: lossErr == nil && pingMetricGroupsHaveData(loss),
|
||||
}, nil
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func groupPingMetricAggregatePointsByTask(points []metric.AggregatePoint) map[string][]metric.AggregatePoint {
|
||||
out := make(map[string][]metric.AggregatePoint)
|
||||
func groupPingMetricAggregatePointsByEntity(points []metric.AggregatePoint) map[string]map[string][]metric.AggregatePoint {
|
||||
out := make(map[string]map[string][]metric.AggregatePoint)
|
||||
for _, point := range points {
|
||||
taskID := strings.TrimSpace(point.Tags["task_id"])
|
||||
if taskID == "" {
|
||||
continue
|
||||
}
|
||||
out[taskID] = append(out[taskID], point)
|
||||
byTask := out[point.EntityID]
|
||||
if byTask == nil {
|
||||
byTask = make(map[string][]metric.AggregatePoint)
|
||||
out[point.EntityID] = byTask
|
||||
}
|
||||
byTask[taskID] = append(byTask[taskID], point)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
@@ -9,8 +9,8 @@ import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/komari-monitor/komari/internal/metricstore"
|
||||
"github.com/komari-monitor/komari/database/models"
|
||||
"github.com/komari-monitor/komari/internal/metricstore"
|
||||
"github.com/komari-monitor/komari/pkg/metric"
|
||||
"github.com/komari-monitor/komari/pkg/rpc"
|
||||
)
|
||||
@@ -411,7 +411,7 @@ func TestPublicMetricUsesRawWindowOnlyForCurrentlyRetainedRange(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadPublicMetricPointsUsesRollupsForRestartGapWithoutDuplicatingRaw(t *testing.T) {
|
||||
func TestLoadPublicMetricPointsReturnsOnlyRawAfterRestart(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
dsn := filepath.Join(t.TempDir(), "metrics.db")
|
||||
policy := metric.RollupPolicy{
|
||||
@@ -473,28 +473,16 @@ func TestLoadPublicMetricPointsUsesRollupsForRestartGapWithoutDuplicatingRaw(t *
|
||||
if err != nil {
|
||||
t.Fatalf("load mixed restart window: %v", err)
|
||||
}
|
||||
if !got.downsampled || got.interval != time.Minute {
|
||||
t.Fatalf("restart fallback metadata = %#v", got)
|
||||
if got.downsampled || got.interval != 0 {
|
||||
t.Fatalf("raw query metadata = %#v", got)
|
||||
}
|
||||
if len(got.points) != 3 {
|
||||
t.Fatalf("restart window returned %d points, want two rollups and one raw: %#v", len(got.points), got.points)
|
||||
if len(got.points) != 1 {
|
||||
t.Fatalf("restart window returned %d points, want one raw point: %#v", len(got.points), got.points)
|
||||
}
|
||||
values := []float64{1, 2, 3}
|
||||
for i, point := range got.points {
|
||||
if point.Value == nil || *point.Value != values[i] {
|
||||
t.Fatalf("point %d = %#v, want value %v", i, point, values[i])
|
||||
}
|
||||
if got.points[0].Value == nil || *got.points[0].Value != 3 {
|
||||
t.Fatalf("raw point = %#v, want value 3", got.points[0])
|
||||
}
|
||||
if got.points[2].Count != 1 || got.points[2].Labels["source"] != "raw" {
|
||||
t.Fatalf("post-restart exact point changed: %#v", got.points[2])
|
||||
}
|
||||
|
||||
// The current hot rollup covers the same minute as the exact point. It must
|
||||
// be omitted rather than returned as a fourth, duplicate observation.
|
||||
currentBucket := afterRestart.Timestamp.Truncate(time.Minute)
|
||||
for _, point := range got.points {
|
||||
if point.Time.Equal(currentBucket) {
|
||||
t.Fatalf("current raw minute also returned its rollup: %#v", got.points)
|
||||
}
|
||||
if got.points[0].Count != 1 || got.points[0].Labels["source"] != "raw" {
|
||||
t.Fatalf("post-restart exact point changed: %#v", got.points[0])
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,228 @@
|
||||
// Package upload provides the shared chunked archive upload flow.
|
||||
package upload
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
|
||||
"github.com/google/uuid"
|
||||
"github.com/komari-monitor/komari/web/backup"
|
||||
)
|
||||
|
||||
const ChunkSize int64 = 5 * 1024 * 1024
|
||||
|
||||
type Purpose string
|
||||
|
||||
const (
|
||||
PurposeBackup Purpose = "backup"
|
||||
PurposePlugin Purpose = "plugin"
|
||||
PurposeTheme Purpose = "theme"
|
||||
)
|
||||
|
||||
var ErrNotFound = errors.New("upload not found")
|
||||
|
||||
type Metadata struct {
|
||||
Purpose Purpose `json:"purpose"`
|
||||
Size int64 `json:"size"`
|
||||
Filename string `json:"filename"`
|
||||
}
|
||||
|
||||
type Session struct {
|
||||
ID string
|
||||
Metadata Metadata
|
||||
Directory string
|
||||
ArchivePath string
|
||||
}
|
||||
|
||||
type Store struct {
|
||||
Root string
|
||||
MaxSize int64
|
||||
}
|
||||
|
||||
var DefaultStore = &Store{
|
||||
Root: filepath.Join(".", "data", ".uploading"),
|
||||
MaxSize: backup.MaxArchiveSize,
|
||||
}
|
||||
|
||||
func (s *Store) Init(purpose Purpose, filename string, size int64) (Session, error) {
|
||||
if !isKnownPurpose(purpose) {
|
||||
return Session{}, fmt.Errorf("invalid upload purpose")
|
||||
}
|
||||
if size <= 0 || size > s.MaxSize {
|
||||
return Session{}, fmt.Errorf("size must be between 1 and %d bytes", s.MaxSize)
|
||||
}
|
||||
|
||||
id := uuid.NewString()
|
||||
directory := filepath.Join(s.Root, id)
|
||||
if err := os.MkdirAll(directory, 0755); err != nil {
|
||||
return Session{}, fmt.Errorf("create upload directory: %w", err)
|
||||
}
|
||||
metadata := Metadata{Purpose: purpose, Size: size, Filename: filename}
|
||||
data, err := json.Marshal(metadata)
|
||||
if err != nil {
|
||||
_ = os.RemoveAll(directory)
|
||||
return Session{}, fmt.Errorf("encode upload metadata: %w", err)
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(directory, "upload.json"), data, 0600); err != nil {
|
||||
_ = os.RemoveAll(directory)
|
||||
return Session{}, fmt.Errorf("write upload metadata: %w", err)
|
||||
}
|
||||
return Session{ID: id, Metadata: metadata, Directory: directory}, nil
|
||||
}
|
||||
|
||||
func (s *Store) SaveChunk(uploadID string, index int64, source io.Reader) error {
|
||||
session, err := s.load(uploadID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
expectedSize, err := expectedChunkSize(session.Metadata.Size, index)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
temporary, err := os.CreateTemp(session.Directory, fmt.Sprintf(".%d-*.part", index))
|
||||
if err != nil {
|
||||
return fmt.Errorf("create chunk: %w", err)
|
||||
}
|
||||
temporaryPath := temporary.Name()
|
||||
defer os.Remove(temporaryPath)
|
||||
|
||||
written, copyErr := io.Copy(temporary, io.LimitReader(source, ChunkSize+1))
|
||||
closeErr := temporary.Close()
|
||||
if copyErr != nil {
|
||||
return fmt.Errorf("write chunk: %w", copyErr)
|
||||
}
|
||||
if closeErr != nil {
|
||||
return fmt.Errorf("close chunk: %w", closeErr)
|
||||
}
|
||||
if written != expectedSize {
|
||||
return fmt.Errorf("chunk %d has size %d, want %d", index, written, expectedSize)
|
||||
}
|
||||
|
||||
chunkPath := filepath.Join(session.Directory, chunkFilename(index))
|
||||
if err := os.Remove(chunkPath); err != nil && !os.IsNotExist(err) {
|
||||
return fmt.Errorf("replace chunk: %w", err)
|
||||
}
|
||||
if err := os.Rename(temporaryPath, chunkPath); err != nil {
|
||||
return fmt.Errorf("publish chunk: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *Store) Merge(uploadID string) (Session, error) {
|
||||
session, err := s.load(uploadID)
|
||||
if err != nil {
|
||||
return Session{}, err
|
||||
}
|
||||
|
||||
temporary, err := os.CreateTemp(session.Directory, ".merged-*.zip")
|
||||
if err != nil {
|
||||
return Session{}, fmt.Errorf("create merged archive: %w", err)
|
||||
}
|
||||
temporaryPath := temporary.Name()
|
||||
defer func() {
|
||||
_ = temporary.Close()
|
||||
_ = os.Remove(temporaryPath)
|
||||
}()
|
||||
|
||||
for index := int64(0); index < chunkCount(session.Metadata.Size); index++ {
|
||||
chunkPath := filepath.Join(session.Directory, chunkFilename(index))
|
||||
info, err := os.Stat(chunkPath)
|
||||
if err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
return Session{}, fmt.Errorf("chunk %d is missing", index)
|
||||
}
|
||||
return Session{}, fmt.Errorf("read chunk %d: %w", index, err)
|
||||
}
|
||||
expectedSize, _ := expectedChunkSize(session.Metadata.Size, index)
|
||||
if info.Size() != expectedSize {
|
||||
return Session{}, fmt.Errorf("chunk %d has size %d, want %d", index, info.Size(), expectedSize)
|
||||
}
|
||||
chunk, err := os.Open(chunkPath)
|
||||
if err != nil {
|
||||
return Session{}, fmt.Errorf("open chunk %d: %w", index, err)
|
||||
}
|
||||
_, copyErr := io.Copy(temporary, chunk)
|
||||
closeErr := chunk.Close()
|
||||
if copyErr != nil {
|
||||
return Session{}, fmt.Errorf("merge chunk %d: %w", index, copyErr)
|
||||
}
|
||||
if closeErr != nil {
|
||||
return Session{}, fmt.Errorf("close chunk %d: %w", index, closeErr)
|
||||
}
|
||||
}
|
||||
if err := temporary.Close(); err != nil {
|
||||
return Session{}, fmt.Errorf("close merged archive: %w", err)
|
||||
}
|
||||
|
||||
archivePath := filepath.Join(session.Directory, "archive.zip")
|
||||
if err := os.Rename(temporaryPath, archivePath); err != nil {
|
||||
return Session{}, fmt.Errorf("publish merged archive: %w", err)
|
||||
}
|
||||
session.ArchivePath = archivePath
|
||||
return session, nil
|
||||
}
|
||||
|
||||
func (s *Store) Cancel(uploadID string) error {
|
||||
if !validUploadID(uploadID) {
|
||||
return fmt.Errorf("invalid upload id")
|
||||
}
|
||||
if err := os.RemoveAll(filepath.Join(s.Root, uploadID)); err != nil {
|
||||
return fmt.Errorf("remove upload: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *Store) load(uploadID string) (Session, error) {
|
||||
if !validUploadID(uploadID) {
|
||||
return Session{}, fmt.Errorf("invalid upload id")
|
||||
}
|
||||
directory := filepath.Join(s.Root, uploadID)
|
||||
data, err := os.ReadFile(filepath.Join(directory, "upload.json"))
|
||||
if err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
return Session{}, ErrNotFound
|
||||
}
|
||||
return Session{}, fmt.Errorf("read upload metadata: %w", err)
|
||||
}
|
||||
var metadata Metadata
|
||||
if err := json.Unmarshal(data, &metadata); err != nil {
|
||||
return Session{}, fmt.Errorf("read upload metadata: %w", err)
|
||||
}
|
||||
if !isKnownPurpose(metadata.Purpose) || metadata.Size <= 0 || metadata.Size > s.MaxSize {
|
||||
return Session{}, fmt.Errorf("invalid upload metadata")
|
||||
}
|
||||
return Session{ID: uploadID, Metadata: metadata, Directory: directory}, nil
|
||||
}
|
||||
|
||||
func isKnownPurpose(purpose Purpose) bool {
|
||||
return purpose == PurposeBackup || purpose == PurposePlugin || purpose == PurposeTheme
|
||||
}
|
||||
|
||||
func validUploadID(uploadID string) bool {
|
||||
_, err := uuid.Parse(uploadID)
|
||||
return err == nil
|
||||
}
|
||||
|
||||
func chunkCount(size int64) int64 {
|
||||
return (size + ChunkSize - 1) / ChunkSize
|
||||
}
|
||||
|
||||
func expectedChunkSize(size, index int64) (int64, error) {
|
||||
if index < 0 || index >= chunkCount(size) {
|
||||
return 0, fmt.Errorf("invalid chunk index")
|
||||
}
|
||||
if index == chunkCount(size)-1 {
|
||||
return size - index*ChunkSize, nil
|
||||
}
|
||||
return ChunkSize, nil
|
||||
}
|
||||
|
||||
func chunkFilename(index int64) string {
|
||||
return strconv.FormatInt(index, 10) + ".part"
|
||||
}
|
||||
@@ -0,0 +1,127 @@
|
||||
package upload
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"strconv"
|
||||
|
||||
"github.com/gin-gonic/gin"
|
||||
"github.com/komari-monitor/komari/web/api"
|
||||
)
|
||||
|
||||
const maxChunkRequestSize = ChunkSize + 1*1024*1024
|
||||
|
||||
type Result struct {
|
||||
Message string
|
||||
Data any
|
||||
}
|
||||
|
||||
type Finalizer func(Session) (Result, error)
|
||||
|
||||
type Handler struct {
|
||||
Store *Store
|
||||
Finalizers map[Purpose]Finalizer
|
||||
}
|
||||
|
||||
func NewHandler(store *Store, finalizers map[Purpose]Finalizer) *Handler {
|
||||
return &Handler{Store: store, Finalizers: finalizers}
|
||||
}
|
||||
|
||||
func (h *Handler) Init(c *gin.Context) {
|
||||
var request struct {
|
||||
Purpose Purpose `json:"purpose" binding:"required"`
|
||||
Size int64 `json:"size" binding:"required"`
|
||||
Filename string `json:"filename"`
|
||||
}
|
||||
if err := c.ShouldBindJSON(&request); err != nil {
|
||||
api.RespondError(c, http.StatusBadRequest, fmt.Sprintf("invalid request: %v", err))
|
||||
return
|
||||
}
|
||||
if _, ok := h.Finalizers[request.Purpose]; !ok {
|
||||
api.RespondError(c, http.StatusBadRequest, "unsupported upload purpose")
|
||||
return
|
||||
}
|
||||
session, err := h.Store.Init(request.Purpose, request.Filename, request.Size)
|
||||
if err != nil {
|
||||
api.RespondError(c, http.StatusBadRequest, err.Error())
|
||||
return
|
||||
}
|
||||
api.RespondSuccess(c, gin.H{
|
||||
"upload_id": session.ID,
|
||||
"chunk_size": ChunkSize,
|
||||
})
|
||||
}
|
||||
|
||||
func (h *Handler) Chunk(c *gin.Context) {
|
||||
c.Request.Body = http.MaxBytesReader(c.Writer, c.Request.Body, maxChunkRequestSize)
|
||||
uploadID := c.PostForm("upload_id")
|
||||
index, err := strconv.ParseInt(c.PostForm("chunk_index"), 10, 64)
|
||||
if err != nil {
|
||||
api.RespondError(c, http.StatusBadRequest, "chunk_index must be an integer")
|
||||
return
|
||||
}
|
||||
chunk, _, err := c.Request.FormFile("chunk_data")
|
||||
if err != nil {
|
||||
api.RespondError(c, http.StatusBadRequest, fmt.Sprintf("get chunk data: %v", err))
|
||||
return
|
||||
}
|
||||
defer chunk.Close()
|
||||
if err := h.Store.SaveChunk(uploadID, index, chunk); err != nil {
|
||||
h.respondUploadError(c, err)
|
||||
return
|
||||
}
|
||||
api.RespondSuccess(c, gin.H{"received": true, "chunk_index": index})
|
||||
}
|
||||
|
||||
func (h *Handler) Merge(c *gin.Context) {
|
||||
var request struct {
|
||||
UploadID string `json:"upload_id" binding:"required"`
|
||||
}
|
||||
if err := c.ShouldBindJSON(&request); err != nil {
|
||||
api.RespondError(c, http.StatusBadRequest, fmt.Sprintf("invalid request: %v", err))
|
||||
return
|
||||
}
|
||||
session, err := h.Store.Merge(request.UploadID)
|
||||
if err != nil {
|
||||
_ = h.Store.Cancel(request.UploadID)
|
||||
h.respondUploadError(c, err)
|
||||
return
|
||||
}
|
||||
defer h.Store.Cancel(session.ID)
|
||||
|
||||
finalize, ok := h.Finalizers[session.Metadata.Purpose]
|
||||
if !ok {
|
||||
api.RespondError(c, http.StatusBadRequest, "unsupported upload purpose")
|
||||
return
|
||||
}
|
||||
result, err := finalize(session)
|
||||
if err != nil {
|
||||
api.RespondError(c, http.StatusBadRequest, err.Error())
|
||||
return
|
||||
}
|
||||
api.RespondSuccessMessage(c, result.Message, result.Data)
|
||||
}
|
||||
|
||||
func (h *Handler) Cancel(c *gin.Context) {
|
||||
var request struct {
|
||||
UploadID string `json:"upload_id" binding:"required"`
|
||||
}
|
||||
if err := c.ShouldBindJSON(&request); err != nil {
|
||||
api.RespondError(c, http.StatusBadRequest, fmt.Sprintf("invalid request: %v", err))
|
||||
return
|
||||
}
|
||||
if err := h.Store.Cancel(request.UploadID); err != nil {
|
||||
api.RespondError(c, http.StatusBadRequest, err.Error())
|
||||
return
|
||||
}
|
||||
api.RespondSuccess(c, gin.H{})
|
||||
}
|
||||
|
||||
func (h *Handler) respondUploadError(c *gin.Context, err error) {
|
||||
if errors.Is(err, ErrNotFound) {
|
||||
api.RespondError(c, http.StatusNotFound, "upload not found or expired")
|
||||
return
|
||||
}
|
||||
api.RespondError(c, http.StatusBadRequest, err.Error())
|
||||
}
|
||||
Reference in new issue
Block a user