Compare commits

...
20 Commits
Author SHA1 Message Date
Akizon77 bf6b45ec3a fix(metricstore): #646 skip traffic deltas on agent restart or counter reset 2026-08-13 17:10:59 +08:00
Akizon77 4a3400bee6 feat(notifier): warn users to migrate before traffic report removal 2026-08-13 15:45:56 +08:00
Akizon77 4caa067556 feat(plugin): reload plugin after config update 2026-08-13 14:28:32 +08:00
Akizon77 2f3959adde fix(child_process): stop inheriting runtime timeout for child processes
Replace default runtime timeout with explicit timeout only, avoiding unexpected termination of long-running child processes when a runtime-level timeout is configured.
2026-08-13 14:13:26 +08:00
Akizon77 63a7ffddcf fix(plugin): normalize RPC results before resolving promises
RPC results were passed directly to goja's resolve, which could fail for
non-JSON-compatible Go types. Add normalizeRPCValue to round-trip the
result through JSON, ensuring it becomes plain data structures. Handle
serialization errors properly by rejecting the promise.
2026-08-13 13:05:12 +08:00
Akizon77 92d679ffdb feat(console): format values as JSON and error messages
Improve console output by replacing fmt.Sprint with a dedicated
value formatter that renders null, strings, errors, and JSON-serializable
objects more intuitively for JavaScript runtime users.
2026-08-13 11:45:10 +08:00
Akizon77 7c30fc4dc9 feat: #635 add historical backup cleanup and harden upgrade rollback 2026-08-13 09:43:05 +08:00
Akizon77 773a3715dc feat(upload): add chunked upload store for themes/plugins 2026-08-12 22:38:07 +08:00
Akizon77 75d7b6c7a3 feat(migration): cap structure upgrade progress at 80% 2026-08-12 20:38:31 +08:00
Akizon77 c294c17642 feat: add managed configuration selectors 2026-08-12 16:46:05 +08:00
Akizon77 2883bf6295 feat: add raw SQL query and exec RPC methods 2026-08-11 17:41:46 +08:00
Akizon77 14bf8bde7c build: use COPY --chmod in Dockerfile 2026-08-07 16:57:13 +08:00
Akizon77 f043f2890c docs: update README 2026-08-07 16:23:31 +08:00
Akizon77 057108a2f3 docs: update README 2026-08-07 16:13:50 +08:00
Akizon77 5879d4b3da pref: mem alloc 2026-08-07 13:31:05 +08:00
Akizon77 16769d5687 pref: reduce cpu usage while querying metrics 2026-08-06 23:14:31 +08:00
Akizon77 5ea32c48b6 remove undefined metrics before reclaiming space 2026-08-06 20:25:36 +08:00
Akizon77 a06d922d78 fix(metricstore): do not delete metric history during startup 2026-08-06 20:06:38 +08:00
Akizon77 0ed14fc73d pref: 减少重复查询 2026-08-06 19:53:19 +08:00
Akizon77 61bd0c8bdd perf(metric): reduce SQLite memory usage and make mmap opt-in
Lower default cache size from 16MB to 4MB and journal size limit from 16MB to 4MB. Change MMapSizeBytes handling so 0 explicitly disables mmap instead of being overwritten with a 32MB default; negative values are normalized to 0. Update tests to match the new default journal size limit.
2026-08-05 21:13:19 +08:00
66 changed files with 3546 additions and 1874 deletions

No files matched your search

+1 -4
View File
@@ -8,10 +8,7 @@ ARG TARGETARCH
RUN apk add --no-cache ca-certificates curl tzdata
COPY komari-${TARGETOS}-${TARGETARCH} /app/komari
RUN chmod +x /app/komari
COPY --chmod=755 komari-${TARGETOS}-${TARGETARCH} /app/komari
ENV GIN_MODE=release
ENV KOMARI_LISTEN=0.0.0.0:25774
+42 -23
View File
@@ -5,43 +5,62 @@
![komari](https://socialify.git.ci/komari-monitor/komari/image?description=1&font=Inter&forks=1&issues=1&language=1&logo=https%3A%2F%2Fraw.githubusercontent.com%2Fkomari-monitor%2Fkomari-web%2Fd54ce1288df41ead08aa19f8700186e68028a889%2Fpublic%2Ffavicon.png&name=1&owner=1&pattern=Plus&pulls=1&stargazers=1&theme=Auto)
[简体中文](./README_zh-cn.md)
[English](./README.md) | [简体中文](./README_zh-cn.md)
Komari is a lightweight, self-hosted server monitoring tool designed to provide a simple and efficient solution for monitoring server performance. It supports viewing server status through a web interface and collects data through a lightweight agent.
Komari is a lightweight, self-hosted server monitoring solution. It provides a simple and efficient way to track server performance through a web interface, with metrics collected by a lightweight agent.
> [!WARNING]
> Komari is a self-hosted monitoring and control program intended only for systems you own or are authorized to administer. Do not weaponize Komari, deploy it without consent, or use it for unauthorized access, persistence, command execution, or other abusive activity. For context on real-world abuse risks, see Huntress' analysis: [Komari C2 agent abuse](https://www.huntress.com/blog/komari-c2-agent-abuse).
> Users are solely responsible for how they deploy and operate Komari. The developers do not accept responsibility for unauthorized or abusive use, or for any resulting consequences.
> On Windows, when remote control is enabled, the client displays a Windows notification at each user login to remind the user that Komari is remote control software.
> Komari is a self-hosted monitoring and control application. Deploy it only on systems you own or are authorized to manage. You are solely responsible for how you deploy and use Komari. The developers accept no liability for unauthorized access, persistence, command execution, other misuse, or any resulting consequences.
[文档/Documentation](https://komari-document.pages.dev/) | [文档(镜像站 By Geekertao)](https://www.komari.wiki) | [Telegram Group](https://t.me/komari_monitor)
[Documentation](https://www.komari.wiki/) | [Telegram Group](https://t.me/komari_monitor)
## Features
- **Lightweight and Efficient**: Low resource consumption, suitable for servers of all sizes.
- **Self-hosted**: Complete control over data privacy, easy to deploy.
- **Web Interface**: Intuitive monitoring dashboard, easy to use.
- **Real-time monitoring**: Displays monitoring data at one-second intervals.
- **Lightweight and efficient**: Uses minimal system resources and works well on servers of any size.
- **Self-hosted**: Keeps you in control of your data and privacy.
- **Web interface**: Provides an intuitive, easy-to-use monitoring dashboard.
- **Extensible**: Supports custom themes and plugins.
## One-click Deployment
## Quick Start
- Rainyun - CNY 4.5/month
| Platform | Description |
| --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------ |
| <a href="https://app.rainyun.com/apps/rca/store/6780/NzYxNzAz_"><img src="https://rainyun-apps.cn-nb1.rains3.com/materials/deploy-on-rainyun-cn.svg" alt="Rainyun" width="180"></a> | Deploy websites, databases, and hundreds of popular apps in seconds with flexible hourly billing. |
| <a href="https://apps.fit2cloud.com/1panel/komari"><img src="https://raw.githubusercontent.com/komari-monitor/public/refs/heads/main/images/1panel-logo-blue.png" alt="1Panel App Store" width="180"></a> | A modern, open-source Linux server management panel for websites, databases, containers, files, backups, security, and AI, with one-click deployment from its app store. |
[![](https://rainyun-apps.cn-nb1.rains3.com/materials/deploy-on-rainyun-cn.svg)](https://app.rainyun.com/apps/rca/store/6780/NzYxNzAz_)
For instructions on Docker deployment, binary installation, building from source, and updates, see the [installation guide](https://www.komari.wiki/en/install/quick-start).
- 1Panel App Store
## Screenshots
Available on 1Panel App Store. Install via **App Store > Utilities > Komari**.
| Page | Screenshot |
| ------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
| Home Dashboard | <img src="https://b2.akz.moe/awesome-pictures/komari-screenshot/%E4%B8%BB%E9%A1%B5%E4%BB%AA%E8%A1%A8%E7%9B%98-en.webp" width="800" alt="Home Dashboard"> |
| Admin Dashboard | <img src="https://b2.akz.moe/awesome-pictures/komari-screenshot/%E5%90%8E%E5%8F%B0%E4%BB%AA%E8%A1%A8%E7%9B%98-en.webp" width="800" alt="Admin Dashboard"> |
| History Charts | <img src="https://b2.akz.moe/awesome-pictures/komari-screenshot/%E5%8E%86%E5%8F%B2%E5%9B%BE%E8%A1%A8-en.webp" width="800" alt="History Charts"> |
| Web Terminal | <img src="https://b2.akz.moe/awesome-pictures/komari-screenshot/%E7%BD%91%E9%A1%B5%E7%BB%88%E7%AB%AF.webp" width="800" alt="Web Terminal"> |
| Customizable Themes | <img src="https://b2.akz.moe/awesome-pictures/komari-screenshot/%E4%B8%BB%E9%A2%98%E5%8F%AF%E8%87%AA%E5%AE%9A%E4%B9%89-en.webp" width="800" alt="Customizable Themes"> |
| Theme Market | <img src="https://b2.akz.moe/awesome-pictures/komari-screenshot/%E4%B8%BB%E9%A2%98%E5%B8%82%E5%9C%BA-en.webp" width="800" alt="Theme Market"> |
For Docker, binary, source-build, and update instructions, see the [installation guide](https://komari-document.pages.dev/en/install/quick-start).
## Sponsors
## Contributing
Interested in sponsoring Komari? Contact the developer via [email](mailto:komari@akz.moe) or [Telegram](https://t.me/mamomoe).
Issues and Pull Requests are welcome!
| Sponsor | Description |
| ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
| <a href="https://whmcs.as211392.com/aff.php?aff=110"><img src="https://raw.githubusercontent.com/komari-monitor/public/refs/heads/main/images/dreamcloud.png" alt="Dream Cloud" width="180"></a> | Cost-effective Asia-Pacific hosting with direct connectivity and robust DDoS protection, backed by transparent capacity claims. |
| <a href="https://sharon.io"><img src="https://raw.githubusercontent.com/komari-monitor/public/refs/heads/main/images/sharon-networks.webp" alt="Sharon Networks" width="180"></a> | Premium China-optimized connectivity from Asia-Pacific data centers, featuring low latency, high bandwidth, and Tbps-scale local DDoS mitigation. Join the [Telegram community](https://t.me/SharonNetwork) to participate in charitable initiatives and community giveaways. |
## Acknowledgements
## Contributors
| Acknowledgement | Details |
| ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
| <a href="https://whmcs.as211392.com/aff.php?aff=110"><img src="https://raw.githubusercontent.com/komari-monitor/public/refs/heads/main/images/dreamcloud.png" alt="Dream Cloud" width="180"></a> | 极高性价比解锁直连亚太高防,真高防,不虚标,打死退款 |
| <a href="https://sharon.io"><img src="https://raw.githubusercontent.com/komari-monitor/public/refs/heads/main/images/sharon-networks.webp" alt="Sharon Networks" width="180"></a> | 亚太数据中心提供顶级的中国优化网络接入 · 低延时&高带宽&提供Tbps级本地清洗高防服务, 为您的业务保驾护航, 为您的客户提供极致体验. 加入社区 [Telegram群组](https://t.me/SharonNetwork) 可参与公益募捐或群内抽奖免费使用 |
| Open-source community | Everyone who has submitted pull requests or created themes. |
Thanks to everyone who has contributed code, themes, plugins, documentation, translations, bug reports, or feedback to Komari.
<a href="https://github.com/komari-monitor/komari/graphs/contributors"><img src="https://contributors-img.web.app/image?repo=komari-monitor/komari" alt="Komari contributors" width="600"></a>
## Support the Project
If Komari has been useful to you, consider buying me a coffee. Thank you for your support!
| WeChat Pay | TRON Network |
| ------------------------------------------------------------------------------------------------------------ | ------------------------------------------------------------------------------------------- |
| ![WeChat Pay QR code](https://b2.akz.moe/awesome-pictures/%E5%BE%AE%E4%BF%A1%E8%B5%9E%E8%B5%8F%E7%A0%81.png) | ![TRON Network QR code](https://b2.akz.moe/awesome-pictures/PixPin_2026-08-07_15-16-52.png) |
+35 -18
View File
@@ -5,45 +5,62 @@
![komari](https://socialify.git.ci/komari-monitor/komari/image?description=1&font=Inter&forks=1&issues=1&language=1&logo=https%3A%2F%2Fraw.githubusercontent.com%2Fkomari-monitor%2Fkomari-web%2Fd54ce1288df41ead08aa19f8700186e68028a889%2Fpublic%2Ffavicon.png&name=1&owner=1&pattern=Plus&pulls=1&stargazers=1&theme=Auto)
[简体中文](./README_zh-cn.md)
[English](./README.md) | [简体中文](./README_zh-cn.md)
Komari 是一款轻量级的自托管服务器监控工具,旨在提供简单、高效的服务器性能监控解决方案。它支持通过 Web 界面查看服务器状态,并通过轻量级 Agent 收集数据。
> [!WARNING]
> Komari 是一款自托管的监控/控制程序,仅应部署在你拥有或已获得授权管理的系统上。请勿将 Komari 武器化,或在未获授权的情况下部署、访问、持久化、执行命令及从事其他滥用行为。关于现实中的滥用风险,可参考 Huntress 的分析:[Komari C2 agent abuse](https://www.huntress.com/blog/komari-c2-agent-abuse)。
> 用户需要自行承担部署和使用 Komari 的责任。开发者不对未经授权或滥用行为及其后果承担责任。
> 在 Windows 端开启远程控制后,客户端会在每次用户登录时通过 Windows 通知提醒用户 Komari 是一款远程控制软件。
> Komari 是一款自托管的监控/控制程序,仅应部署在你拥有或已获得授权管理的系统上。在未获授权的情况下部署、访问、持久化、执行命令及从事其他滥用行为,用户需要自行承担部署和使用 Komari 的责任。开发者不对未经授权或滥用行为及其后果承担责任。
[文档/Documentation](https://www.komari.wiki/) | [Telegram 群](https://t.me/komari_monitor)
[文档](https://www.komari.wiki/) | [Telegram 群](https://t.me/komari_monitor)
## 特性
- **实时监控**: 秒级实时数据展示。
- **轻量高效**:低资源占用,适合各种规模的服务器。
- **自托管**:完全掌控数据隐私,部署简单。
- **Web 界面**:直观的监控仪表盘,易于使用。
- **极强的可扩展性**: 支持自定义主题和插件。
## 快速开始
### 0. 容器云一键部署
- 雨云云应用 - CNY 4.5/月
[![](https://rainyun-apps.cn-nb1.rains3.com/materials/deploy-on-rainyun-cn.svg)](https://app.rainyun.com/apps/rca/store/6780/NzYxNzAz_)
- 1Panel 应用商店
已上架1Panel应用商店,应用商店-实用工具-Komari 即可安装
| 平台 | 介绍 |
| -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------- |
| <a href="https://app.rainyun.com/apps/rca/store/6780/NzYxNzAz_"><img src="https://rainyun-apps.cn-nb1.rains3.com/materials/deploy-on-rainyun-cn.svg" alt="Rainyun" width="180"></a> | 秒级部署网站、数据库及数百款热门 App,并采用按小时灵活计费。 |
| <a href="https://apps.fit2cloud.com/1panel/komari"><img src="https://raw.githubusercontent.com/komari-monitor/public/refs/heads/main/images/1panel-logo-blue.png" alt="1Panel Appstore" width="180"></a> | 现代化、开源的Linux 服务器运维管理面板,提供网站、数据库、容器、文件、备份、安全与AI 管理能力,支持应用商店一键部署。 |
Docker、二进制文件、源码构建和更新说明,请参阅 [安装指南](https://www.komari.wiki/install/quick-start).
## 贡献
## 截图
欢迎提交 Issue 或 Pull Request!
| 页面 | 截图 |
| ------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------ |
| 主页仪表盘 | <img src="https://b2.akz.moe/awesome-pictures/komari-screenshot/%E4%B8%BB%E9%A1%B5%E4%BB%AA%E8%A1%A8%E7%9B%98.webp" width="800" alt="主页仪表盘"> |
| 后台仪表盘 | <img src="https://b2.akz.moe/awesome-pictures/komari-screenshot/%E5%90%8E%E5%8F%B0%E4%BB%AA%E8%A1%A8%E7%9B%98.webp" width="800" alt="后台仪表盘"> |
| 历史图表 | <img src="https://b2.akz.moe/awesome-pictures/komari-screenshot/%E5%8E%86%E5%8F%B2%E5%9B%BE%E8%A1%A8.webp" width="800" alt="历史图表"> |
| 网页终端 | <img src="https://b2.akz.moe/awesome-pictures/komari-screenshot/%E7%BD%91%E9%A1%B5%E7%BB%88%E7%AB%AF.webp" width="800" alt="网页终端"> |
| 主题可自定义 | <img src="https://b2.akz.moe/awesome-pictures/komari-screenshot/%E4%B8%BB%E9%A2%98%E5%8F%AF%E8%87%AA%E5%AE%9A%E4%B9%89.webp" width="800" alt="主题可自定义"> |
| 主题市场 | <img src="https://b2.akz.moe/awesome-pictures/komari-screenshot/%E4%B8%BB%E9%A2%98%E5%B8%82%E5%9C%BA.webp" width="800" alt="主题市场"> |
## 鸣谢
## 赞助商
有意赞助 Komari?请通过 [电子邮箱](mailto:komari@akz.moe) 或 [Telegram](https://t.me/mamomoe) 联系开发者。
| 赞助商 | 描述 |
| ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
| <a href="https://whmcs.as211392.com/aff.php?aff=110"><img src="https://raw.githubusercontent.com/komari-monitor/public/refs/heads/main/images/dreamcloud.png" alt="Dream Cloud" width="180"></a> | 极高性价比解锁直连亚太高防,真高防,不虚标,打死退款 |
| <a href="https://sharon.io"><img src="https://raw.githubusercontent.com/komari-monitor/public/refs/heads/main/images/sharon-networks.webp" alt="Sharon Networks" width="180"></a> | 亚太数据中心提供顶级的中国优化网络接入 · 低延时&高带宽&提供Tbps级本地清洗高防服务, 为您的业务保驾护航, 为您的客户提供极致体验. 加入社区 [Telegram群组](https://t.me/SharonNetwork) 可参与公益募捐或群内抽奖免费使用 |
| 开源社区 | 提交 PR、制作主题的各位开发者 |
## 贡献者
感谢所有为 Komari 贡献代码、主题、插件、文档、翻译、问题报告或反馈的朋友。
<a href="https://github.com/komari-monitor/komari/graphs/contributors"><img src="https://contributors-img.web.app/image?repo=komari-monitor/komari" alt="Komari 贡献者" width="600"></a>
## 支持项目
如果 Komari 对你有所帮助,欢迎请作者喝一杯奶茶。感谢你的支持!
| 微信赞赏码 | TRON Network |
| ------------------------------------------------------------------------------------------------ | --------------------------------------------------------------------------- |
| ![wechat](https://b2.akz.moe/awesome-pictures/%E5%BE%AE%E4%BF%A1%E8%B5%9E%E8%B5%8F%E7%A0%81.png) | ![TRON](https://b2.akz.moe/awesome-pictures/PixPin_2026-08-07_15-16-52.png) |
+25 -4
View File
@@ -15,16 +15,37 @@ const (
// Theme represents a komari theme information
type Theme struct {
Name string `json:"name"` // 主题名称
Name any `json:"name"` // 主题名称,支持字符串或多语言对象
Short string `json:"short"` // 短名称,用作文件夹名
Description string `json:"description"` // 主题描述
Description any `json:"description"` // 主题描述,支持字符串或多语言对象
Version string `json:"version"` // 版本号
Author string `json:"author"` // 作者
Author any `json:"author"` // 作者,支持字符串或多语言对象
URL string `json:"url"` // 主题URL
Preview string `json:"preview"` // 预览图片相对路径
Configuration Configuration `json:"configuration"` // 声明配置项
}
// IsLocalizedText reports whether a manifest text field has a usable value.
func IsLocalizedText(value any) bool {
switch text := value.(type) {
case string:
return strings.TrimSpace(text) != ""
case map[string]any:
for _, item := range text {
if itemText, ok := item.(string); ok && strings.TrimSpace(itemText) != "" {
return true
}
}
case map[string]string:
for _, item := range text {
if strings.TrimSpace(item) != "" {
return true
}
}
}
return false
}
type Configuration struct {
Type string `json:"type"` // managed raw redirect
Icon string `json:"icon"` // 图标
@@ -36,7 +57,7 @@ type ManagedThemeConfigurationItem struct {
Key string `json:"key"`
Name any `json:"name"`
Required bool `json:"required"`
Type string `json:"type"` // string number select switch title
Type string `json:"type"` // string number select switch title textbox richtext nodes pingtasks
Options string `json:"options"`
Default any `json:"default"`
Help any `json:"help"`
+31 -52
View File
@@ -3,16 +3,17 @@ package database
import (
"context"
"encoding/json"
logger "github.com/komari-monitor/komari/utils/log"
"os"
"path/filepath"
"strings"
"github.com/gin-gonic/gin"
"github.com/komari-monitor/komari/database/dbcore"
"github.com/komari-monitor/komari/internal/metricstore"
"github.com/komari-monitor/komari/database/models"
"github.com/komari-monitor/komari/internal/config"
"github.com/komari-monitor/komari/internal/managedconfig"
"github.com/komari-monitor/komari/internal/metricstore"
logger "github.com/komari-monitor/komari/utils/log"
"github.com/komari-monitor/komari/web/public"
)
func GetPublicInfo() (map[string]interface{}, error) {
@@ -69,58 +70,20 @@ func GetPublicInfo() (map[string]interface{}, error) {
if err != nil {
logger.Infof("database", "%v", err)
}
// Try to load theme declaration file and merge defaults for managed configuration
// Theme declarations live in ./data/theme/<short>/komari-theme.json
if cst.Theme != "" && cst.Theme != "default" {
themeConfigPath := filepath.Join("./data/theme", cst.Theme, "komari-theme.json")
if _, err := os.Stat(themeConfigPath); err == nil {
b, err := os.ReadFile(themeConfigPath)
if err == nil {
var themeDecl struct {
Configuration struct {
Type string `json:"type"`
Data []models.ManagedThemeConfigurationItem `json:"data"`
} `json:"configuration"`
}
if err := json.Unmarshal(b, &themeDecl); err == nil {
if themeDecl.Configuration.Type == "managed" {
for _, item := range themeDecl.Configuration.Data {
if item.Key == "" {
continue
}
// missing
if _, exists := tc_data[item.Key]; !exists {
var def any = item.Default
// select
if item.Type == "select" {
if def == nil || def == "" {
if item.Options != "" {
opts := strings.Split(item.Options, ",")
if len(opts) > 0 {
def = strings.TrimSpace(opts[0])
}
}
}
}
// number->0, string->"", switch->false
if def == nil {
switch item.Type {
case "number":
def = 0
case "switch":
def = false
default:
def = ""
}
}
tc_data[item.Key] = def
}
}
}
}
items := themeConfigurationItems(cst.Theme)
if cst.Theme != "default" {
for _, item := range items {
if item.Key == "" {
continue
}
if _, exists := tc_data[item.Key]; !exists {
tc_data[item.Key] = managedconfig.DefaultValue(item)
}
}
}
if err := managedconfig.ResolveForOutput(tc_data, items); err != nil {
return nil, err
}
return gin.H{
"sitename": cst.Sitename,
@@ -140,3 +103,19 @@ func GetPublicInfo() (map[string]interface{}, error) {
"theme_settings": tc_data,
}, nil
}
func themeConfigurationItems(short string) []models.ManagedThemeConfigurationItem {
var manifest models.Theme
if short == "default" {
data, err := public.PublicFS.ReadFile("defaultTheme/komari-theme.json")
if err != nil || json.Unmarshal(data, &manifest) != nil {
return nil
}
} else {
data, err := os.ReadFile(filepath.Join("./data/theme", short, "komari-theme.json"))
if err != nil || json.Unmarshal(data, &manifest) != nil {
return nil
}
}
return managedconfig.Items(manifest.Configuration)
}
+38 -5
View File
@@ -29,6 +29,8 @@ INSTALL_DIR="/opt/komari"
DATA_DIR="/opt/komari"
SERVICE_NAME="komari"
BINARY_PATH="$INSTALL_DIR/komari"
BACKUP_DIR="$INSTALL_DIR/backup"
DATA_BACKUP_DIR="$DATA_DIR/data/backup"
DEFAULT_PORT="25774"
LISTEN_PORT=""
REPO="komari-monitor/komari"
@@ -419,6 +421,26 @@ show_access_info() {
ui_msgbox "安装完成" "$content"
}
# Remove historical upgrade backups
cleanup_backups() {
if ! ui_yesno "确认清理备份" "将删除 Komari 的二进制升级备份和数据升级压缩包。\n\n您确定要继续吗?"; then
log_info "备份清理已取消"
return 0
fi
log_step "清理升级历史备份..."
rm -f -- "${BINARY_PATH}.backup."*
local backup_dir
for backup_dir in "$BACKUP_DIR" "$DATA_BACKUP_DIR"; do
if [ -d "$backup_dir" ]; then
find "$backup_dir" -maxdepth 1 -type f -name '*.zip' -delete
fi
done
ui_msgbox "清理完成" "升级历史备份已清理。\n\n清理范围:\n 二进制备份: ${BINARY_PATH}.backup.*\n 数据压缩包: $BACKUP_DIR\n 数据压缩包: $DATA_BACKUP_DIR"
}
# Upgrade function
upgrade_komari() {
log_step "升级 Komari..."
@@ -439,14 +461,23 @@ upgrade_komari() {
log_step "停止 Komari 服务..."
systemctl stop ${SERVICE_NAME}.service
log_step "清理旧的二进制备份..."
rm -f -- "${BINARY_PATH}.backup."*
local backup_path="${BINARY_PATH}.backup.$(date +%Y%m%d_%H%M%S)"
log_step "备份当前二进制文件..."
cp "$BINARY_PATH" "${BINARY_PATH}.backup.$(date +%Y%m%d_%H%M%S)"
if ! cp "$BINARY_PATH" "$backup_path"; then
log_error "备份当前二进制文件失败,正在启动服务"
systemctl start ${SERVICE_NAME}.service
ui_msgbox "错误" "备份当前二进制文件失败,升级已取消。"
return 1
fi
local arch=$(detect_arch)
local download_url=$(get_download_url "$arch")
if [ $? -ne 0 ]; then
log_error "获取下载链接失败,正在从备份恢复"
mv "${BINARY_PATH}.backup."* "$BINARY_PATH"
mv "$backup_path" "$BINARY_PATH"
systemctl start ${SERVICE_NAME}.service
ui_msgbox "错误" "获取下载链接失败,已从备份恢复。"
return 1
@@ -455,7 +486,7 @@ upgrade_komari() {
log_step "下载最新版本..."
if ! curl -L -o "$BINARY_PATH" "$download_url"; then
log_error "下载失败,正在从备份恢复"
mv "${BINARY_PATH}.backup."* "$BINARY_PATH"
mv "$backup_path" "$BINARY_PATH"
systemctl start ${SERVICE_NAME}.service
ui_msgbox "错误" "下载失败,已从备份恢复。"
return 1
@@ -593,7 +624,8 @@ main_menu() {
"5" "查看日志" \
"6" "重启服务" \
"7" "停止服务" \
"8" "退出")
"8" "清理升级历史备份" \
"9" "退出")
# 用户在 TUI 中取消(ESC/Cancel)则退出
if [ $? -ne 0 ] && tui_enabled; then
@@ -609,7 +641,8 @@ main_menu() {
5) show_logs ;;
6) restart_service ;;
7) stop_service ;;
8)
8) cleanup_backups ;;
9)
tui_enabled && clear
exit 0
;;
+144
View File
@@ -0,0 +1,144 @@
// Package managedconfig implements the shared managed-configuration behavior
// used by themes and plugins.
package managedconfig
import (
"encoding/json"
"strings"
"github.com/komari-monitor/komari/database/dbcore"
"github.com/komari-monitor/komari/database/models"
)
const (
TypeNodes = "nodes"
TypePingTasks = "pingtasks"
)
// Items returns the declared managed configuration items. An omitted type is
// managed, matching the theme configuration default.
func Items(configuration models.Configuration) []models.ManagedThemeConfigurationItem {
if configuration.Type != "" && !strings.EqualFold(configuration.Type, models.ThemeConfigurationManaged) {
return nil
}
raw, err := json.Marshal(configuration.Data)
if err != nil {
return nil
}
var items []models.ManagedThemeConfigurationItem
if err := json.Unmarshal(raw, &items); err != nil {
return nil
}
return items
}
func DefaultValue(item models.ManagedThemeConfigurationItem) any {
value := item.Default
if item.Type == "select" && (value == nil || value == "") && item.Options != "" {
options := strings.Split(item.Options, ",")
if len(options) > 0 {
return strings.TrimSpace(options[0])
}
}
if value != nil {
return value
}
switch item.Type {
case "number":
return float64(0)
case "switch":
return false
case TypeNodes, TypePingTasks:
return "[]"
default:
return ""
}
}
// ResolveForOutput decodes declared selectors, drops deleted references, and
// returns typed arrays suitable for public theme settings and plugin config.
func ResolveForOutput(values map[string]any, items []models.ManagedThemeConfigurationItem) error {
hasNodes := false
hasPingTasks := false
for _, item := range items {
switch item.Type {
case TypeNodes:
hasNodes = true
case TypePingTasks:
hasPingTasks = true
}
}
db := dbcore.GetDBInstance()
liveNodes := map[string]struct{}{}
if hasNodes {
var nodes []models.Client
if err := db.Select("uuid").Find(&nodes).Error; err != nil {
return err
}
for _, node := range nodes {
liveNodes[node.UUID] = struct{}{}
}
}
livePingTasks := map[uint]struct{}{}
if hasPingTasks {
var tasks []models.PingTask
if err := db.Select("id").Find(&tasks).Error; err != nil {
return err
}
for _, task := range tasks {
livePingTasks[task.Id] = struct{}{}
}
}
for _, item := range items {
if item.Key == "" {
continue
}
switch item.Type {
case TypeNodes:
selected := NodeIDs(values[item.Key])
filtered := make([]string, 0, len(selected))
for _, id := range selected {
if _, ok := liveNodes[id]; ok {
filtered = append(filtered, id)
}
}
values[item.Key] = filtered
case TypePingTasks:
selected := PingTaskIDs(values[item.Key])
filtered := make([]uint, 0, len(selected))
for _, id := range selected {
if _, ok := livePingTasks[id]; ok {
filtered = append(filtered, id)
}
}
values[item.Key] = filtered
}
}
return nil
}
func NodeIDs(value any) []string {
raw, ok := value.(string)
if !ok {
return nil
}
var ids []string
if json.Unmarshal([]byte(raw), &ids) != nil {
return nil
}
return ids
}
func PingTaskIDs(value any) []uint {
raw, ok := value.(string)
if !ok {
return nil
}
var ids []uint
if json.Unmarshal([]byte(raw), &ids) != nil {
return nil
}
return ids
}
-9
View File
@@ -92,9 +92,6 @@ func createMetricDefinitionsWithDefaultRetention(ctx context.Context, s *metric.
}
if err == nil {
if existing.RetentionDays == 0 {
if _, err := s.SetMetricRetention(ctx, def.Name, 0); err != nil {
return fmt.Errorf("failed to preserve disabled metric %s: %w", def.Name, err)
}
continue
}
def.RetentionDays = existing.RetentionDays
@@ -103,11 +100,5 @@ func createMetricDefinitionsWithDefaultRetention(ctx context.Context, s *metric.
return fmt.Errorf("failed to create metric %s: %w", def.Name, err)
}
}
for _, name := range obsoleteBuiltinMetricNames {
if err := s.DeleteMetric(ctx, name); err != nil {
return fmt.Errorf("failed to remove obsolete metric %s: %w", name, err)
}
}
return nil
}
+25 -1
View File
@@ -84,7 +84,31 @@ func ReclaimSpace(ctx context.Context) (MaintenanceResult, error) {
}
maintenanceCtx := context.Background()
result.Before, result.BeforeSizeError = activeStore.StorageSize(maintenanceCtx)
maintenanceErr := activeStore.ReclaimSpace(maintenanceCtx)
maintenanceErr := deleteUndefinedMetrics(maintenanceCtx, activeStore)
if maintenanceErr == nil {
maintenanceErr = activeStore.ReclaimSpace(maintenanceCtx)
}
result.After, result.AfterSizeError = activeStore.StorageSize(maintenanceCtx)
return result, maintenanceErr
}
func deleteUndefinedMetrics(ctx context.Context, s *metric.Store) error {
definitions, err := s.ListMetrics(ctx)
if err != nil {
return fmt.Errorf("list metric definitions before reclaim: %w", err)
}
builtin := make(map[string]struct{}, len(builtinMetricNames))
for _, name := range builtinMetricNames {
builtin[name] = struct{}{}
}
for _, definition := range definitions {
if _, ok := builtin[definition.Name]; ok {
continue
}
if err := s.DeleteMetric(ctx, definition.Name); err != nil {
return fmt.Errorf("delete undefined metric %q before reclaim: %w", definition.Name, err)
}
}
return nil
}
+2 -7
View File
@@ -32,13 +32,6 @@ var loadRecordMetricNames = []string{
MetricProcess, MetricConnections, MetricConnectionsUDP,
}
var obsoleteBuiltinMetricNames = []string{
"memory.total",
"swap.total",
"temperature",
"disk.total",
}
// gpuDeviceRecordMetricNames are stored separately from the entity-level GPU
// average and are included when deleting all system records.
var gpuDeviceRecordMetricNames = []string{
@@ -50,6 +43,8 @@ var recordMetricNames = joinMetricNames(loadRecordMetricNames, gpuDeviceRecordMe
// Ping has an independent retention and cleanup boundary.
var pingMetricNames = []string{MetricPingLatency, MetricPingLoss}
var builtinMetricNames = joinMetricNames(recordMetricNames, pingMetricNames)
func metricNameForRecordField(name string) (string, bool) {
switch name {
case "cpu":
+10 -7
View File
@@ -3,7 +3,6 @@ package metricstore
import (
"context"
"database/sql"
"errors"
"path/filepath"
"testing"
"time"
@@ -283,7 +282,7 @@ func TestCreateMetricDefinitionsUsesExplicitRetentionAndPreservesOverrides(t *te
}
}
func TestCreateMetricDefinitionsRemovesObsoleteMetrics(t *testing.T) {
func TestCreateMetricDefinitionsKeepsExistingMetrics(t *testing.T) {
ctx := context.Background()
s, err := metric.Open(ctx, metric.SQLite(":memory:", metric.WithMaxOpenConns(1)))
if err != nil {
@@ -299,15 +298,19 @@ func TestCreateMetricDefinitionsRemovesObsoleteMetrics(t *testing.T) {
if err := createMetricDefinitions(ctx, s); err != nil {
t.Fatalf("refresh built-in definitions: %v", err)
}
if _, err := s.GetMetric(ctx, "memory.total"); !errors.Is(err, metric.ErrNotFound) {
t.Fatalf("obsolete definition error = %v, want ErrNotFound", err)
definition, err := s.GetMetric(ctx, "memory.total")
if err != nil {
t.Fatalf("existing definition was removed: %v", err)
}
if definition.RetentionDays != 1 {
t.Fatalf("existing retention = %d, want 1", definition.RetentionDays)
}
points, err := s.Query(ctx, metric.Query{MetricName: "memory.total", EntityID: "node-a", Start: time.Now().UTC().Add(-time.Hour), End: time.Now().UTC().Add(time.Hour)})
if err != nil {
t.Fatalf("query obsolete points: %v", err)
t.Fatalf("query existing points: %v", err)
}
if len(points) != 0 {
t.Fatalf("obsolete points remain: %#v", points)
if len(points) != 1 || points[0].Value != 1024 {
t.Fatalf("existing points were removed: %#v", points)
}
}
+63
View File
@@ -0,0 +1,63 @@
package metricstore
import (
"context"
"database/sql"
"fmt"
"github.com/komari-monitor/komari/pkg/metric"
)
// QueryContext runs a raw read query against the active metric store. The
// caller must invoke the returned release function after closing rows.
func QueryContext(ctx context.Context, query string, args ...any) (*sql.Rows, metric.Driver, func(), error) {
return QueryForDriver(ctx, func(metric.Driver) (string, error) {
return query, nil
}, args...)
}
// QueryForDriver runs a raw read query whose SQL depends on the active store's
// driver. The caller must invoke the returned release function after closing rows.
func QueryForDriver(ctx context.Context, query func(metric.Driver) (string, error), args ...any) (*sql.Rows, metric.Driver, func(), error) {
if err := storeOperations.AcquireShared(ctx); err != nil {
return nil, "", nil, fmt.Errorf("wait for metric store operations before query: %w", err)
}
storeMu.RLock()
activeStore := store
storeMu.RUnlock()
if activeStore == nil {
storeOperations.ReleaseShared()
return nil, "", nil, ErrStoreNotInitialized
}
statement, err := query(activeStore.Driver())
if err != nil {
storeOperations.ReleaseShared()
return nil, "", nil, err
}
rows, err := activeStore.QueryContext(ctx, statement, args...)
if err != nil {
storeOperations.ReleaseShared()
return nil, "", nil, err
}
return rows, activeStore.Driver(), storeOperations.ReleaseShared, nil
}
// ExecContext runs a raw statement against the active metric store.
func ExecContext(ctx context.Context, query string, args ...any) (sql.Result, metric.Driver, error) {
if err := storeOperations.Acquire(ctx); err != nil {
return nil, "", fmt.Errorf("wait for metric store operations before execution: %w", err)
}
defer storeOperations.Release()
storeMu.RLock()
activeStore := store
storeMu.RUnlock()
if activeStore == nil {
return nil, "", ErrStoreNotInitialized
}
result, err := activeStore.ExecContext(ctx, query, args...)
return result, activeStore.Driver(), err
}
+9 -2
View File
@@ -27,6 +27,8 @@ type reportTrafficValues struct {
totalUp int64
hasDown bool
totalDown int64
hasUptime bool
uptime int64
}
var reportTrafficStates sync.Map
@@ -384,12 +386,15 @@ func writeReportBatch(ctx context.Context, reports []v1.Report) ([]v1.Report, er
if !values.timestamp.IsZero() && !report.UpdatedAt.After(values.timestamp) {
report.UpdatedAt = values.timestamp.Add(time.Millisecond)
}
agentRestart := values.hasUptime && report.Uptime < values.uptime
counterResetUp := values.hasUp && report.Network.TotalUp < values.totalUp
counterResetDown := values.hasDown && report.Network.TotalDown < values.totalDown
trafficUp := int64(0)
if values.hasUp {
if values.hasUp && !agentRestart && !counterResetUp {
trafficUp = TrafficCounterDelta(report.Network.TotalUp, values.totalUp)
}
trafficDown := int64(0)
if values.hasDown {
if values.hasDown && !agentRestart && !counterResetDown {
trafficDown = TrafficCounterDelta(report.Network.TotalDown, values.totalDown)
}
points = append(points, reportMetricPoints(report, trafficUp, trafficDown)...)
@@ -398,6 +403,8 @@ func writeReportBatch(ctx context.Context, reports []v1.Report) ([]v1.Report, er
values.totalUp = report.Network.TotalUp
values.hasDown = true
values.totalDown = report.Network.TotalDown
values.hasUptime = true
values.uptime = report.Uptime
pendingStates[state] = values
prepared[i] = report
}
+46 -4
View File
@@ -87,11 +87,11 @@ func TestWriteReportStoresMinuteMetricsAndResetAwareTraffic(t *testing.T) {
t.Fatalf("write reset report: %v", err)
}
assertMetricValues(t, s, MetricTrafficUp, report.UUID, base.Add(-time.Second), base.Add(time.Minute), []float64{0, 50, 20})
assertMetricValues(t, s, MetricTrafficDown, report.UUID, base.Add(-time.Second), base.Add(time.Minute), []float64{0, 60, 30})
assertMetricValues(t, s, MetricTrafficUp, report.UUID, base.Add(-time.Second), base.Add(time.Minute), []float64{0, 50, 0})
assertMetricValues(t, s, MetricTrafficDown, report.UUID, base.Add(-time.Second), base.Add(time.Minute), []float64{0, 60, 0})
assertMetricValues(t, s, MetricNetTotalUp, report.UUID, base.Add(-time.Second), base.Add(time.Minute), []float64{100, 150, 20})
assertMetricAggregate(t, s, MetricTrafficUp, report.UUID, base.Add(-time.Second), base.Add(time.Minute), metric.AggSum, 70, 3)
assertMetricAggregate(t, s, MetricTrafficDown, report.UUID, base.Add(-time.Second), base.Add(time.Minute), metric.AggSum, 90, 3)
assertMetricAggregate(t, s, MetricTrafficUp, report.UUID, base.Add(-time.Second), base.Add(time.Minute), metric.AggSum, 50, 3)
assertMetricAggregate(t, s, MetricTrafficDown, report.UUID, base.Add(-time.Second), base.Add(time.Minute), metric.AggSum, 60, 3)
gpuPoints, err := s.Query(ctx, metric.Query{
MetricName: MetricGPUDeviceUsage,
@@ -370,6 +370,48 @@ func TestTrafficCounterDelta(t *testing.T) {
}
}
func TestWriteReportRebasesTrafficAfterAgentRestart(t *testing.T) {
ctx := context.Background()
s := useReportTestStore(t, nil)
base := time.Now().UTC().Truncate(time.Minute).Add(5 * time.Second)
report := v1.Report{
UUID: "restarted-node",
UpdatedAt: base,
Uptime: 1000,
Network: v1.NetworkReport{TotalUp: 100, TotalDown: 200},
}
if _, err := WriteReport(ctx, report); err != nil {
t.Fatalf("write first report: %v", err)
}
report.UpdatedAt = base.Add(3 * time.Second)
report.Uptime = 1003
report.Network.TotalUp = 150
report.Network.TotalDown = 260
if _, err := WriteReport(ctx, report); err != nil {
t.Fatalf("write continuous report: %v", err)
}
report.UpdatedAt = base.Add(6 * time.Second)
report.Uptime = 1
report.Network.TotalUp = 155
report.Network.TotalDown = 265
if _, err := WriteReport(ctx, report); err != nil {
t.Fatalf("write report after agent restart: %v", err)
}
report.UpdatedAt = base.Add(9 * time.Second)
report.Uptime = 4
report.Network.TotalUp = 180
report.Network.TotalDown = 300
if _, err := WriteReport(ctx, report); err != nil {
t.Fatalf("write report after new baseline: %v", err)
}
assertMetricValues(t, s, MetricTrafficUp, report.UUID, base.Add(-time.Second), base.Add(time.Minute), []float64{0, 50, 0, 25})
assertMetricValues(t, s, MetricTrafficDown, report.UUID, base.Add(-time.Second), base.Add(time.Minute), []float64{0, 60, 0, 35})
}
func TestWriteReportNormalizesReceiveTimeToUTC(t *testing.T) {
ctx := context.Background()
s := useReportTestStore(t, nil)
+8 -43
View File
@@ -5,10 +5,10 @@ import (
"errors"
"fmt"
"path/filepath"
"strings"
"github.com/komari-monitor/komari/database/dbcore"
"github.com/komari-monitor/komari/database/models"
"github.com/komari-monitor/komari/internal/managedconfig"
"gorm.io/gorm"
)
@@ -26,7 +26,11 @@ func GetConfiguration(short string) (map[string]any, error) {
if err != nil {
return values, nil // not installed (or unreadable): keep saved values
}
mergeConfigurationDefaults(values, info.Configuration)
items := managedconfig.Items(info.Configuration)
mergeConfigurationDefaults(values, items)
if err := managedconfig.ResolveForOutput(values, items); err != nil {
return nil, fmt.Errorf("resolve plugin configuration: %w", err)
}
return values, nil
}
@@ -48,18 +52,7 @@ func savedConfiguration(short string) (map[string]any, error) {
// mergeConfigurationDefaults fills manifest-declared defaults for keys that
// are missing from the saved values, following the theme merge rules.
func mergeConfigurationDefaults(values map[string]any, configuration models.Configuration) {
if configuration.Type != models.ThemeConfigurationManaged {
return
}
raw, err := json.Marshal(configuration.Data)
if err != nil {
return
}
var items []models.ManagedThemeConfigurationItem
if err := json.Unmarshal(raw, &items); err != nil {
return
}
func mergeConfigurationDefaults(values map[string]any, items []models.ManagedThemeConfigurationItem) {
for _, item := range items {
if item.Key == "" {
continue
@@ -67,38 +60,10 @@ func mergeConfigurationDefaults(values map[string]any, configuration models.Conf
if _, exists := values[item.Key]; exists {
continue
}
values[item.Key] = configurationDefault(item)
values[item.Key] = managedconfig.DefaultValue(item)
}
}
// configurationDefault resolves one item's effective default, mirroring the
// theme rules: select falls back to its first option; nil defaults become
// 0 for number, false for switch and "" otherwise.
func configurationDefault(item models.ManagedThemeConfigurationItem) any {
def := item.Default
if item.Type == "select" {
if def == nil || def == "" {
if item.Options != "" {
opts := strings.Split(item.Options, ",")
if len(opts) > 0 {
return strings.TrimSpace(opts[0])
}
}
}
}
if def == nil {
switch item.Type {
case "number":
return float64(0) // JSON numbers decode as float64
case "switch":
return false
default:
return ""
}
}
return def
}
// SaveConfiguration upserts the configuration values of one plugin.
func SaveConfiguration(short string, data map[string]any) error {
if data == nil {
+54
View File
@@ -14,6 +14,8 @@ import (
"time"
"github.com/gin-gonic/gin"
"github.com/komari-monitor/komari/database/dbcore"
"github.com/komari-monitor/komari/database/models"
"github.com/komari-monitor/komari/pkg/rpc"
)
@@ -398,6 +400,58 @@ func TestGetConfigurationMergesManifestDefaults(t *testing.T) {
t.Errorf("mode still default = %v", values["mode"])
}
}
func TestGetConfigurationResolvesSelectorValues(t *testing.T) {
withTempDataDir(t)
if err := dbcore.GetDBInstance().Create(&models.Client{UUID: "node-a"}).Error; err != nil {
t.Fatal(err)
}
if err := dbcore.GetDBInstance().Create(&models.PingTask{Id: 8}).Error; err != nil {
t.Fatal(err)
}
zipPath := writePluginZip(t, map[string]string{
"komari-plugin.json": `{"name":"Cfg","short":"cfg","version":"1.0.0","configuration":{"type":"managed","data":[
{"key":"nodes","name":"Nodes","type":"nodes"},
{"key":"tasks","name":"Tasks","type":"pingtasks"}
]}}`,
"script.js": `function load() {}`,
})
if _, err := InstallZip(zipPath); err != nil {
t.Fatal(err)
}
if err := SaveConfiguration("cfg", map[string]any{
"nodes": `["node-a","deleted-node"]`,
"tasks": `[8,7]`,
}); err != nil {
t.Fatal(err)
}
values, err := GetConfiguration("cfg")
if err != nil {
t.Fatal(err)
}
nodes, ok := values["nodes"].([]string)
if !ok || len(nodes) != 1 || nodes[0] != "node-a" {
t.Fatalf("nodes = %#v", values["nodes"])
}
tasks, ok := values["tasks"].([]uint)
if !ok || len(tasks) != 1 || tasks[0] != 8 {
t.Fatalf("tasks = %#v", values["tasks"])
}
var stored models.PluginConfiguration
if err := dbcore.GetDBInstance().Where("short = ?", "cfg").First(&stored).Error; err != nil {
t.Fatal(err)
}
var raw map[string]any
if err := json.Unmarshal([]byte(stored.Data), &raw); err != nil {
t.Fatal(err)
}
if raw["nodes"] != `["node-a","deleted-node"]` || raw["tasks"] != `[8,7]` {
t.Fatalf("stored = %#v", raw)
}
}
func TestGetConfigFromScript(t *testing.T) {
withTempDataDir(t)
gin.SetMode(gin.TestMode)
+2 -22
View File
@@ -41,7 +41,7 @@ func validateManifest(info *models.Plugin) error {
if !validShort(info.Short) {
return fmt.Errorf("plugin short %q is invalid: only letters, digits, '_' and '-' are allowed", info.Short)
}
if !localizedText(info.Name) {
if !models.IsLocalizedText(info.Name) {
return fmt.Errorf("plugin name is required")
}
if info.Entry == "" {
@@ -90,7 +90,7 @@ func validatePage(page *models.PluginPage) error {
default:
return fmt.Errorf("plugin page type %q is invalid: use \"iframe\" or \"redirect\"", page.Type)
}
if !localizedText(page.Title) {
if !models.IsLocalizedText(page.Title) {
return fmt.Errorf("plugin page %q requires a title", page.Title)
}
return nil
@@ -113,23 +113,3 @@ func isSafeInternalPath(target string) bool {
}
return true
}
// localizedText reports whether v is a usable name/description/author value:
// a non-empty string, or an i18n object with at least one non-empty value.
func localizedText(v any) bool {
switch t := v.(type) {
case nil:
return false
case string:
return strings.TrimSpace(t) != ""
case map[string]any:
for _, value := range t {
if s, ok := value.(string); ok && strings.TrimSpace(s) != "" {
return true
}
}
return false
default:
return false
}
}
+17
View File
@@ -153,6 +153,13 @@ func SetEnabled(short string, enabled, approved bool) error {
return global.setEnabled(short, enabled, approved)
}
// Reload unloads and loads an enabled plugin. The load hook completes before
// this function returns; a failed load is persisted as the plugin's last
// error and disables the plugin, matching the normal enable/startup path.
func Reload(short string) error {
return global.reload(short)
}
// CloseAll unloads every loaded plugin. Used during server shutdown.
func CloseAll() error {
return global.closeAll()
@@ -521,6 +528,16 @@ func (m *Manager) restartPlugin(short string) error {
return nil
}
func (m *Manager) reload(short string) error {
if !m.stateStore().get(short).Enabled {
return nil
}
if err := m.unload(short); err != nil && !errors.Is(err, errNotLoaded) {
return err
}
return m.restartPlugin(short)
}
// disableWithError persists the auto-disable state for one plugin and wraps
// the load failure for the startup error summary.
func (m *Manager) disableWithError(short string, st PluginState, err error) error {
+20 -1
View File
@@ -3,6 +3,7 @@ package plugin
import (
"bytes"
"context"
"encoding/json"
"errors"
"fmt"
"net/http"
@@ -143,7 +144,12 @@ func (m *Manager) registerServerModule(host *jsruntime.Host, registry *require.R
_ = reject(jsRPCError(vm, resp.Error))
return
}
_ = resolve(resp.Result)
result, err := normalizeRPCValue(resp.Result)
if err != nil {
_ = reject(vm.NewGoError(err))
return
}
_ = resolve(result)
})
}()
return vm.ToValue(promise)
@@ -266,6 +272,19 @@ func jsRPCError(vm *goja.Runtime, e *rpc.JsonRpcError) *goja.Object {
return obj
}
func normalizeRPCValue(value any) (any, error) {
data, err := json.Marshal(value)
if err != nil {
return nil, fmt.Errorf("serialize RPC result: %w", err)
}
var normalized any
if err := json.Unmarshal(data, &normalized); err != nil {
return nil, fmt.Errorf("decode RPC result: %w", err)
}
return normalized, nil
}
// registerRoute registers (or reuses) a gin route slot for one plugin. It is
// called from the plugin's own event loop during script evaluation and takes
// the manager lock itself. The critical section is short and never waits on
+2 -1
View File
@@ -705,7 +705,8 @@ function runtimeInfo() {
| ChildProcess `kill/ref/unref/disconnect/send` | `部分实现` 或调用抛错,见下文 |
支持的 options:`cwd`、`env`、`shell`、`timeout`、`encoding`、`maxBuffer`。
`cwd` 受 `BaseDir` 限制;请求的 timeout 和 maxBuffer 只能收紧 Go 侧全局上限,不能放大。
`cwd` 受 `BaseDir` 限制;子进程默认不继承 runtime 的 `Timeout`,只有显式传入 `timeout` 时才会自动结束。
请求的 `maxBuffer` 只能收紧 Go 侧全局上限,不能放大。
`stdio/input/detached/uid/gid/windowsHide/windowsVerbatimArguments/serialization` 等未实现。
- `spawn()` 的 `stdin` 是 `stream.Writable`(写入经后台写队列串行执行,`end()` 后关闭管道),
+2 -2
View File
@@ -163,7 +163,7 @@ func (m *Module) childCommandArguments(vm *goja.Runtime, call goja.FunctionCall,
}
func (m *Module) childOptions(vm *goja.Runtime, value goja.Value) childCommandOptions {
options := childCommandOptions{cwd: m.filesystem.Cwd(), env: os.Environ(), timeout: m.runtime.Timeout(), maxBuffer: m.maxOutput}
options := childCommandOptions{cwd: m.filesystem.Cwd(), env: os.Environ(), maxBuffer: m.maxOutput}
if value == nil || goja.IsUndefined(value) || goja.IsNull(value) {
return options
}
@@ -190,7 +190,7 @@ func (m *Module) childOptions(vm *goja.Runtime, value goja.Value) childCommandOp
}
if timeoutValue := object.Get("timeout"); timeoutValue != nil && timeoutValue.ToInteger() > 0 {
timeout := timeoutValue.ToInteger()
options.timeout = min(options.timeout, time.Duration(timeout)*time.Millisecond)
options.timeout = time.Duration(timeout) * time.Millisecond
}
if encoding := object.Get("encoding"); encoding != nil && !goja.IsUndefined(encoding) && !goja.IsNull(encoding) {
options.encoding = encoding.String()
+20 -3
View File
@@ -5,6 +5,7 @@ package console
// substitutions. assert(false, ...) and trace() include a stack trace.
import (
"encoding/json"
"fmt"
"io"
"strings"
@@ -121,11 +122,27 @@ func formatConsoleValues(values []goja.Value) string {
}
parts := make([]string, 0, len(args))
for _, arg := range args {
parts = append(parts, fmt.Sprint(arg))
parts = append(parts, formatConsoleValue(arg))
}
return strings.Join(parts, " ")
}
func formatConsoleValue(value any) string {
switch value := value.(type) {
case nil:
return "null"
case string:
return value
case error:
return value.Error()
}
if data, err := json.Marshal(value); err == nil {
return string(data)
}
return fmt.Sprint(value)
}
func formatConsoleString(format string, args []any) string {
var builder strings.Builder
argIndex := 0
@@ -154,7 +171,7 @@ func formatConsoleString(format string, args []any) string {
argIndex++
switch format[i] {
case 's', 'o', 'O':
builder.WriteString(fmt.Sprint(arg))
builder.WriteString(formatConsoleValue(arg))
case 'd', 'i':
builder.WriteString(fmt.Sprintf("%d", toConsoleInteger(arg)))
case 'f':
@@ -168,7 +185,7 @@ func formatConsoleString(format string, args []any) string {
if argIndex < len(args) {
remaining := make([]string, 0, len(args)-argIndex)
for _, arg := range args[argIndex:] {
remaining = append(remaining, fmt.Sprint(arg))
remaining = append(remaining, formatConsoleValue(arg))
}
if builder.Len() > 0 {
builder.WriteByte(' ')
+609
View File
@@ -0,0 +1,609 @@
package metric
import (
"context"
"database/sql"
"fmt"
"sort"
"strings"
"time"
)
type batchSeriesGroupKey struct {
resolution time.Duration
needDigest bool
}
type batchSeriesGroup struct {
key batchSeriesGroupKey
metricNames map[string]struct{}
fields rollupReadFields
}
type seriesIdentity struct {
metricName string
entityID string
tagsHash string
}
type seriesReadMeta struct {
id int64
metricName string
entityID string
tagsHash string
tagsJSON string
tags map[string]string
}
type rollupRateSample struct {
timestamp int64
value float64
}
type rollupAggregateState struct {
summary *rollupBucket
rateSamples []rollupRateSample
tags map[string]string
}
type metricSeriesAccumulator struct {
spec BatchSeriesSpec
compression float64
needDigest bool
needSummary bool
needRate bool
groups map[rollupKey]*rollupAggregateState
rateGroups map[rollupKey]*rollupAggregateState
}
// SeriesBatch is the single rollup query implementation. It resolves metric
// metadata once, scans each selected backing tier once, and streams rows into
// the final output buckets.
func (s *Store) SeriesBatch(ctx context.Context, query BatchSeriesQuery, now time.Time) (BatchSeriesResult, error) {
if err := s.ensureOpen(); err != nil {
return BatchSeriesResult{}, err
}
if err := query.Validate(); err != nil {
return BatchSeriesResult{}, err
}
return s.seriesBatchAt(ctx, query.normalized(), now.UTC(), nil)
}
// GetMetrics loads the requested metric definitions with one database query.
func (s *Store) GetMetrics(ctx context.Context, metricNames []string) (map[string]Definition, error) {
if err := s.ensureOpen(); err != nil {
return nil, err
}
if len(metricNames) == 0 {
return map[string]Definition{}, nil
}
return s.loadBatchDefinitions(ctx, metricNames)
}
func (s *Store) seriesBatchAt(ctx context.Context, query BatchSeriesQuery, now time.Time, forcedResolutions map[string]time.Duration) (BatchSeriesResult, error) {
metricNames := make([]string, 0, len(query.Specs))
for _, spec := range query.Specs {
metricNames = append(metricNames, spec.MetricName)
}
definitions, err := s.loadBatchDefinitions(ctx, metricNames)
if err != nil {
return BatchSeriesResult{}, err
}
result := BatchSeriesResult{
Definitions: definitions,
Values: make(map[string]map[Aggregation][]AggregatePoint, len(query.Specs)),
}
accumulators := make(map[string]*metricSeriesAccumulator, len(query.Specs))
groups := make(map[batchSeriesGroupKey]*batchSeriesGroup)
for _, spec := range query.Specs {
values := make(map[Aggregation][]AggregatePoint, len(spec.Aggregations))
accumulator := &metricSeriesAccumulator{
spec: spec, compression: s.cfg.RollupPolicy.compression(),
groups: make(map[rollupKey]*rollupAggregateState), rateGroups: make(map[rollupKey]*rollupAggregateState),
}
for _, aggregation := range spec.Aggregations {
values[aggregation] = []AggregatePoint{}
if aggregation == AggRate {
accumulator.needRate = true
} else {
accumulator.needSummary = true
}
if isPercentile(aggregation) {
accumulator.needDigest = true
}
}
result.Values[spec.MetricName] = values
accumulators[spec.MetricName] = accumulator
policy := s.cfg.RollupPolicy
if definition, ok := definitions[spec.MetricName]; ok {
if definition.RetentionDays <= 0 {
continue
}
policy = policy.withMetricRetention(time.Duration(definition.RetentionDays) * 24 * time.Hour)
}
if len(policy.Tiers) == 0 {
continue
}
resolution := seriesResolutionForPolicy(query.Start, spec.Interval, now, policy)
if forced, ok := forcedResolutions[spec.MetricName]; ok {
resolution = forced
}
key := batchSeriesGroupKey{resolution: resolution, needDigest: accumulator.needDigest}
group := groups[key]
if group == nil {
group = &batchSeriesGroup{key: key, metricNames: make(map[string]struct{})}
groups[key] = group
}
group.metricNames[spec.MetricName] = struct{}{}
group.fields |= rollupFieldsForAggregations(spec.Aggregations)
}
seriesByID, seriesByIdentity, err := s.loadSeriesDictionary(ctx, seriesDictionaryPlan{
MetricNames: metricNames,
EntityIDs: query.EntityIDs,
Tags: query.Tags,
})
if err != nil {
return BatchSeriesResult{}, err
}
orderedGroups := make([]*batchSeriesGroup, 0, len(groups))
for _, group := range groups {
orderedGroups = append(orderedGroups, group)
}
sort.Slice(orderedGroups, func(i, j int) bool {
if orderedGroups[i].key.resolution != orderedGroups[j].key.resolution {
return orderedGroups[i].key.resolution < orderedGroups[j].key.resolution
}
return !orderedGroups[i].key.needDigest && orderedGroups[j].key.needDigest
})
s.rollupViewMu.RLock()
defer s.rollupViewMu.RUnlock()
for _, group := range orderedGroups {
if err := s.scanPersistedRollupGroup(ctx, query, group, seriesByID, accumulators); err != nil {
return BatchSeriesResult{}, err
}
if err := s.scanMemoryRollupGroup(query, group, seriesByIdentity, accumulators); err != nil {
return BatchSeriesResult{}, err
}
}
for metricName, accumulator := range accumulators {
values, err := accumulator.points(query.Order)
if err != nil {
return BatchSeriesResult{}, err
}
result.Values[metricName] = values
}
return result, nil
}
func (s *Store) loadBatchDefinitions(ctx context.Context, metricNames []string) (map[string]Definition, error) {
args := make([]any, 0, len(metricNames))
placeholders := make([]string, 0, len(metricNames))
for _, metricName := range metricNames {
args = append(args, metricName)
placeholders = append(placeholders, s.dialect.placeholder(len(args)))
}
rows, err := s.reader().QueryContext(ctx, fmt.Sprintf(
"SELECT name, type, unit, description, retention_days, metadata, created_at_milli, updated_at_milli FROM %s WHERE name IN (%s) ORDER BY name ASC",
s.tables.definitions, strings.Join(placeholders, ", "),
), args...)
if err != nil {
return nil, err
}
defer rows.Close()
definitions := make(map[string]Definition, len(metricNames))
for rows.Next() {
definition, err := scanDefinition(rows)
if err != nil {
return nil, err
}
definitions[definition.Name] = definition
}
return definitions, rows.Err()
}
func (s *Store) loadSeriesDictionary(ctx context.Context, plan seriesDictionaryPlan) (map[int64]*seriesReadMeta, map[seriesIdentity]*seriesReadMeta, error) {
rendered := s.dialect.renderSeriesDictionary(s.tables, s.cfg.TablePrefix+"series_metric_entity_idx", plan)
rows, err := s.reader().QueryContext(ctx, rendered.Query, rendered.Args...)
if err != nil {
return nil, nil, err
}
defer rows.Close()
byID := make(map[int64]*seriesReadMeta)
byIdentity := make(map[seriesIdentity]*seriesReadMeta)
for rows.Next() {
var meta seriesReadMeta
var rawTags any
if err := rows.Scan(&meta.id, &meta.metricName, &meta.entityID, &meta.tagsHash, &rawTags); err != nil {
return nil, nil, err
}
meta.tagsJSON, err = rawJSONToString(rawTags)
if err != nil {
return nil, nil, err
}
meta.tags, err = rollupTagsFromJSON(meta.tagsJSON)
if err != nil {
return nil, nil, err
}
byID[meta.id] = &meta
byIdentity[seriesIdentity{metricName: meta.metricName, entityID: meta.entityID, tagsHash: meta.tagsHash}] = &meta
}
return byID, byIdentity, rows.Err()
}
func (s *Store) scanPersistedRollupGroup(ctx context.Context, query BatchSeriesQuery, group *batchSeriesGroup, seriesByID map[int64]*seriesReadMeta, accumulators map[string]*metricSeriesAccumulator) error {
seriesIDs := make([]int64, 0)
for seriesID, meta := range seriesByID {
if _, ok := group.metricNames[meta.metricName]; ok {
seriesIDs = append(seriesIDs, seriesID)
}
}
if len(seriesIDs) == 0 {
return nil
}
sort.Slice(seriesIDs, func(i, j int) bool { return seriesIDs[i] < seriesIDs[j] })
resolutionID, found, err := s.resolveResolutionID(ctx, group.key.resolution)
if err != nil || !found {
return err
}
rendered := s.dialect.renderRollupRead(s.tables, s.cfg.TablePrefix+"rollups_resolution_bucket_idx", rollupReadPlan{
SeriesIDs: seriesIDs,
ResolutionID: resolutionID,
StartMilli: bucketStartMillis(query.Start.UnixMilli(), group.key.resolution.Milliseconds()),
EndMilli: query.End.UnixMilli(),
Fields: group.fields,
})
rows, err := s.reader().QueryContext(ctx, rendered.Query, rendered.Args...)
if err != nil {
return err
}
defer rows.Close()
var seriesID, bucket, count, firstTS, lastTS int64
var sum, sumSq, min, max, firstVal, lastVal float64
var digestBlob []byte
destinations := []any{&seriesID, &bucket, &count}
if group.fields&rollupReadSum != 0 {
destinations = append(destinations, &sum)
}
if group.fields&rollupReadSumSq != 0 {
destinations = append(destinations, &sumSq)
}
if group.fields&rollupReadMin != 0 {
destinations = append(destinations, &min)
}
if group.fields&rollupReadMax != 0 {
destinations = append(destinations, &max)
}
if group.fields&rollupReadFirst != 0 {
destinations = append(destinations, &firstVal, &firstTS)
}
if group.fields&rollupReadLast != 0 {
destinations = append(destinations, &lastVal, &lastTS)
}
if group.fields&rollupReadDigest != 0 {
destinations = append(destinations, &digestBlob)
}
for rows.Next() {
err = rows.Scan(destinations...)
if err != nil {
return err
}
meta := seriesByID[seriesID]
accumulator := accumulators[meta.metricName]
state := accumulator.consume(meta, bucket, count, sum, sumSq, min, max, firstVal, firstTS, lastVal, lastTS, nil)
if accumulator.needDigest {
if err := mergeEncodedRollupDigest(state.summary.digest, count, min, max, digestBlob); err != nil {
return err
}
}
}
return rows.Err()
}
func rollupFieldsForAggregations(aggregations []Aggregation) rollupReadFields {
var fields rollupReadFields
for _, aggregation := range aggregations {
switch aggregation {
case AggAvg, AggSum:
fields |= rollupReadSum
case AggMin:
fields |= rollupReadMin
case AggMax:
fields |= rollupReadMax
case AggFirst:
fields |= rollupReadFirst
case AggLast, AggRate:
fields |= rollupReadLast
case AggStdDev:
fields |= rollupReadSum | rollupReadSumSq
default:
if isPercentile(aggregation) {
fields |= rollupReadMin | rollupReadMax | rollupReadDigest
}
}
}
return fields
}
func (s *Store) resolveResolutionID(ctx context.Context, resolution time.Duration) (int64, bool, error) {
var resolutionID int64
err := s.reader().QueryRowContext(ctx,
"SELECT id FROM "+s.tables.resolutions+" WHERE resolution_milli = "+s.dialect.placeholder(1),
resolution.Milliseconds(),
).Scan(&resolutionID)
if err == sql.ErrNoRows {
return 0, false, nil
}
return resolutionID, err == nil, err
}
func (s *Store) scanMemoryRollupGroup(query BatchSeriesQuery, group *batchSeriesGroup, seriesByIdentity map[seriesIdentity]*seriesReadMeta, accumulators map[string]*metricSeriesAccumulator) error {
if group.key.resolution == time.Minute {
return s.scanHotRollupGroup(query, group, seriesByIdentity, accumulators)
}
return s.scanCoarseRollupGroup(query, group, seriesByIdentity, accumulators)
}
func (s *Store) scanHotRollupGroup(query BatchSeriesQuery, group *batchSeriesGroup, seriesByIdentity map[seriesIdentity]*seriesReadMeta, accumulators map[string]*metricSeriesAccumulator) error {
start := bucketStartMillis(query.Start.UnixMilli(), time.Minute.Milliseconds())
end := query.End.UnixMilli()
entityIDs := stringSet(query.EntityIDs)
s.hotMu.RLock()
defer s.hotMu.RUnlock()
for key, bucket := range s.hot {
if _, ok := group.metricNames[key.metricName]; !ok || key.bucket < start || key.bucket > end || bucket.count == 0 {
continue
}
if len(entityIDs) > 0 {
if _, ok := entityIDs[key.entityID]; !ok {
continue
}
}
meta, matched, err := memorySeriesMeta(seriesByIdentity, key.metricName, key.entityID, key.tagsHash, bucket.tagsJSON, query.Tags)
if err != nil {
return err
}
if !matched {
continue
}
accumulators[key.metricName].consume(meta, key.bucket, bucket.count, bucket.sum, bucket.sumSq, bucket.min, bucket.max, bucket.firstVal, bucket.firstTS, bucket.lastVal, bucket.lastTS, bucket.digest)
}
return nil
}
func (s *Store) scanCoarseRollupGroup(query BatchSeriesQuery, group *batchSeriesGroup, seriesByIdentity map[seriesIdentity]*seriesReadMeta, accumulators map[string]*metricSeriesAccumulator) error {
start := bucketStartMillis(query.Start.UnixMilli(), group.key.resolution.Milliseconds())
end := query.End.UnixMilli()
entityIDs := stringSet(query.EntityIDs)
s.coarseMu.RLock()
defer s.coarseMu.RUnlock()
for key, parent := range s.coarse {
if key.interval != group.key.resolution || key.bucket < start || key.bucket > end {
continue
}
if _, ok := group.metricNames[key.metricName]; !ok {
continue
}
if len(entityIDs) > 0 {
if _, ok := entityIDs[key.entityID]; !ok {
continue
}
}
childKeys := make([]rollupKey, 0, len(parent.children))
for childKey := range parent.children {
childKeys = append(childKeys, childKey)
}
if len(childKeys) == 0 {
continue
}
sortRollupKeys(childKeys)
firstChild := parent.children[childKeys[0]]
meta, matched, err := memorySeriesMeta(seriesByIdentity, key.metricName, key.entityID, key.tagsHash, firstChild.tagsJSON, query.Tags)
if err != nil {
return err
}
if !matched {
continue
}
summary := newRollupBucketWithDigest(s.cfg.RollupPolicy.compression(), group.key.needDigest)
for _, childKey := range childKeys {
child := parent.children[childKey]
mergeRollupValues(summary, child.count, child.sum, child.sumSq, child.min, child.max, child.firstVal, child.firstTS, child.lastVal, child.lastTS, child.digest, group.key.needDigest)
}
accumulators[key.metricName].consume(meta, key.bucket, summary.count, summary.sum, summary.sumSq, summary.min, summary.max, summary.firstVal, summary.firstTS, summary.lastVal, summary.lastTS, summary.digest)
}
return nil
}
func memorySeriesMeta(seriesByIdentity map[seriesIdentity]*seriesReadMeta, metricName, entityID, tagsHash, tagsJSON string, filter map[string]string) (*seriesReadMeta, bool, error) {
identity := seriesIdentity{metricName: metricName, entityID: entityID, tagsHash: tagsHash}
if meta := seriesByIdentity[identity]; meta != nil {
return meta, true, nil
}
tags, err := rollupTagsFromJSON(tagsJSON)
if err != nil {
return nil, false, err
}
for key, value := range filter {
if tags[key] != value {
return nil, false, nil
}
}
meta := &seriesReadMeta{metricName: metricName, entityID: entityID, tagsHash: tagsHash, tagsJSON: tagsJSON, tags: tags}
seriesByIdentity[identity] = meta
return meta, true, nil
}
func stringSet(values []string) map[string]struct{} {
set := make(map[string]struct{}, len(values))
for _, value := range values {
set[value] = struct{}{}
}
return set
}
func (a *metricSeriesAccumulator) consume(meta *seriesReadMeta, sourceBucket, count int64, sum, sumSq, min, max, firstVal float64, firstTS int64, lastVal float64, lastTS int64, digest *TDigest) *rollupAggregateState {
key := rollupKey{bucket: bucketStartMillis(sourceBucket, a.spec.Interval.Milliseconds())}
if a.spec.PreserveSeries {
key.entityID = meta.entityID
key.tagsHash = meta.tagsHash
}
if a.needSummary {
state := a.groups[key]
if state == nil {
state = &rollupAggregateState{summary: newRollupBucketWithDigest(a.compression, a.needDigest)}
if a.spec.PreserveSeries {
state.tags = meta.tags
}
a.groups[key] = state
}
mergeRollupValues(state.summary, count, sum, sumSq, min, max, firstVal, firstTS, lastVal, lastTS, digest, a.needDigest)
if !a.needRate {
return state
}
}
if a.needRate {
rateKey := rollupKey{
bucket: bucketStartMillis(sourceBucket, a.spec.Interval.Milliseconds()),
entityID: meta.entityID,
tagsHash: meta.tagsHash,
}
state := a.rateGroups[rateKey]
if state == nil {
state = &rollupAggregateState{tags: meta.tags}
a.rateGroups[rateKey] = state
}
state.rateSamples = append(state.rateSamples, rollupRateSample{timestamp: lastTS, value: lastVal})
}
return a.groups[key]
}
func mergeEncodedRollupDigest(target *TDigest, count int64, min, max float64, blob []byte) error {
if count == 0 {
return nil
}
if len(blob) == 0 {
if min != max {
return fmt.Errorf("metric: missing t-digest for rollup with %d samples", count)
}
target.Add(min, float64(count))
return nil
}
digest, err := DecodeTDigest(blob)
if err != nil {
return err
}
target.Merge(digest)
return nil
}
func mergeRollupValues(target *rollupBucket, count int64, sum, sumSq, min, max, firstVal float64, firstTS int64, lastVal float64, lastTS int64, digest *TDigest, includeDigest bool) {
if count == 0 {
return
}
if target.count == 0 {
target.min, target.max = min, max
target.firstVal, target.firstTS = firstVal, firstTS
target.lastVal, target.lastTS = lastVal, lastTS
} else {
if min < target.min {
target.min = min
}
if max > target.max {
target.max = max
}
if firstTS < target.firstTS {
target.firstVal, target.firstTS = firstVal, firstTS
}
if lastTS > target.lastTS {
target.lastVal, target.lastTS = lastVal, lastTS
}
}
target.count += count
target.sum += sum
target.sumSq += sumSq
if includeDigest && digest != nil {
target.digest.Merge(digest)
}
}
func (a *metricSeriesAccumulator) points(order Order) (map[Aggregation][]AggregatePoint, error) {
values := make(map[Aggregation][]AggregatePoint, len(a.spec.Aggregations))
var summaryKeys []rollupKey
if a.needSummary {
summaryKeys = make([]rollupKey, 0, len(a.groups))
for key := range a.groups {
summaryKeys = append(summaryKeys, key)
}
sortRollupKeys(summaryKeys)
}
var rateKeys []rollupKey
if a.needRate {
rateKeys = make([]rollupKey, 0, len(a.rateGroups))
for key, state := range a.rateGroups {
rateKeys = append(rateKeys, key)
sort.SliceStable(state.rateSamples, func(i, j int) bool { return state.rateSamples[i].timestamp < state.rateSamples[j].timestamp })
}
sortRollupKeys(rateKeys)
}
for _, aggregation := range a.spec.Aggregations {
keys := summaryKeys
if aggregation == AggRate {
keys = rateKeys
}
start, end, step := 0, len(keys), 1
if order == OrderDesc {
start, end, step = len(keys)-1, -1, -1
}
points := make([]AggregatePoint, 0, len(keys))
for keyIndex := start; keyIndex != end; keyIndex += step {
key := keys[keyIndex]
state := a.groups[key]
count, value := 0, 0.0
if aggregation == AggRate {
state = a.rateGroups[key]
value = rollupCounterRate(state.rateSamples)
count = len(state.rateSamples)
} else {
var ok bool
value, ok = state.summary.value(aggregation)
if !ok {
return nil, fmt.Errorf("%w: aggregation %q requires raw samples", ErrInvalidArgument, aggregation)
}
count = int(state.summary.count)
}
points = append(points, AggregatePoint{
MetricName: a.spec.MetricName,
EntityID: key.entityID,
Bucket: fromMillis(key.bucket),
Value: value,
Count: count,
Tags: state.tags,
})
}
values[aggregation] = points
}
return values, nil
}
func rollupCounterRate(samples []rollupRateSample) float64 {
if len(samples) < 2 {
return 0
}
seconds := float64(samples[len(samples)-1].timestamp-samples[0].timestamp) / 1000
if seconds <= 0 {
return 0
}
increase := 0.0
for i := 1; i < len(samples); i++ {
if delta := samples[i].value - samples[i-1].value; delta > 0 {
increase += delta
}
}
return increase / seconds
}
+6
View File
@@ -104,6 +104,12 @@ func (s *Store) addCoarseChildren(metricName string, interval time.Duration, hig
// late-arrival window. It flushes one tier at a time so a sealed child can feed
// the next parent without any periodic rewrite of that parent.
func (s *Store) flushClosedCoarseRollups(ctx context.Context, now time.Time) (int, error) {
s.rollupViewMu.Lock()
defer s.rollupViewMu.Unlock()
return s.flushClosedCoarseRollupsUnderView(ctx, now)
}
func (s *Store) flushClosedCoarseRollupsUnderView(ctx context.Context, now time.Time) (int, error) {
now = now.UTC()
written := 0
for {
+3 -3
View File
@@ -158,11 +158,11 @@ func DefaultConfig(driver Driver, dsn string) Config {
SQLite: SQLiteOptions{
PerformanceProfile: SQLiteProfileBalanced,
BusyTimeout: 5 * time.Second,
CacheSizeKB: 16 * 1024,
CacheSizeKB: 4 * 1024,
TempStoreMemory: false,
MMapSizeBytes: 32 * 1024 * 1024,
MMapSizeBytes: 0,
WALAutoCheckpoint: 4000,
JournalSizeLimitBytes: 16 * 1024 * 1024,
JournalSizeLimitBytes: 4 * 1024 * 1024,
},
RollupPolicy: RollupPolicy{
RawRetention: 10 * time.Minute,
+2 -62
View File
@@ -35,6 +35,8 @@ type dialect interface {
//
// blobType 是 rollup t-digest sketch 的列类型。
blobType() string
renderSeriesDictionary(tables tables, indexName string, plan seriesDictionaryPlan) renderedSQL
renderRollupRead(tables tables, indexName string, plan rollupReadPlan) renderedSQL
}
// tables stores the physical table names used by a Store.
@@ -287,65 +289,3 @@ func (postgresDialect) jsonExtractEquals(column, key, placeholder string) string
// escaping.
return fmt.Sprintf("%s->>'%s' = %s", column, sqlSingleQuote(key), placeholder)
}
// sqlAggValueExpr returns the SQL value expression that computes agg over the
// "value" column for the given driver, and whether that aggregation can be
// pushed down to this backend. Aggregations that need the ordered raw series
// (first/last/rate) are never pushed down here. Percentiles and population
// standard deviation are pushed down only where the backend has a portable,
// semantics-matching function:
//
// - STDDEV_POP: MySQL and PostgreSQL (matches the in-memory population stddev,
// which divides by N). SQLite has no built-in, so it falls back to memory.
// - percentile_cont(p) WITHIN GROUP (ORDER BY value): PostgreSQL only. This
// matches the in-memory linear-interpolation percentile for ANY p (p50,
// p95, p99, p99.9, ...). MySQL lacks a portable continuous-percentile
// function and SQLite has none, so both fall back to memory.
//
// The simple reductions (avg/min/max/sum/count) are portable everywhere.
//
// sqlAggValueExpr 返回给定驱动上对 "value" 列计算 agg 的 SQL 值表达式,
// 以及该聚合是否能下推到该后端。需要有序原始序列的聚合(first/last/rate)
// 永远不会在这里下推。百分位和总体标准差只会在后端提供可移植且语义匹配的函数时
// 下推:
//
// - STDDEV_POP:MySQL 和 PostgreSQL 支持(匹配内存中的总体标准差,
// 即除以 N)。SQLite 没有内置函数,因此回退到内存计算。
// - percentile_cont(p) WITHIN GROUP (ORDER BY value):仅 PostgreSQL 支持。
// 它匹配内存中任意 p(p50、p95、p99、p99.9 等)的线性插值百分位。
// MySQL 没有可移植的连续百分位函数,SQLite 也没有,因此两者都回退到内存计算。
//
// 简单归约(avg/min/max/sum/count)在所有后端都可移植。
func sqlAggValueExpr(driver Driver, agg Aggregation) (string, bool) {
switch agg {
case AggAvg:
return "AVG(value)", true
case AggMin:
return "MIN(value)", true
case AggMax:
return "MAX(value)", true
case AggSum:
return "SUM(value)", true
case AggCount:
return "COUNT(*)", true
case AggStdDev:
switch driver {
case DriverMySQL, DriverPostgreSQL:
return "STDDEV_POP(value)", true
default:
return "", false
}
default:
// Any percentile (fixed p50/p95/p99 and arbitrary pXX / pXX.X all land
// here). PostgreSQL has a portable continuous percentile that matches the
// in-memory linear-interpolation percentile; other backends fall back to
// the in-memory path.
if frac, ok := percentileFractionString(agg); ok {
if driver == DriverPostgreSQL {
return fmt.Sprintf("percentile_cont(%s) WITHIN GROUP (ORDER BY value)", frac), true
}
return "", false
}
return "", false
}
}
+24 -3
View File
@@ -60,7 +60,11 @@ func (s *Store) writePreparedHotRollups(ctx context.Context, prepared []prepared
if err != nil {
return err
}
_, err = s.flushClosedCoarseRollups(ctx, now)
if len(rebuild) > 0 {
_, err = s.flushClosedCoarseRollupsUnderView(ctx, now)
} else {
_, err = s.flushClosedCoarseRollups(ctx, now)
}
return err
}
@@ -78,8 +82,25 @@ func (s *Store) rebuildHotRollupsLocked(keys map[hotRollupKey]struct{}, compress
if labelID, ok := series.labelIDs[key.labelsHash]; ok {
bucket.labelsJSON = series.labelsJSON[labelID]
}
for _, sample := range series.allSamples() {
if bucketStartMillis(sample.timestamp, minuteMillis) != key.bucket || series.labelHashes[sample.labelID] != key.labelsHash {
bucketEnd := key.bucket + minuteMillis
decoder := newRawSampleDecoder(series.compressed)
for sample, more := decoder.next(); more; sample, more = decoder.next() {
if sample.timestamp < key.bucket {
continue
}
if sample.timestamp >= bucketEnd {
break
}
if series.labelHashes[sample.labelID] != key.labelsHash {
continue
}
bucket.labelsJSON = series.labelsJSON[sample.labelID]
bucket.addPoint(sample.value, sample.timestamp)
}
directStart := sort.Search(len(series.samples), func(i int) bool { return series.samples[i].timestamp >= key.bucket })
directEnd := sort.Search(len(series.samples), func(i int) bool { return series.samples[i].timestamp >= bucketEnd })
for _, sample := range series.samples[directStart:directEnd] {
if series.labelHashes[sample.labelID] != key.labelsHash {
continue
}
bucket.labelsJSON = series.labelsJSON[sample.labelID]
+32 -17
View File
@@ -634,7 +634,7 @@ func TestSeriesMergesLabelsWithoutCollapsingRawShape(t *testing.T) {
}
}
func TestCoveringSeriesResolutionUsesFinestTierCoveringStart(t *testing.T) {
func TestSeriesResolutionDoesNotInspectStoredCoverage(t *testing.T) {
ctx := context.Background()
policy := RollupPolicy{
Tiers: []RollupTier{
@@ -686,16 +686,13 @@ func TestCoveringSeriesResolutionUsesFinestTierCoveringStart(t *testing.T) {
t.Fatalf("commit seed tiers: %v", err)
}
resolution, err := s.coveringSeriesResolution(ctx, Query{MetricName: "coverage-tier", EntityID: "n1", Start: now.Add(-6 * time.Hour), End: now}, time.Minute)
if err != nil {
t.Fatalf("select covering tier: %v", err)
}
if resolution != 5*time.Minute {
t.Fatalf("covering tier = %s, want 5m", resolution)
resolution := seriesResolutionForPolicy(now.Add(-6*time.Hour), time.Minute, now, policy)
if resolution != time.Minute {
t.Fatalf("selected tier = %s, want 1m", resolution)
}
}
func TestSeriesFallsBackWhenPreferredPolicyTierHasNoRows(t *testing.T) {
func TestSeriesReturnsEmptyWhenSelectedTierHasNoRows(t *testing.T) {
ctx := context.Background()
policy := RollupPolicy{
Tiers: []RollupTier{
@@ -743,8 +740,8 @@ func TestSeriesFallsBackWhenPreferredPolicyTierHasNoRows(t *testing.T) {
if err != nil {
t.Fatalf("query series: %v", err)
}
if len(got) != 1 || got[0].Value != 42 || got[0].Count != 1 {
t.Fatalf("series did not fall back to existing hourly tier: %#v", got)
if len(got) != 0 {
t.Fatalf("series read a non-selected hourly tier: %#v", got)
}
}
@@ -931,15 +928,33 @@ func TestSeriesHasNoTrailingEmptyBucketAcrossDashboardRanges(t *testing.T) {
t.Fatalf("create metric: %v", err)
}
now := time.Date(2026, 7, 24, 12, 0, 0, 0, time.UTC)
points := make([]Point, 0, 60*24*60)
for at := now.Add(-60 * 24 * time.Hour); at.Before(now); at = at.Add(time.Minute) {
points = append(points, Point{MetricName: "coverage", EntityID: "n1", Timestamp: at, Value: 1})
tagsHash, tagsJSON, err := tagsFingerprint(nil)
if err != nil {
t.Fatalf("fingerprint tags: %v", err)
}
if err := s.WriteBatch(ctx, points); err != nil {
t.Fatalf("write history: %v", err)
tx, err := s.db.BeginTx(ctx, nil)
if err != nil {
t.Fatalf("begin history transaction: %v", err)
}
if _, err := s.Compact(ctx, now); err != nil {
t.Fatalf("compact history: %v", err)
metricPolicy := policy.withMetricRetention(90 * 24 * time.Hour)
for _, tierIndex := range []int{0, 2, 3} {
tier := metricPolicy.Tiers[tierIndex]
buckets := make(map[rollupKey]*rollupBucket)
start := bucketStartMillis(now.Add(-tier.Retention).UnixMilli(), tier.Interval.Milliseconds())
for bucketStart := start; bucketStart < now.UnixMilli(); bucketStart += tier.Interval.Milliseconds() {
bucket := newRollupBucket(policy.compression())
bucket.tagsHash, bucket.tagsJSON = tagsHash, tagsJSON
bucket.labelsHash, bucket.labelsJSON = emptyLabelsHash, "{}"
bucket.addPoint(1, bucketStart)
buckets[rollupKey{entityID: "n1", tagsHash: tagsHash, labelsHash: emptyLabelsHash, bucket: bucketStart}] = bucket
}
if _, err := s.writeRollupBucketsTx(ctx, "coverage", tier.Interval, buckets, tx); err != nil {
_ = tx.Rollback()
t.Fatalf("seed %s history: %v", tier.Interval, err)
}
}
if err := tx.Commit(); err != nil {
t.Fatalf("commit history: %v", err)
}
for _, duration := range []time.Duration{time.Hour, 6 * time.Hour, 7 * 24 * time.Hour, 30 * 24 * time.Hour, 60 * 24 * time.Hour} {
interval := s.CompatibleSeriesInterval(now.Add(-duration), now, time.Minute)
+33 -64
View File
@@ -104,10 +104,11 @@ func TestTagFilterPushdownWithPaging(t *testing.T) {
}
}
// TestSQLAggregateMatchesInMemory compares SQL and in-memory aggregation.
// TestAggregateMatchesAggregatePoints compares Store aggregation with the
// standalone aggregation helper.
//
// TestSQLAggregateMatchesInMemory 对比 SQL 下推聚合和内存聚合结果。
func TestSQLAggregateMatchesInMemory(t *testing.T) {
// TestAggregateMatchesAggregatePoints 对比 Store 聚合和独立聚合 helper 的结果。
func TestAggregateMatchesAggregatePoints(t *testing.T) {
ctx := context.Background()
s := newMemStore(t)
if err := s.CreateMetric(ctx, Definition{Name: "g", Type: TypeGauge, RetentionDays: 30}); err != nil {
@@ -131,10 +132,9 @@ func TestSQLAggregateMatchesInMemory(t *testing.T) {
Aggregation: AggAvg,
Interval: 15 * time.Second,
}
// SQL pushdown path.
sqlRes, err := s.Aggregate(ctx, q)
storeRes, err := s.Aggregate(ctx, q)
if err != nil {
t.Fatalf("sql aggregate: %v", err)
t.Fatalf("store aggregate: %v", err)
}
// In-memory reference over the same raw points.
pts, err := s.Query(ctx, q.Query)
@@ -145,12 +145,12 @@ func TestSQLAggregateMatchesInMemory(t *testing.T) {
if err != nil {
t.Fatalf("mem aggregate: %v", err)
}
if len(sqlRes) != len(memRes) {
t.Fatalf("bucket count mismatch: sql=%d mem=%d", len(sqlRes), len(memRes))
if len(storeRes) != len(memRes) {
t.Fatalf("bucket count mismatch: store=%d helper=%d", len(storeRes), len(memRes))
}
for i := range sqlRes {
if !sqlRes[i].Bucket.Equal(memRes[i].Bucket) || sqlRes[i].Value != memRes[i].Value || sqlRes[i].Count != memRes[i].Count {
t.Fatalf("bucket %d mismatch: sql=%#v mem=%#v", i, sqlRes[i], memRes[i])
for i := range storeRes {
if !storeRes[i].Bucket.Equal(memRes[i].Bucket) || storeRes[i].Value != memRes[i].Value || storeRes[i].Count != memRes[i].Count {
t.Fatalf("bucket %d mismatch: store=%#v helper=%#v", i, storeRes[i], memRes[i])
}
}
}
@@ -260,10 +260,10 @@ func TestStdDevPopMatchesCalculateStats(t *testing.T) {
}
}
// TestAggregateStdDevSQLiteUsesMemoryPath verifies SQLite stddev fallback.
// TestAggregateStdDev verifies raw population standard deviation aggregation.
//
// TestAggregateStdDevSQLiteUsesMemoryPath 验证 SQLite 标准差聚合会回退到内存路径。
func TestAggregateStdDevSQLiteUsesMemoryPath(t *testing.T) {
// TestAggregateStdDev 验证原始样本总体标准差聚合。
func TestAggregateStdDev(t *testing.T) {
ctx := context.Background()
s := newMemStore(t)
if err := s.CreateMetric(ctx, Definition{Name: "sd", Type: TypeGauge, RetentionDays: 30}); err != nil {
@@ -294,36 +294,6 @@ func TestAggregateStdDevSQLiteUsesMemoryPath(t *testing.T) {
}
}
// TestSQLAggValueExprPushdownMatrix verifies aggregation pushdown support.
//
// TestSQLAggValueExprPushdownMatrix 验证各后端支持的聚合下推矩阵。
func TestSQLAggValueExprPushdownMatrix(t *testing.T) {
cases := []struct {
driver Driver
agg Aggregation
ok bool
}{
{DriverSQLite, AggAvg, true},
{DriverSQLite, AggStdDev, false}, // no STDDEV_POP in sqlite -> memory
{DriverSQLite, AggP95, false}, // no percentile in sqlite -> memory
{DriverMySQL, AggStdDev, true}, // STDDEV_POP
{DriverMySQL, AggP95, false}, // no portable continuous percentile
{DriverPostgreSQL, AggStdDev, true},
{DriverPostgreSQL, AggP50, true}, // percentile_cont
{DriverPostgreSQL, AggP99, true},
{DriverPostgreSQL, AggFirst, false}, // needs ordered raw series
}
for _, c := range cases {
expr, ok := sqlAggValueExpr(c.driver, c.agg)
if ok != c.ok {
t.Fatalf("%s/%s: pushdown=%v want %v (expr=%q)", c.driver, c.agg, ok, c.ok, expr)
}
if ok && expr == "" {
t.Fatalf("%s/%s: pushdown ok but empty expr", c.driver, c.agg)
}
}
}
// TestSQLiteReadPoolOpens verifies SQLite read-pool creation.
//
// TestSQLiteReadPoolOpens 验证 SQLite 只读连接池会按配置打开。
@@ -403,10 +373,10 @@ func TestWriteBatchAtomicAcrossChunks(t *testing.T) {
}
}
// TestAggregateBucketPagingSQLPath verifies bucket paging in SQL aggregation.
// TestAggregateBucketPaging verifies bucket paging.
//
// TestAggregateBucketPagingSQLPath 验证 SQL 聚合路径按桶分页。
func TestAggregateBucketPagingSQLPath(t *testing.T) {
// TestAggregateBucketPaging 验证聚合桶分页。
func TestAggregateBucketPaging(t *testing.T) {
ctx := context.Background()
s := newMemStore(t)
if err := s.CreateMetric(ctx, Definition{Name: "bp", Type: TypeGauge, RetentionDays: 30}); err != nil {
@@ -441,10 +411,11 @@ func TestAggregateBucketPagingSQLPath(t *testing.T) {
}
}
// TestAggregateBucketPagingMemoryPathMatchesSQL compares bucket paging paths.
// TestAggregateBucketPagingMatchesAcrossAggregations compares bucket paging
// across aggregation types.
//
// TestAggregateBucketPagingMemoryPathMatchesSQL 对比内存聚合和 SQL 聚合的桶分页语义。
func TestAggregateBucketPagingMemoryPathMatchesSQL(t *testing.T) {
// TestAggregateBucketPagingMatchesAcrossAggregations 对比不同聚合类型的桶分页语义。
func TestAggregateBucketPagingMatchesAcrossAggregations(t *testing.T) {
ctx := context.Background()
s := newMemStore(t)
if err := s.CreateMetric(ctx, Definition{Name: "bp2", Type: TypeGauge, RetentionDays: 30}); err != nil {
@@ -459,28 +430,26 @@ func TestAggregateBucketPagingMemoryPathMatchesSQL(t *testing.T) {
t.Fatalf("write: %v", err)
}
q := Query{MetricName: "bp2", EntityID: "n1", Start: base, End: base.Add(time.Minute)}
// AggMin (SQL pushdown) and AggP50 (memory fallback on sqlite) with identical
// bucket paging must select the same bucket window (the same timestamps).
sqlPaged, err := s.Aggregate(ctx, AggregateQuery{Query: q, Aggregation: AggMin, Interval: 10 * time.Second, BucketLimit: 3, BucketOffset: 2})
minPaged, err := s.Aggregate(ctx, AggregateQuery{Query: q, Aggregation: AggMin, Interval: 10 * time.Second, BucketLimit: 3, BucketOffset: 2})
if err != nil {
t.Fatalf("sql aggregate: %v", err)
t.Fatalf("min aggregate: %v", err)
}
memPaged, err := s.Aggregate(ctx, AggregateQuery{Query: q, Aggregation: AggP50, Interval: 10 * time.Second, BucketLimit: 3, BucketOffset: 2})
p50Paged, err := s.Aggregate(ctx, AggregateQuery{Query: q, Aggregation: AggP50, Interval: 10 * time.Second, BucketLimit: 3, BucketOffset: 2})
if err != nil {
t.Fatalf("mem aggregate: %v", err)
t.Fatalf("p50 aggregate: %v", err)
}
if len(sqlPaged) != 3 || len(memPaged) != 3 {
t.Fatalf("expected 3 buckets each, got sql=%d mem=%d", len(sqlPaged), len(memPaged))
if len(minPaged) != 3 || len(p50Paged) != 3 {
t.Fatalf("expected 3 buckets each, got min=%d p50=%d", len(minPaged), len(p50Paged))
}
for i := range sqlPaged {
if !sqlPaged[i].Bucket.Equal(memPaged[i].Bucket) {
t.Fatalf("bucket window mismatch at %d: sql=%v mem=%v", i, sqlPaged[i].Bucket, memPaged[i].Bucket)
for i := range minPaged {
if !minPaged[i].Bucket.Equal(p50Paged[i].Bucket) {
t.Fatalf("bucket window mismatch at %d: min=%v p50=%v", i, minPaged[i].Bucket, p50Paged[i].Bucket)
}
}
// Per-bucket single point, so min == p50 == the value.
for i := range sqlPaged {
if sqlPaged[i].Value != memPaged[i].Value {
t.Fatalf("value mismatch at %d: sql=%v mem=%v", i, sqlPaged[i].Value, memPaged[i].Value)
for i := range minPaged {
if minPaged[i].Value != p50Paged[i].Value {
t.Fatalf("value mismatch at %d: min=%v p50=%v", i, minPaged[i].Value, p50Paged[i].Value)
}
}
}
+3
View File
@@ -125,6 +125,9 @@ func (s *Store) ReclaimSpace(ctx context.Context) error {
if _, err := s.db.ExecContext(ctx, sqliteVacuumSQL); err != nil {
return fmt.Errorf("metric: vacuum sqlite database: %w", err)
}
if _, err := s.db.ExecContext(ctx, "PRAGMA optimize"); err != nil {
return fmt.Errorf("metric: optimize sqlite database: %w", err)
}
// VACUUM itself can populate the WAL; truncate it again so the reported
// physical size reflects the completed reclamation.
return sqliteCheckpoint(ctx, s.db)
+8 -1
View File
@@ -21,7 +21,14 @@ func (s *Store) Migrate(ctx context.Context) error {
return err
}
}
return s.createNormalizedIndexes(ctx)
if err := s.createNormalizedIndexes(ctx); err != nil {
return err
}
if s.cfg.Driver == DriverSQLite {
_, err := s.db.ExecContext(ctx, "PRAGMA optimize")
return err
}
return nil
}
func (s *Store) normalizedSchemaStatements() []string {
+13 -7
View File
@@ -18,7 +18,7 @@ func TestPostgreSQLIntegration(t *testing.T) {
t.Skip("METRIC_POSTGRES_DSN is not set")
}
runSQLIntegration(t, "postgres", PostgreSQL(dsn), true)
runSQLIntegration(t, "postgres", PostgreSQL(dsn))
runSQLRestructureIntegration(t, "postgres", PostgreSQL(dsn))
}
@@ -31,14 +31,24 @@ func TestMySQLIntegration(t *testing.T) {
t.Skip("METRIC_MYSQL_DSN is not set")
}
runSQLIntegration(t, "mysql", MySQL(dsn), false)
runSQLIntegration(t, "mysql", MySQL(dsn))
runSQLRestructureIntegration(t, "mysql", MySQL(dsn))
}
func TestMariaDBIntegration(t *testing.T) {
dsn := os.Getenv("METRIC_MARIADB_DSN")
if dsn == "" {
t.Skip("METRIC_MARIADB_DSN is not set")
}
runSQLIntegration(t, "mariadb", MySQL(dsn))
runSQLRestructureIntegration(t, "mariadb", MySQL(dsn))
}
// runSQLIntegration exercises the SQL store against an external database.
//
// runSQLIntegration 在外部数据库上执行通用 SQL 集成测试流程。
func runSQLIntegration(t *testing.T, name string, cfg Config, expectSQLPercentile bool) {
func runSQLIntegration(t *testing.T, name string, cfg Config) {
t.Helper()
ctx, cancel := context.WithTimeout(context.Background(), 6*time.Minute)
@@ -160,10 +170,6 @@ func runSQLIntegration(t *testing.T, name string, cfg Config, expectSQLPercentil
if len(p95Buckets) != 1 || p95Buckets[0].Count != 3 || p95Buckets[0].Value <= 28 {
t.Fatalf("unexpected p95 aggregate: %#v", p95Buckets)
}
if _, ok := sqlAggValueExpr(cfg.Driver, AggP95); ok != expectSQLPercentile {
t.Fatalf("%s p95 pushdown expectation mismatch: got %v want %v", name, ok, expectSQLPercentile)
}
latest, err := store.Latest(ctx, "http.latency", "api-1", 1)
if err != nil {
t.Fatalf("latest: %v", err)
+308 -173
View File
@@ -38,6 +38,61 @@ type compressedRawSamples struct {
lastStamp int64
}
type rawSampleDecoder struct {
data []byte
offset int
index int
count int
previous int64
}
func newRawSampleDecoder(samples compressedRawSamples) rawSampleDecoder {
return rawSampleDecoder{data: samples.data, count: samples.count}
}
func (d *rawSampleDecoder) next() (rawSample, bool) {
if d.index == d.count {
return rawSample{}, false
}
var timestamp int64
if d.index == 0 {
value, size := binary.Varint(d.data[d.offset:])
d.offset += size
timestamp = value
} else {
delta, size := binary.Uvarint(d.data[d.offset:])
d.offset += size
timestamp = d.previous + int64(delta)
}
value := math.Float64frombits(binary.LittleEndian.Uint64(d.data[d.offset:]))
d.offset += 8
labelID, size := binary.Uvarint(d.data[d.offset:])
d.offset += size
d.index++
d.previous = timestamp
return rawSample{timestamp: timestamp, value: value, labelID: uint32(labelID)}, true
}
func newCompressedRawSamples(capacity int) compressedRawSamples {
return compressedRawSamples{data: make([]byte, 0, capacity)}
}
func (c *compressedRawSamples) appendSample(sample rawSample) {
if c.count == 0 {
c.data = appendVarint(c.data, sample.timestamp)
} else {
c.data = appendUvarint(c.data, uint64(sample.timestamp-c.lastStamp))
}
c.data = appendRawSample(c.data, sample)
c.count++
c.lastStamp = sample.timestamp
}
func (c compressedRawSamples) firstStamp() int64 {
value, _ := binary.Varint(c.data)
return value
}
// rawSeries stores exact samples compactly for one logical series. Tags are
// shared by the series and labels are interned into small integer ids.
type rawSeries struct {
@@ -170,31 +225,98 @@ func markRawRebuild(rebuild map[hotRollupKey]struct{}, point preparedMetricPoint
rebuild[hotRollupKey{metricName: point.metricName, entityID: point.entityID, tagsHash: point.tagsHash, labelsHash: labelsHash, bucket: bucketStartMillis(point.timestamp, time.Minute.Milliseconds())}] = struct{}{}
}
type rawBatchPoints struct {
points []Point
tagsHashes []string
order Order
}
func (p *rawBatchPoints) Len() int { return len(p.points) }
func (p *rawBatchPoints) Less(i, j int) bool {
if p.order == OrderDesc {
return rawPointLess(p.points[j], p.tagsHashes[j], p.points[i], p.tagsHashes[i])
}
return rawPointLess(p.points[i], p.tagsHashes[i], p.points[j], p.tagsHashes[j])
}
func (p *rawBatchPoints) Swap(i, j int) {
p.points[i], p.points[j] = p.points[j], p.points[i]
p.tagsHashes[i], p.tagsHashes[j] = p.tagsHashes[j], p.tagsHashes[i]
}
func (p *rawBatchPoints) grow(additional int) {
if additional <= cap(p.points)-len(p.points) {
return
}
pointCount := len(p.points) + additional
points := make([]Point, len(p.points), pointCount)
copy(points, p.points)
p.points = points
tagsHashes := make([]string, len(p.tagsHashes), pointCount)
copy(tagsHashes, p.tagsHashes)
p.tagsHashes = tagsHashes
}
func (s *Store) queryRawPoints(ctx context.Context, query Query) ([]Point, error) {
entityIDs := []string(nil)
if query.EntityID != "" {
entityIDs = []string{query.EntityID}
}
batch, err := s.queryRawPointsBatch(ctx, BatchQuery{
MetricNames: []string{query.MetricName},
EntityIDs: entityIDs,
Start: query.Start,
End: query.End,
Tags: query.Tags,
Order: query.Order,
})
if err != nil {
return nil, err
}
points := batch[query.MetricName]
startIndex := query.Offset
if startIndex > len(points) {
startIndex = len(points)
}
endIndex := len(points)
if query.Limit > 0 && startIndex+query.Limit < endIndex {
endIndex = startIndex + query.Limit
}
return points[startIndex:endIndex], nil
}
func (s *Store) queryRawPointsBatch(ctx context.Context, query BatchQuery) (map[string][]Point, error) {
if err := ctx.Err(); err != nil {
return nil, err
}
start, end := query.Start.UnixMilli(), query.End.UnixMilli()
s.rawMu.Lock()
defer s.rawMu.Unlock()
now := time.Now().UTC()
visibleCutoff := now.Add(-rawMemoryRetention).UnixMilli()
visibleCutoff := time.Now().UTC().Add(-rawMemoryRetention).UnixMilli()
if start < visibleCutoff {
start = visibleCutoff
}
s.compressRawBeforeLocked("", now.Add(-directRawRetention).UnixMilli())
s.pruneRawBeforeLocked("", rawMemoryCutoff(now))
type resultPoint struct {
point Point
entityID string
tagsHash string
metricNames := make(map[string]struct{}, len(query.MetricNames))
matched := make(map[string]*rawBatchPoints, len(query.MetricNames))
for _, metricName := range query.MetricNames {
metricNames[metricName] = struct{}{}
matched[metricName] = &rawBatchPoints{order: query.Order}
}
matched := make([]resultPoint, 0)
entityIDs := make(map[string]struct{}, len(query.EntityIDs))
for _, entityID := range query.EntityIDs {
entityIDs[entityID] = struct{}{}
}
s.rawMu.RLock()
defer s.rawMu.RUnlock()
for key, series := range s.raw {
if key.metricName != query.MetricName || (query.EntityID != "" && key.entityID != query.EntityID) {
if _, ok := metricNames[key.metricName]; !ok {
continue
}
if len(entityIDs) > 0 {
if _, ok := entityIDs[key.entityID]; !ok {
continue
}
}
tags, ok, err := matchRawTags(series.tagsJSON, query.Tags)
if err != nil {
return nil, err
@@ -202,68 +324,88 @@ func (s *Store) queryRawPoints(ctx context.Context, query Query) ([]Point, error
if !ok {
continue
}
all := series.allSamples()
labels := make(map[uint32]map[string]string)
first := sort.Search(len(all), func(i int) bool { return all[i].timestamp >= start })
for _, sample := range all[first:] {
if sample.timestamp > end {
break
}
labelMap := labels[sample.labelID]
if labelMap == nil {
labelMap, err = decodeMapString(series.labelsJSON[sample.labelID])
directStart := sort.Search(len(series.samples), func(i int) bool { return series.samples[i].timestamp >= start })
directEnd := sort.Search(len(series.samples), func(i int) bool { return series.samples[i].timestamp > end })
additional := directEnd - directStart
if series.compressed.count > 0 && start <= series.compressed.firstStamp() && end >= series.compressed.lastStamp {
additional += series.compressed.count
}
metricPoints := matched[key.metricName]
metricPoints.grow(additional)
if series.compressed.count > 0 && start <= series.compressed.lastStamp && end >= series.compressed.firstStamp() {
decoder := newRawSampleDecoder(series.compressed)
for sample, more := decoder.next(); more; sample, more = decoder.next() {
if sample.timestamp < start {
continue
}
if sample.timestamp > end {
break
}
labelMap, err := rawSampleLabels(series, labels, sample.labelID)
if err != nil {
return nil, err
}
labels[sample.labelID] = labelMap
appendRawQueryPoint(metricPoints, key, tags, labelMap, sample)
}
matched = append(matched, resultPoint{
point: Point{MetricName: key.metricName, EntityID: key.entityID, Timestamp: fromMillis(sample.timestamp), Value: sample.value, Tags: cloneStringMap(tags), Labels: cloneStringMap(labelMap)},
entityID: key.entityID,
tagsHash: key.tagsHash,
})
}
for _, sample := range series.samples[directStart:directEnd] {
labelMap, err := rawSampleLabels(series, labels, sample.labelID)
if err != nil {
return nil, err
}
appendRawQueryPoint(metricPoints, key, tags, labelMap, sample)
}
}
sort.Slice(matched, func(i, j int) bool {
left, right := matched[i], matched[j]
less := left.point.Timestamp.Before(right.point.Timestamp)
if left.point.Timestamp.Equal(right.point.Timestamp) {
less = left.entityID < right.entityID || (left.entityID == right.entityID && left.tagsHash < right.tagsHash)
}
if query.Order == OrderDesc {
return !less && !(left.point.Timestamp.Equal(right.point.Timestamp) && left.entityID == right.entityID && left.tagsHash == right.tagsHash)
}
return less
})
startIndex := query.Offset
if startIndex > len(matched) {
startIndex = len(matched)
}
endIndex := len(matched)
if query.Limit > 0 && startIndex+query.Limit < endIndex {
endIndex = startIndex + query.Limit
}
result := make([]Point, 0, endIndex-startIndex)
for _, item := range matched[startIndex:endIndex] {
result = append(result, item.point)
result := make(map[string][]Point, len(matched))
for metricName, metricPoints := range matched {
sort.Sort(metricPoints)
result[metricName] = metricPoints.points
}
return result, nil
}
func rawSampleLabels(series *rawSeries, labels map[uint32]map[string]string, labelID uint32) (map[string]string, error) {
if labelMap := labels[labelID]; labelMap != nil {
return labelMap, nil
}
labelMap, err := decodeMapString(series.labelsJSON[labelID])
if err != nil {
return nil, err
}
labels[labelID] = labelMap
return labelMap, nil
}
func appendRawQueryPoint(points *rawBatchPoints, key rawSeriesKey, tags, labels map[string]string, sample rawSample) {
points.points = append(points.points, Point{
MetricName: key.metricName, EntityID: key.entityID,
Timestamp: fromMillis(sample.timestamp), Value: sample.value, Tags: tags, Labels: labels,
})
points.tagsHashes = append(points.tagsHashes, key.tagsHash)
}
func rawPointLess(left Point, leftTagsHash string, right Point, rightTagsHash string) bool {
if !left.Timestamp.Equal(right.Timestamp) {
return left.Timestamp.Before(right.Timestamp)
}
if left.EntityID != right.EntityID {
return left.EntityID < right.EntityID
}
return leftTagsHash < rightTagsHash
}
func (s *Store) rawEntityIDs(ctx context.Context, query Query) ([]string, error) {
if err := ctx.Err(); err != nil {
return nil, err
}
start, end := query.Start.UnixMilli(), query.End.UnixMilli()
s.rawMu.Lock()
defer s.rawMu.Unlock()
now := time.Now().UTC()
visibleCutoff := now.Add(-rawMemoryRetention).UnixMilli()
s.rawMu.RLock()
defer s.rawMu.RUnlock()
visibleCutoff := time.Now().UTC().Add(-rawMemoryRetention).UnixMilli()
if start < visibleCutoff {
start = visibleCutoff
}
s.compressRawBeforeLocked("", now.Add(-directRawRetention).UnixMilli())
s.pruneRawBeforeLocked("", rawMemoryCutoff(now))
seen := make(map[string]struct{})
for key, series := range s.raw {
if key.metricName != query.MetricName || (query.EntityID != "" && key.entityID != query.EntityID) {
@@ -276,9 +418,25 @@ func (s *Store) rawEntityIDs(ctx context.Context, query Query) ([]string, error)
if !ok {
continue
}
all := series.allSamples()
index := sort.Search(len(all), func(i int) bool { return all[i].timestamp >= start })
if index < len(all) && all[index].timestamp <= end {
found := false
if series.compressed.count > 0 && start <= series.compressed.lastStamp && end >= series.compressed.firstStamp() {
decoder := newRawSampleDecoder(series.compressed)
for sample, more := decoder.next(); more; sample, more = decoder.next() {
if sample.timestamp < start {
continue
}
if sample.timestamp > end {
break
}
found = true
break
}
}
if !found {
index := sort.Search(len(series.samples), func(i int) bool { return series.samples[i].timestamp >= start })
found = index < len(series.samples) && series.samples[index].timestamp <= end
}
if found {
seen[key.entityID] = struct{}{}
}
}
@@ -355,32 +513,48 @@ func (s *Store) pruneRawBeforeLocked(metricName string, beforeMilli int64) int64
if metricName != "" && key.metricName != metricName {
continue
}
compressed := series.decodeCompressed()
compressedIndex := sort.Search(len(compressed), func(i int) bool { return compressed[i].timestamp >= beforeMilli })
if compressedIndex > 0 {
compressed = compressed[compressedIndex:]
series.compressed.set(compressed)
deleted += int64(compressedIndex)
}
if len(compressed) == 0 {
series.compressed = compressedRawSamples{}
}
compressedDeleted := pruneCompressedRawSamples(&series.compressed, beforeMilli)
deleted += int64(compressedDeleted)
directIndex := sort.Search(len(series.samples), func(i int) bool { return series.samples[i].timestamp >= beforeMilli })
if directIndex > 0 {
deleted += int64(directIndex)
series.samples = append([]rawSample(nil), series.samples[directIndex:]...)
copy(series.samples, series.samples[directIndex:])
series.samples = series.samples[:len(series.samples)-directIndex]
}
if series.compressed.count == 0 && len(series.samples) == 0 {
delete(s.raw, key)
continue
}
if compressedIndex > 0 || directIndex > 0 {
if compressedDeleted > 0 || directIndex > 0 {
compactRawLabels(series)
}
}
return deleted
}
func pruneCompressedRawSamples(samples *compressedRawSamples, beforeMilli int64) int {
if samples.count == 0 || samples.firstStamp() >= beforeMilli {
return 0
}
if samples.lastStamp < beforeMilli {
deleted := samples.count
*samples = compressedRawSamples{}
return deleted
}
decoder := newRawSampleDecoder(*samples)
kept := newCompressedRawSamples(len(samples.data))
deleted := 0
for sample, more := decoder.next(); more; sample, more = decoder.next() {
if sample.timestamp < beforeMilli {
deleted++
continue
}
kept.appendSample(sample)
}
*samples = kept
return deleted
}
func (s *Store) compressRawBeforeLocked(metricName string, beforeMilli int64) {
for key, series := range s.raw {
if metricName != "" && key.metricName != metricName {
@@ -391,70 +565,38 @@ func (s *Store) compressRawBeforeLocked(metricName string, beforeMilli int64) {
continue
}
series.compressed.append(series.samples[:index])
series.samples = append([]rawSample(nil), series.samples[index:]...)
}
}
func (s *rawSeries) decodeCompressed() []rawSample {
if s.compressed.count == 0 {
return nil
}
result := make([]rawSample, 0, s.compressed.count)
for offset, previous := 0, int64(0); offset < len(s.compressed.data); {
var timestamp int64
if len(result) == 0 {
first, n := binary.Varint(s.compressed.data[offset:])
if n <= 0 {
return result
}
offset += n
timestamp = first
} else {
delta, dn := binary.Uvarint(s.compressed.data[offset:])
if dn <= 0 {
return result
}
offset += dn
timestamp = previous + int64(delta)
copy(series.samples, series.samples[index:])
series.samples = series.samples[:len(series.samples)-index]
if len(series.samples) == 0 {
series.samples = nil
}
if offset+8 > len(s.compressed.data) {
return result
}
value := math.Float64frombits(binary.LittleEndian.Uint64(s.compressed.data[offset:]))
offset += 8
labelID, ln := binary.Uvarint(s.compressed.data[offset:])
if ln <= 0 {
return result
}
offset += ln
result = append(result, rawSample{timestamp: timestamp, value: value, labelID: uint32(labelID)})
previous = timestamp
}
return result
}
func (s *rawSeries) allSamples() []rawSample {
compressed := s.decodeCompressed()
result := make([]rawSample, 0, len(compressed)+len(s.samples))
result = append(result, compressed...)
result = append(result, s.samples...)
return result
}
func (s *rawSeries) replaceCompressed(sample rawSample) (rawSample, bool) {
samples := s.decodeCompressed()
index := sort.Search(len(samples), func(i int) bool { return samples[i].timestamp >= sample.timestamp })
var old rawSample
replaced := index < len(samples) && samples[index].timestamp == sample.timestamp
if replaced {
old = samples[index]
samples[index] = sample
} else {
samples = append(samples, rawSample{})
copy(samples[index+1:], samples[index:])
samples[index] = sample
replaced := false
inserted := false
decoder := newRawSampleDecoder(s.compressed)
encoded := newCompressedRawSamples(len(s.compressed.data) + binary.MaxVarintLen64 + 9)
for current, more := decoder.next(); more; current, more = decoder.next() {
if !inserted && sample.timestamp < current.timestamp {
encoded.appendSample(sample)
inserted = true
}
if !inserted && sample.timestamp == current.timestamp {
old = current
replaced = true
encoded.appendSample(sample)
inserted = true
continue
}
encoded.appendSample(current)
}
s.compressed.set(samples)
if !inserted {
encoded.appendSample(sample)
}
s.compressed = encoded
return old, replaced
}
@@ -463,51 +605,39 @@ func (c *compressedRawSamples) append(samples []rawSample) {
return
}
if c.count > 0 && samples[0].timestamp <= c.lastStamp {
merged := make([]rawSample, 0, c.count+len(samples))
merged = append(merged, c.decode()...)
for _, sample := range samples {
index := sort.Search(len(merged), func(i int) bool { return merged[i].timestamp >= sample.timestamp })
if index < len(merged) && merged[index].timestamp == sample.timestamp {
merged[index] = sample
decoder := newRawSampleDecoder(*c)
merged := newCompressedRawSamples(len(c.data) + len(samples)*10)
current, more := decoder.next()
for sampleIndex := 0; more || sampleIndex < len(samples); {
if !more {
merged.appendSample(samples[sampleIndex])
sampleIndex++
continue
}
merged = append(merged, rawSample{})
copy(merged[index+1:], merged[index:])
merged[index] = sample
if sampleIndex == len(samples) {
merged.appendSample(current)
current, more = decoder.next()
continue
}
next := samples[sampleIndex]
switch {
case next.timestamp < current.timestamp:
merged.appendSample(next)
sampleIndex++
case next.timestamp == current.timestamp:
merged.appendSample(next)
sampleIndex++
current, more = decoder.next()
default:
merged.appendSample(current)
current, more = decoder.next()
}
}
c.set(merged)
*c = merged
return
}
for _, sample := range samples {
if c.count == 0 {
c.data = appendVarint(c.data, sample.timestamp)
} else {
c.data = appendUvarint(c.data, uint64(sample.timestamp-c.lastStamp))
}
c.data = appendRawSample(c.data, sample)
c.count++
c.lastStamp = sample.timestamp
}
}
func (c *compressedRawSamples) decode() []rawSample {
series := rawSeries{compressed: *c}
return series.decodeCompressed()
}
func (c *compressedRawSamples) set(samples []rawSample) {
c.data = nil
c.count = 0
c.lastStamp = 0
for _, sample := range samples {
if c.count == 0 {
c.data = appendVarint(c.data, sample.timestamp)
} else {
c.data = appendUvarint(c.data, uint64(sample.timestamp-c.lastStamp))
}
c.data = appendRawSample(c.data, sample)
c.count++
c.lastStamp = sample.timestamp
c.appendSample(sample)
}
}
@@ -534,9 +664,12 @@ func compactRawLabels(series *rawSeries) {
if len(series.labelHashes) <= 1 {
return
}
all := series.allSamples()
used := make([]bool, len(series.labelHashes))
for _, sample := range all {
decoder := newRawSampleDecoder(series.compressed)
for sample, more := decoder.next(); more; sample, more = decoder.next() {
used[sample.labelID] = true
}
for _, sample := range series.samples {
used[sample.labelID] = true
}
usedCount := 0
@@ -563,11 +696,13 @@ func compactRawLabels(series *rawSeries) {
labelsJSON = append(labelsJSON, series.labelsJSON[oldID])
labelIDs[hash] = newID
}
compressed := series.decodeCompressed()
for i := range compressed {
compressed[i].labelID = remap[compressed[i].labelID]
decoder = newRawSampleDecoder(series.compressed)
compressed := newCompressedRawSamples(len(series.compressed.data))
for sample, more := decoder.next(); more; sample, more = decoder.next() {
sample.labelID = remap[sample.labelID]
compressed.appendSample(sample)
}
series.compressed.set(compressed)
series.compressed = compressed
for i := range series.samples {
series.samples[i].labelID = remap[series.samples[i].labelID]
}
+113 -6
View File
@@ -124,14 +124,121 @@ func TestCompressedRawSamplesRoundTripExactly(t *testing.T) {
}
var compressed compressedRawSamples
compressed.append(samples)
decoded := compressed.decode()
if len(decoded) != len(samples) {
t.Fatalf("decoded sample count = %d, want %d", len(decoded), len(samples))
}
decoder := newRawSampleDecoder(compressed)
for i := range samples {
if decoded[i].timestamp != samples[i].timestamp || decoded[i].labelID != samples[i].labelID || math.Float64bits(decoded[i].value) != math.Float64bits(samples[i].value) {
t.Fatalf("decoded sample %d = %#v, want %#v", i, decoded[i], samples[i])
decoded, more := decoder.next()
if !more {
t.Fatalf("decoded sample count = %d, want %d", i, len(samples))
}
if decoded.timestamp != samples[i].timestamp || decoded.labelID != samples[i].labelID || math.Float64bits(decoded.value) != math.Float64bits(samples[i].value) {
t.Fatalf("decoded sample %d = %#v, want %#v", i, decoded, samples[i])
}
}
if _, more := decoder.next(); more {
t.Fatal("decoder returned extra sample")
}
}
func TestCompressedRawSamplesMergeInsertionAndReplacement(t *testing.T) {
base := time.Now().UTC().Add(-5 * time.Minute).UnixMilli()
var compressed compressedRawSamples
compressed.append([]rawSample{
{timestamp: base, value: 1, labelID: 0},
{timestamp: base + 2000, value: 3, labelID: 2},
})
compressed.append([]rawSample{
{timestamp: base + 1000, value: math.Copysign(0, -1), labelID: 1},
{timestamp: base + 2000, value: 33, labelID: 3},
})
want := []rawSample{
{timestamp: base, value: 1, labelID: 0},
{timestamp: base + 1000, value: math.Copysign(0, -1), labelID: 1},
{timestamp: base + 2000, value: 33, labelID: 3},
}
decoder := newRawSampleDecoder(compressed)
for i := range want {
got, more := decoder.next()
if !more || got.timestamp != want[i].timestamp || got.labelID != want[i].labelID || math.Float64bits(got.value) != math.Float64bits(want[i].value) {
t.Fatalf("sample %d = %#v, want %#v", i, got, want[i])
}
}
if _, more := decoder.next(); more {
t.Fatal("decoder returned extra merged sample")
}
}
func TestRawQueryCompressedDirectBoundaryClosedRange(t *testing.T) {
ctx := context.Background()
s := newMemStore(t)
if err := s.CreateMetric(ctx, Definition{Name: "raw-boundary", RetentionDays: 1}); err != nil {
t.Fatal(err)
}
now := time.Now().UTC().Truncate(time.Millisecond)
compressedAt := now.Add(-time.Minute)
directAt := compressedAt.Add(time.Millisecond)
series := &rawSeries{
tagsJSON: "{}", labelIDs: map[string]uint32{emptyLabelsHash: 0},
labelHashes: []string{emptyLabelsHash}, labelsJSON: []string{"{}"},
samples: []rawSample{{timestamp: directAt.UnixMilli(), value: 2}},
}
series.compressed.append([]rawSample{{timestamp: compressedAt.UnixMilli(), value: 1}})
s.raw[rawSeriesKey{metricName: "raw-boundary", entityID: "n1", tagsHash: emptyLabelsHash}] = series
ascending, err := s.Query(ctx, Query{MetricName: "raw-boundary", Start: compressedAt, End: directAt, Order: OrderAsc})
if err != nil {
t.Fatal(err)
}
if len(ascending) != 2 || ascending[0].Value != 1 || ascending[1].Value != 2 {
t.Fatalf("closed ascending range = %#v", ascending)
}
descending, err := s.Query(ctx, Query{MetricName: "raw-boundary", Start: compressedAt, End: directAt, Order: OrderDesc, Offset: 1, Limit: 1})
if err != nil {
t.Fatal(err)
}
if len(descending) != 1 || descending[0].Value != 1 {
t.Fatalf("paged descending range = %#v", descending)
}
}
func TestRawPruneStreamsAndReclaimsLabels(t *testing.T) {
ctx := context.Background()
s := newMemStore(t)
if err := s.CreateMetric(ctx, Definition{Name: "raw-prune", RetentionDays: 1}); err != nil {
t.Fatal(err)
}
now := time.Now().UTC().Truncate(time.Millisecond)
cutoff := now.Add(-3 * time.Minute)
series := &rawSeries{
tagsJSON: "{}",
labelIDs: map[string]uint32{"old": 0, "middle": 1, "keep": 2},
labelHashes: []string{"old", "middle", "keep"},
labelsJSON: []string{`{"label":"old"}`, `{"label":"middle"}`, `{"label":"keep"}`},
samples: []rawSample{{timestamp: now.Add(-30 * time.Second).UnixMilli(), value: 4, labelID: 2}},
}
series.compressed.append([]rawSample{
{timestamp: now.Add(-5 * time.Minute).UnixMilli(), value: 1, labelID: 0},
{timestamp: now.Add(-4 * time.Minute).UnixMilli(), value: 2, labelID: 1},
{timestamp: cutoff.UnixMilli(), value: 3, labelID: 2},
})
s.raw[rawSeriesKey{metricName: "raw-prune", entityID: "n1", tagsHash: emptyLabelsHash}] = series
if deleted := s.deleteRawBefore("raw-prune", cutoff.UnixMilli()); deleted != 2 {
t.Fatalf("deleted %d samples, want 2", deleted)
}
if len(series.labelsJSON) != 1 || series.labelsJSON[0] != `{"label":"keep"}` || series.compressed.count != 1 || series.samples[0].labelID != 0 {
t.Fatalf("compacted series = %#v", series)
}
decoder := newRawSampleDecoder(series.compressed)
kept, more := decoder.next()
if !more || kept.timestamp != cutoff.UnixMilli() || kept.labelID != 0 || kept.value != 3 {
t.Fatalf("kept compressed sample = %#v", kept)
}
points, err := s.Query(ctx, Query{MetricName: "raw-prune", Start: cutoff, End: now})
if err != nil {
t.Fatal(err)
}
if len(points) != 2 || points[0].Labels["label"] != "keep" || points[1].Labels["label"] != "keep" {
t.Fatalf("pruned query = %#v", points)
}
}
+2 -85
View File
@@ -832,7 +832,7 @@ func TestValidateNormalizedRestructureRejectsMissingDigest(t *testing.T) {
if err := s.CreateMetric(ctx, Definition{Name: "invalid", Type: TypeGauge, RetentionDays: 1}); err != nil {
t.Fatal(err)
}
at := time.Now().UTC().Add(-2 * time.Minute)
at := time.Now().UTC().Truncate(time.Minute).Add(-2 * time.Minute)
if err := s.WriteBatch(ctx, []Point{
{MetricName: "invalid", EntityID: "n1", Timestamp: at, Value: 1},
{MetricName: "invalid", EntityID: "n1", Timestamp: at.Add(time.Second), Value: 2},
@@ -857,7 +857,7 @@ func TestValidateNormalizedRestructureRejectsWrongDigestCompression(t *testing.T
if err := s.CreateMetric(ctx, Definition{Name: "invalid-compression", Type: TypeGauge, RetentionDays: 1}); err != nil {
t.Fatal(err)
}
at := time.Now().UTC().Add(-2 * time.Minute)
at := time.Now().UTC().Truncate(time.Minute).Add(-2 * time.Minute)
if err := s.WriteBatch(ctx, []Point{
{MetricName: "invalid-compression", EntityID: "n1", Timestamp: at, Value: 1},
{MetricName: "invalid-compression", EntityID: "n1", Timestamp: at.Add(time.Second), Value: 2},
@@ -929,7 +929,6 @@ func TestRestructureDump(t *testing.T) {
if err := s.ReclaimSpace(ctx); err != nil {
t.Fatalf("reclaim rebuilt store: %v", err)
}
assertRestructuredDumpQueryContinuity(t, ctx, s)
after, err := s.StorageSize(ctx)
if err != nil {
t.Fatalf("measure rebuilt storage: %v", err)
@@ -944,85 +943,3 @@ func TestRestructureDump(t *testing.T) {
}
t.Logf("restructured rows=%d metrics=%d before=%d after=%d saved=%d percent=%.2f", result.RowsCopied, result.Metrics, before, after, saved, percent)
}
func assertRestructuredDumpQueryContinuity(t *testing.T, ctx context.Context, s *Store) {
t.Helper()
var metricName, entityID string
var latestMilli int64
err := s.db.QueryRowContext(ctx, fmt.Sprintf(`SELECT series.metric_name, series.entity_id,
(SELECT MAX(recent.last_ts_milli) FROM %s recent
JOIN %s recent_series ON recent_series.id = recent.series_id
WHERE recent_series.metric_name = series.metric_name AND recent_series.entity_id = series.entity_id)
FROM %s daily
JOIN %s series ON series.id = daily.series_id
JOIN %s daily_resolution ON daily_resolution.id = daily.resolution_id
WHERE daily_resolution.resolution_milli = ?
GROUP BY series.metric_name, series.entity_id
ORDER BY COUNT(DISTINCT daily.bucket_milli) DESC, series.metric_name, series.entity_id LIMIT 1`,
s.tables.rollups, s.tables.series,
s.tables.rollups, s.tables.series, s.tables.resolutions),
(24*time.Hour).Milliseconds()).Scan(&metricName, &entityID, &latestMilli)
if err != nil {
t.Fatalf("select migrated series: %v", err)
}
now := fromMillis(latestMilli).Add(time.Millisecond)
for _, duration := range []time.Duration{time.Hour, 6 * time.Hour, 7 * 24 * time.Hour, 30 * 24 * time.Hour, 60 * 24 * time.Hour} {
start := now.Add(-duration)
interval := s.CompatibleSeriesInterval(start, now, time.Minute)
resolution, err := s.coveringSeriesResolution(ctx, Query{MetricName: metricName, EntityID: entityID, Start: start, End: now}, interval)
if err != nil {
t.Fatalf("select %s backing tier: %v", duration, err)
}
series, err := s.Series(ctx, AggregateQuery{
Query: Query{MetricName: metricName, EntityID: entityID, Start: start, End: now},
Aggregation: AggAvg,
Interval: interval,
}, now)
if err != nil {
t.Fatalf("query %s: %v", duration, err)
}
if len(series) == 0 || series[len(series)-1].Count == 0 {
t.Fatalf("query %s has no final data bucket: %#v", duration, series)
}
expectedStep := max(interval, resolution)
maxGap := time.Duration(0)
for i := 1; i < len(series); i++ {
gap := series[i].Bucket.Sub(series[i-1].Bucket)
if gap > maxGap {
maxGap = gap
}
}
if maxGap > 2*expectedStep {
t.Fatalf("query %s has a %s discontinuity larger than two %s buckets", duration, maxGap, expectedStep)
}
tailGap := now.Sub(series[len(series)-1].Bucket)
if tailGap < 0 || tailGap > 2*expectedStep {
t.Fatalf("query %s final data is %s from the query end, expected at most %s", duration, tailGap, 2*expectedStep)
}
t.Logf("range=%s metric=%s entity=%s requested=%s backing=%s points=%d first=%s last=%s max_gap=%s tail_gap=%s",
duration, metricName, entityID, interval, resolution, len(series), series[0].Bucket, series[len(series)-1].Bucket, maxGap, tailGap)
}
}
func TestRestructuredDumpQueryContinuity(t *testing.T) {
path := os.Getenv("KOMARI_METRIC_RESTRUCTURE_RESULT")
if path == "" {
t.Skip("set KOMARI_METRIC_RESTRUCTURE_RESULT to validate a rebuilt dump")
}
ctx := context.Background()
s, err := Open(ctx, SQLite(path, WithAutoMigrate(false), WithMaxOpenConns(1), WithRollupPolicy(RollupPolicy{
RawRetention: 15 * time.Minute,
Tiers: []RollupTier{
{Interval: time.Minute, Retention: 600 * time.Minute},
{Interval: 5 * time.Minute, Retention: 600 * 5 * time.Minute},
{Interval: time.Hour, Retention: 600 * time.Hour},
{Interval: 24 * time.Hour, Retention: 100 * 365 * 24 * time.Hour},
},
Compression: 30,
})))
if err != nil {
t.Fatalf("open rebuilt dump: %v", err)
}
defer s.Close()
assertRestructuredDumpQueryContinuity(t, ctx, s)
}
+8 -8
View File
@@ -348,12 +348,10 @@ func TestSeriesHybridIncludesPartialCoarseBucketAtRawCutoff(t *testing.T) {
now := time.Date(2026, 6, 18, 12, 50, 0, 0, time.UTC)
old := now.Add(-40 * time.Minute)
recent := now.Add(-5 * time.Minute)
if err := s.WriteBatch(ctx, []Point{
{MetricName: "coarse-cutoff", EntityID: "n1", Timestamp: old, Value: 10},
{MetricName: "coarse-cutoff", EntityID: "n1", Timestamp: recent, Value: 100},
}); err != nil {
t.Fatalf("write: %v", err)
}
writeRollupPointsAt(t, ctx, s, now,
Point{MetricName: "coarse-cutoff", EntityID: "n1", Timestamp: old, Value: 10},
Point{MetricName: "coarse-cutoff", EntityID: "n1", Timestamp: recent, Value: 100},
)
if _, err := s.Compact(ctx, now); err != nil {
t.Fatalf("compact: %v", err)
}
@@ -367,7 +365,7 @@ func TestSeriesHybridIncludesPartialCoarseBucketAtRawCutoff(t *testing.T) {
t.Fatalf("series: %v", err)
}
if len(got) != 1 || got[0].Count != 2 || math.Abs(got[0].Value-55) > 1e-9 {
t.Fatalf("expected coarse rollup and recent raw point, got %#v", got)
t.Fatalf("expected persisted and in-memory hourly data, got %#v", got)
}
}
@@ -411,7 +409,9 @@ func TestSeriesAndAggregateRollupBridgeSealedCoarseTail(t *testing.T) {
read func() ([]AggregatePoint, error)
}{
{name: "series", read: func() ([]AggregatePoint, error) { return s.Series(ctx, query, base.Add(16*time.Minute)) }},
{name: "aggregate", read: func() ([]AggregatePoint, error) { return s.aggregateRollupAt(ctx, query, 5*time.Minute, base.Add(16*time.Minute)) }},
{name: "aggregate", read: func() ([]AggregatePoint, error) {
return s.aggregateRollupAt(ctx, query, 5*time.Minute, base.Add(16*time.Minute))
}},
} {
points, err := result.read()
if err != nil {
+193 -353
View File
@@ -7,292 +7,118 @@ import (
"time"
)
// AggregateRollup reduces one materialized tier into the requested output
// interval. The hot minute is folded in separately so recent charts never wait
// for another report before their final real bucket becomes visible.
// AggregateRollup reduces one explicitly selected materialized tier into the
// requested output interval.
func (s *Store) AggregateRollup(ctx context.Context, query AggregateQuery, resolution time.Duration) ([]AggregatePoint, error) {
return s.aggregateRollupAt(ctx, query, resolution, time.Now().UTC())
}
// aggregateRollupAt reads a sealed rollup prefix at the requested backing
// resolution, then recursively falls back through finer tiers for the
// ten-minute late-arrival tail. Series passes its supplied now value here so
// callers receive a stable view while AggregateRollup keeps its public API.
func (s *Store) aggregateRollupAt(ctx context.Context, query AggregateQuery, resolution time.Duration, now time.Time) ([]AggregatePoint, error) {
if err := s.ensureOpen(); err != nil {
return nil, err
}
if err := query.Validate(); err != nil {
return nil, err
}
if resolution <= 0 {
return nil, fmt.Errorf("%w: rollup resolution must be positive", ErrInvalidArgument)
}
s.rollupViewMu.RLock()
defer s.rollupViewMu.RUnlock()
q := query.Query.normalized()
needDigest := isPercentile(query.Aggregation)
policy, err := s.metricRollupPolicy(ctx, q.MetricName)
entityIDs := []string(nil)
if query.EntityID != "" {
entityIDs = []string{query.EntityID}
}
batch := BatchSeriesQuery{
Specs: []BatchSeriesSpec{{
MetricName: query.MetricName,
Aggregations: []Aggregation{query.Aggregation},
Interval: query.Interval,
PreserveSeries: query.PreserveSeries,
}},
EntityIDs: entityIDs,
Start: query.Start,
End: query.End,
Tags: query.Tags,
Order: query.Order,
}
if err := batch.Validate(); err != nil {
return nil, err
}
result, err := s.seriesBatchAt(ctx, batch.normalized(), now.UTC(), map[string]time.Duration{query.MetricName: resolution})
if err != nil {
return nil, err
}
rows, err := s.hybridRollupRows(ctx, q, policy, resolution, now.UTC(), needDigest)
return pageBuckets(result.Values[query.MetricName][query.Aggregation], query.BucketLimit, query.BucketOffset), nil
}
// Series selects one deterministic backing tier and delegates to SeriesBatch.
func (s *Store) Series(ctx context.Context, query AggregateQuery, now time.Time) ([]AggregatePoint, error) {
if err := s.ensureOpen(); err != nil {
return nil, err
}
if err := query.Validate(); err != nil {
return nil, err
}
entityIDs := []string(nil)
if query.EntityID != "" {
entityIDs = []string{query.EntityID}
}
result, err := s.SeriesBatch(ctx, BatchSeriesQuery{
Specs: []BatchSeriesSpec{{
MetricName: query.MetricName,
Aggregations: []Aggregation{query.Aggregation},
Interval: query.Interval,
PreserveSeries: query.PreserveSeries,
}},
EntityIDs: entityIDs,
Start: query.Start,
End: query.End,
Tags: query.Tags,
Order: query.Order,
}, now)
if err != nil {
return nil, err
}
if query.Aggregation == AggRate {
points := representativePoints(q.MetricName, rows)
return pageBuckets(mustAggregatePoints(points, query), query.BucketLimit, query.BucketOffset), nil
return pageBuckets(result.Values[query.MetricName][query.Aggregation], query.BucketLimit, query.BucketOffset), nil
}
// SeriesAggregates derives multiple aggregations from the same streamed scan.
func (s *Store) SeriesAggregates(ctx context.Context, query Query, aggregations []Aggregation, interval time.Duration, preserveSeries bool, now time.Time) (map[Aggregation][]AggregatePoint, error) {
entityIDs := []string(nil)
if query.EntityID != "" {
entityIDs = []string{query.EntityID}
}
groups := foldRollupRows(nil, rows, query.Interval, s.cfg.RollupPolicy.compression(), query.PreserveSeries, false, needDigest)
result, err := rollupGroupsToPoints(groups, query)
result, err := s.SeriesBatch(ctx, BatchSeriesQuery{
Specs: []BatchSeriesSpec{{
MetricName: query.MetricName,
Aggregations: aggregations,
Interval: interval,
PreserveSeries: preserveSeries,
}},
EntityIDs: entityIDs,
Start: query.Start,
End: query.End,
Tags: query.Tags,
Order: query.Order,
}, now)
if err != nil {
return nil, err
}
return pageBuckets(result, query.BucketLimit, query.BucketOffset), nil
return result.Values[query.MetricName], nil
}
// hybridRollupRows reads each part of a range once. A parent bucket becomes
// eligible only after its end is outside coarseRollupGrace; the remainder is
// represented by the immediate finer tier until minute buckets, where hot
// in-memory observations complete the view.
func (s *Store) hybridRollupRows(ctx context.Context, query Query, policy RollupPolicy, resolution time.Duration, now time.Time, needDigest bool) ([]storedRollup, error) {
if len(policy.Tiers) == 0 {
return nil, nil
func seriesResolutionForPolicy(start time.Time, interval time.Duration, now time.Time, policy RollupPolicy) time.Duration {
preferred := policy.Tiers[0].Interval
if tier := bestRollupTier(policy, interval, start.UTC(), now.UTC()); tier != nil {
return tier.Interval
}
index := -1
for i, tier := range policy.Tiers {
if tier.Interval <= resolution && resolution%tier.Interval == 0 {
index = i
for i := len(policy.Tiers) - 1; i >= 0; i-- {
tier := policy.Tiers[i]
if interval >= tier.Interval && interval%tier.Interval == 0 {
preferred = tier.Interval
break
}
}
if index < 0 {
return s.scanRollupRowsBetween(ctx, query.MetricName, query.EntityID, query.Tags, resolution.Milliseconds(), bucketStartMillis(query.Start.UnixMilli(), resolution.Milliseconds()), query.End.UnixMilli(), needDigest)
}
return s.hybridRollupRowsAtTier(ctx, query, policy, index, query.Start.UTC().UnixMilli(), query.End.UTC().UnixMilli(), now, needDigest)
}
func (s *Store) hybridRollupRowsAtTier(ctx context.Context, query Query, policy RollupPolicy, index int, startMilli, endMilli int64, now time.Time, needDigest bool) ([]storedRollup, error) {
if startMilli > endMilli {
return nil, nil
}
tier := policy.Tiers[index]
intervalMilli := tier.Interval.Milliseconds()
// The greatest bucket start whose end is at least coarseRollupGrace behind
// now. Buckets after it are still mutable in the in-memory parent map.
sealedThrough := bucketStartMillis(now.Add(-coarseRollupGrace-tier.Interval).UnixMilli(), intervalMilli)
rows := make([]storedRollup, 0)
if startMilli <= sealedThrough {
upper := sealedThrough
if upper > endMilli {
upper = endMilli
}
sealed, err := s.scanRollupRowsBetween(ctx, query.MetricName, query.EntityID, query.Tags, intervalMilli, bucketStartMillis(startMilli, intervalMilli), upper, needDigest)
if err != nil {
return nil, err
}
rows = append(rows, sealed...)
}
tailStart := startMilli
if tailStart <= sealedThrough {
tailStart = sealedThrough + intervalMilli
}
if tailStart > endMilli {
return rows, nil
}
if index == 0 {
minute, err := s.scanRollupRowsBetween(ctx, query.MetricName, query.EntityID, query.Tags, intervalMilli, bucketStartMillis(tailStart, intervalMilli), endMilli, needDigest)
if err != nil {
return nil, err
}
hot, err := s.hotRollupRows(query.MetricName, query.EntityID, query.Tags, fromMillis(tailStart), fromMillis(endMilli), needDigest)
if err != nil {
return nil, err
}
return append(rows, append(minute, hot...)...), nil
}
tail, err := s.hybridRollupRowsAtTier(ctx, query, policy, index-1, tailStart, endMilli, now, needDigest)
if err != nil {
return nil, err
}
return append(rows, tail...), nil
}
func mustAggregatePoints(points []Point, query AggregateQuery) []AggregatePoint {
result, err := AggregatePoints(points, query)
if err != nil {
return []AggregatePoint{}
}
return result
}
func representativePoints(metricName string, rows []storedRollup) []Point {
points := make([]Point, 0, len(rows))
for _, row := range rows {
tags, err := rollupTagsFromJSON(row.bucketData.tagsJSON)
if err != nil {
continue
}
labels, err := rollupTagsFromJSON(row.bucketData.labelsJSON)
if err != nil {
continue
}
points = append(points, Point{MetricName: metricName, EntityID: row.entityID, Timestamp: fromMillis(row.bucketData.lastTS), Value: row.bucketData.lastVal, Tags: tags, Labels: labels})
}
return points
}
func foldRollupRows(groups map[rollupKey]*rollupBucket, rows []storedRollup, interval time.Duration, compression float64, preserveSeries, preserveLabels, needDigest bool) map[rollupKey]*rollupBucket {
if groups == nil {
groups = make(map[rollupKey]*rollupBucket)
}
for _, row := range rows {
key := rollupKey{bucket: bucketStartMillis(row.bucket, interval.Milliseconds())}
if preserveSeries {
key.entityID, key.tagsHash = row.entityID, row.bucketData.tagsHash
if preserveLabels {
key.labelsHash = row.bucketData.labelsHash
}
}
bucket := groups[key]
if bucket == nil {
bucket = newRollupBucketWithDigest(compression, needDigest)
bucket.tagsHash, bucket.tagsJSON = row.bucketData.tagsHash, row.bucketData.tagsJSON
bucket.labelsHash, bucket.labelsJSON = row.bucketData.labelsHash, row.bucketData.labelsJSON
groups[key] = bucket
}
bucket.mergeStored(row.bucketData)
}
return groups
}
func rollupGroupsToPoints(groups map[rollupKey]*rollupBucket, query AggregateQuery) ([]AggregatePoint, error) {
keys := make([]rollupKey, 0, len(groups))
for key := range groups {
keys = append(keys, key)
}
sortRollupKeys(keys)
result := make([]AggregatePoint, 0, len(keys))
for _, key := range keys {
bucket := groups[key]
value, ok := bucket.value(query.Aggregation)
if !ok {
return nil, fmt.Errorf("%w: aggregation %q requires raw samples", ErrInvalidArgument, query.Aggregation)
}
tags, err := rollupTagsFromJSON(bucket.tagsJSON)
if err != nil {
return nil, err
}
if !query.PreserveSeries {
tags = map[string]string{}
}
result = append(result, AggregatePoint{MetricName: query.MetricName, EntityID: key.entityID, Bucket: fromMillis(key.bucket), Value: value, Count: int(bucket.count), Tags: tags})
}
return result, nil
}
func (s *Store) scanRollupRowsBetween(ctx context.Context, metricName, entityID string, tags map[string]string, resolutionMilli, lowerBucket, upperBucket int64, needDigest bool) ([]storedRollup, error) {
return s.scanRollupRowsBetweenWith(ctx, s.reader(), metricName, entityID, tags, time.Duration(resolutionMilli)*time.Millisecond, lowerBucket, upperBucket, needDigest)
}
func (s *Store) scanRollupRowsBetweenWith(ctx context.Context, q querier, metricName, entityID string, tags map[string]string, resolution time.Duration, lowerBucket, upperBucket int64, needDigest bool) ([]storedRollup, error) {
args := []any{metricName, resolution.Milliseconds(), lowerBucket, upperBucket}
parts := []string{
"s.metric_name = " + s.dialect.placeholder(1),
"d.resolution_milli = " + s.dialect.placeholder(2),
"r.bucket_milli >= " + s.dialect.placeholder(3),
"r.bucket_milli <= " + s.dialect.placeholder(4),
}
if entityID != "" {
args = append(args, entityID)
parts = append(parts, "s.entity_id = "+s.dialect.placeholder(len(args)))
}
for _, key := range sortedKeys(tags) {
args = append(args, tags[key])
parts = append(parts, s.dialect.jsonExtractEquals("s.tags", key, s.dialect.placeholder(len(args))))
}
columns := "s.entity_id, s.tags_hash, s.tags, l.labels_hash, l.labels, r.bucket_milli, r.count, r.sum, r.sum_sq, r.min_val, r.max_val, r.first_val, r.first_ts_milli, r.last_val, r.last_ts_milli"
if needDigest {
columns += ", r.digest"
}
sqlText := fmt.Sprintf("SELECT %s FROM %s r JOIN %s s ON s.id = r.series_id JOIN %s d ON d.id = r.resolution_id JOIN %s l ON l.id = r.label_id WHERE %s ORDER BY r.bucket_milli ASC", columns, s.tables.rollups, s.tables.series, s.tables.resolutions, s.tables.labels, joinSQLWith(parts, " AND "))
rows, err := q.QueryContext(ctx, sqlText, args...)
if err != nil {
return nil, err
}
defer rows.Close()
return scanStoredRollups(rows, needDigest, s.cfg.RollupPolicy.compression())
}
func joinSQLWith(parts []string, separator string) string {
if len(parts) == 0 {
return ""
}
out := parts[0]
for _, part := range parts[1:] {
out += separator + part
}
return out
}
func scanStoredRollups(rows *sql.Rows, needDigest bool, compression float64) ([]storedRollup, error) {
result := make([]storedRollup, 0)
for rows.Next() {
var entityID, tagsHash, labelsHash string
var rawTags, rawLabels any
var bucket, count, firstTS, lastTS int64
var sum, sumSq, min, max, firstVal, lastVal float64
var digest []byte
args := []any{&entityID, &tagsHash, &rawTags, &labelsHash, &rawLabels, &bucket, &count, &sum, &sumSq, &min, &max, &firstVal, &firstTS, &lastVal, &lastTS}
if needDigest {
args = append(args, &digest)
}
if err := rows.Scan(args...); err != nil {
return nil, err
}
tagsJSON, err := rawJSONToString(rawTags)
if err != nil {
return nil, err
}
labelsJSON, err := rawJSONToString(rawLabels)
if err != nil {
return nil, err
}
var d *TDigest
if needDigest {
d, err = digestFromRollup(count, min, max, digest, compression)
if err != nil {
return nil, err
}
}
result = append(result, storedRollup{entityID: entityID, bucket: bucket, bucketData: &rollupBucket{count: count, sum: sum, sumSq: sumSq, min: min, max: max, firstVal: firstVal, firstTS: firstTS, lastVal: lastVal, lastTS: lastTS, digest: d, tagsHash: tagsHash, tagsJSON: tagsJSON, labelsHash: labelsHash, labelsJSON: labelsJSON}})
}
return result, rows.Err()
}
func rawJSONToString(value any) (string, error) {
switch v := value.(type) {
case nil:
return "{}", nil
case string:
if v == "" {
return "{}", nil
}
return v, nil
case []byte:
if len(v) == 0 {
return "{}", nil
}
return string(v), nil
default:
return "", fmt.Errorf("unsupported JSON column type %T", value)
}
}
func rollupTagsFromJSON(raw string) (map[string]string, error) {
values, err := decodeMapString(raw)
if err != nil {
return nil, err
}
return cloneStringMap(values), nil
return preferred
}
func (s *Store) CompatibleSeriesInterval(start, now time.Time, interval time.Duration) time.Duration {
@@ -324,115 +150,129 @@ func bestRollupTier(policy RollupPolicy, interval time.Duration, start, now time
for i := range policy.Tiers {
tier := &policy.Tiers[i]
if interval >= tier.Interval && interval%tier.Interval == 0 && !now.Add(-tier.Retention).After(start) {
// Prefer the coarsest backing tier compatible with the requested
// output width. hybridRollupRows fills its unsealed tail from finer
// tiers, so this reduces reads without sacrificing recent data.
best = tier
}
}
return best
}
func (s *Store) Series(ctx context.Context, query AggregateQuery, now time.Time) ([]AggregatePoint, error) {
if err := s.ensureOpen(); err != nil {
return nil, err
// scanRollupRowsBetween is retained for compaction maintenance and internal
// verification. User-facing aggregation goes through SeriesBatch.
func (s *Store) scanRollupRowsBetween(ctx context.Context, metricName, entityID string, tags map[string]string, resolutionMilli, lowerBucket, upperBucket int64, needDigest bool) ([]storedRollup, error) {
return s.scanRollupRowsBetweenWith(ctx, s.reader(), metricName, entityID, tags, time.Duration(resolutionMilli)*time.Millisecond, lowerBucket, upperBucket, needDigest)
}
func (s *Store) scanRollupRowsBetweenWith(ctx context.Context, q querier, metricName, entityID string, tags map[string]string, resolution time.Duration, lowerBucket, upperBucket int64, needDigest bool) ([]storedRollup, error) {
args := []any{metricName, resolution.Milliseconds(), lowerBucket, upperBucket}
parts := []string{
"s.metric_name = " + s.dialect.placeholder(1),
"d.resolution_milli = " + s.dialect.placeholder(2),
"r.bucket_milli >= " + s.dialect.placeholder(3),
"r.bucket_milli <= " + s.dialect.placeholder(4),
}
if err := query.Validate(); err != nil {
return nil, err
if entityID != "" {
args = append(args, entityID)
parts = append(parts, "s.entity_id = "+s.dialect.placeholder(len(args)))
}
policy, err := s.metricRollupPolicy(ctx, query.MetricName)
for _, key := range sortedKeys(tags) {
args = append(args, tags[key])
parts = append(parts, s.dialect.jsonExtractEquals("s.tags", key, s.dialect.placeholder(len(args))))
}
columns := "s.entity_id, s.tags_hash, s.tags, l.labels_hash, l.labels, r.bucket_milli, r.count, r.sum, r.sum_sq, r.min_val, r.max_val, r.first_val, r.first_ts_milli, r.last_val, r.last_ts_milli"
if needDigest {
columns += ", r.digest"
}
sqlText := fmt.Sprintf("SELECT %s FROM %s r JOIN %s s ON s.id = r.series_id JOIN %s d ON d.id = r.resolution_id JOIN %s l ON l.id = r.label_id WHERE %s ORDER BY r.bucket_milli ASC", columns, s.tables.rollups, s.tables.series, s.tables.resolutions, s.tables.labels, joinSQLWith(parts, " AND "))
rows, err := q.QueryContext(ctx, sqlText, args...)
if err != nil {
return nil, err
}
if len(policy.Tiers) == 0 {
return []AggregatePoint{}, nil
defer rows.Close()
return scanStoredRollupsForMaintenance(rows, needDigest, s.cfg.RollupPolicy.compression())
}
func joinSQLWith(parts []string, separator string) string {
if len(parts) == 0 {
return ""
}
preferred := policy.Tiers[0].Interval
if tier := bestRollupTier(policy, query.Interval, query.Start.UTC(), now.UTC()); tier != nil {
preferred = tier.Interval
} else {
for i := len(policy.Tiers) - 1; i >= 0; i-- {
tier := policy.Tiers[i]
if query.Interval >= tier.Interval && query.Interval%tier.Interval == 0 {
preferred = tier.Interval
break
out := parts[0]
for _, part := range parts[1:] {
out += separator + part
}
return out
}
func scanStoredRollupsForMaintenance(rows *sql.Rows, needDigest bool, compression float64) ([]storedRollup, error) {
result := make([]storedRollup, 0)
for rows.Next() {
var entityID, tagsHash, tagsJSON, labelsHash, labelsJSON string
var bucket, count, firstTS, lastTS int64
var sum, sumSq, min, max, firstVal, lastVal float64
var digest []byte
var err error
if needDigest {
err = rows.Scan(&entityID, &tagsHash, &tagsJSON, &labelsHash, &labelsJSON, &bucket, &count, &sum, &sumSq, &min, &max, &firstVal, &firstTS, &lastVal, &lastTS, &digest)
} else {
err = rows.Scan(&entityID, &tagsHash, &tagsJSON, &labelsHash, &labelsJSON, &bucket, &count, &sum, &sumSq, &min, &max, &firstVal, &firstTS, &lastVal, &lastTS)
}
if err != nil {
return nil, err
}
var decoded *TDigest
if needDigest {
decoded, err = digestFromRollup(count, min, max, digest, compression)
if err != nil {
return nil, err
}
}
result = append(result, storedRollup{entityID: entityID, bucket: bucket, bucketData: &rollupBucket{
count: count, sum: sum, sumSq: sumSq, min: min, max: max,
firstVal: firstVal, firstTS: firstTS, lastVal: lastVal, lastTS: lastTS,
digest: decoded, tagsHash: tagsHash, tagsJSON: tagsJSON, labelsHash: labelsHash, labelsJSON: labelsJSON,
}})
}
resolution, err := s.coveringSeriesResolutionForPolicy(ctx, query.Query.normalized(), preferred, policy)
return result, rows.Err()
}
func representativePoints(metricName string, rows []storedRollup) []Point {
points := make([]Point, 0, len(rows))
for _, row := range rows {
tags, err := rollupTagsFromJSON(row.bucketData.tagsJSON)
if err != nil {
continue
}
labels, err := rollupTagsFromJSON(row.bucketData.labelsJSON)
if err != nil {
continue
}
points = append(points, Point{MetricName: metricName, EntityID: row.entityID, Timestamp: fromMillis(row.bucketData.lastTS), Value: row.bucketData.lastVal, Tags: tags, Labels: labels})
}
return points
}
func rawJSONToString(value any) (string, error) {
switch v := value.(type) {
case nil:
return "{}", nil
case string:
if v == "" {
return "{}", nil
}
return v, nil
case []byte:
if len(v) == 0 {
return "{}", nil
}
return string(v), nil
default:
return "", fmt.Errorf("unsupported JSON column type %T", value)
}
}
func rollupTagsFromJSON(raw string) (map[string]string, error) {
values, err := decodeMapString(raw)
if err != nil {
return nil, err
}
return s.aggregateRollupAt(ctx, query, resolution, now.UTC())
}
// coveringSeriesResolution falls back to a coarser retained tier only when
// the preferred tier starts after the requested range. This happens directly
// after a legacy migration: historical 5m/hour rollups predate the newly
// written 1m tier. Returning the coarser summary preserves the entire query
// rather than presenting a false gap at that boundary.
func (s *Store) coveringSeriesResolution(ctx context.Context, query Query, preferred time.Duration) (time.Duration, error) {
return s.coveringSeriesResolutionForPolicy(ctx, query, preferred, s.cfg.RollupPolicy)
}
func (s *Store) coveringSeriesResolutionForPolicy(ctx context.Context, query Query, preferred time.Duration, policy RollupPolicy) (time.Duration, error) {
selected := preferred
earliest, latest, found, err := s.rollupTimeBounds(ctx, query, preferred)
if err != nil {
return 0, err
}
startMilli := query.Start.UTC().UnixMilli()
endMilli := query.End.UTC().UnixMilli()
if found && earliest <= startMilli && latest >= startMilli {
return selected, nil
}
selectedFound := found && earliest <= endMilli && latest >= startMilli
for _, tier := range policy.Tiers {
if tier.Interval <= preferred {
continue
}
candidateEarliest, candidateLatest, candidateFound, err := s.rollupTimeBounds(ctx, query, tier.Interval)
if err != nil {
return 0, err
}
if !candidateFound || candidateEarliest > endMilli || candidateLatest < startMilli {
continue
}
if candidateEarliest <= startMilli && candidateLatest >= startMilli {
return tier.Interval, nil
}
if !selectedFound || candidateEarliest < earliest {
selected, earliest, selectedFound = tier.Interval, candidateEarliest, true
}
}
return selected, nil
}
func (s *Store) rollupTimeBounds(ctx context.Context, query Query, interval time.Duration) (int64, int64, bool, error) {
args := []any{query.MetricName}
seriesParts := []string{"s.metric_name = " + s.dialect.placeholder(1)}
if query.EntityID != "" {
args = append(args, query.EntityID)
seriesParts = append(seriesParts, "s.entity_id = "+s.dialect.placeholder(len(args)))
}
for _, key := range sortedKeys(query.Tags) {
args = append(args, query.Tags[key])
seriesParts = append(seriesParts, s.dialect.jsonExtractEquals("s.tags", key, s.dialect.placeholder(len(args))))
}
args = append(args, interval.Milliseconds())
resolutionPlaceholder := s.dialect.placeholder(len(args))
args = append(args, query.End.UTC().UnixMilli())
endPlaceholder := s.dialect.placeholder(len(args))
var earliest, latest sql.NullInt64
err := s.reader().QueryRowContext(ctx, fmt.Sprintf(`SELECT MIN(r.first_ts_milli), MAX(r.last_ts_milli) FROM %s r
WHERE r.series_id IN (SELECT s.id FROM %s s WHERE %s)
AND r.resolution_id IN (SELECT d.id FROM %s d WHERE d.resolution_milli = %s)
AND r.first_ts_milli <= %s`, s.tables.rollups, s.tables.series, joinSQLWith(seriesParts, " AND "),
s.tables.resolutions, resolutionPlaceholder, endPlaceholder),
args...).Scan(&earliest, &latest)
if err != nil {
return 0, 0, false, err
}
return earliest.Int64, latest.Int64, earliest.Valid && latest.Valid, nil
return cloneStringMap(values), nil
}
+152
View File
@@ -0,0 +1,152 @@
package metric
import (
"encoding/json"
"fmt"
"strings"
)
type seriesDictionaryPlan struct {
MetricNames []string
EntityIDs []string
Tags map[string]string
}
type rollupReadPlan struct {
SeriesIDs []int64
ResolutionID int64
StartMilli int64
EndMilli int64
Fields rollupReadFields
}
type rollupReadFields uint16
const (
rollupReadSum rollupReadFields = 1 << iota
rollupReadSumSq
rollupReadMin
rollupReadMax
rollupReadFirst
rollupReadLast
rollupReadDigest
)
type renderedSQL struct {
Query string
Args []any
}
func (d sqliteDialect) renderSeriesDictionary(tables tables, indexName string, plan seriesDictionaryPlan) renderedSQL {
return renderExpandedSeriesDictionary(d, tables, " INDEXED BY "+indexName, plan)
}
func (d mysqlDialect) renderSeriesDictionary(tables tables, indexName string, plan seriesDictionaryPlan) renderedSQL {
return renderExpandedSeriesDictionary(d, tables, " FORCE INDEX ("+indexName+")", plan)
}
func (d postgresDialect) renderSeriesDictionary(tables tables, _ string, plan seriesDictionaryPlan) renderedSQL {
args := []any{plan.MetricNames}
parts := []string{"s.metric_name = ANY(" + d.placeholder(1) + "::text[])"}
if len(plan.EntityIDs) > 0 {
args = append(args, plan.EntityIDs)
parts = append(parts, "s.entity_id = ANY("+d.placeholder(len(args))+"::text[])")
}
for _, key := range sortedKeys(plan.Tags) {
args = append(args, plan.Tags[key])
parts = append(parts, d.jsonExtractEquals("s.tags", key, d.placeholder(len(args))))
}
return renderedSQL{
Query: fmt.Sprintf("SELECT s.id, s.metric_name, s.entity_id, s.tags_hash, s.tags FROM %s s WHERE %s ORDER BY s.id ASC", tables.series, strings.Join(parts, " AND ")),
Args: args,
}
}
func renderExpandedSeriesDictionary(d dialect, tables tables, indexHint string, plan seriesDictionaryPlan) renderedSQL {
args := make([]any, 0, len(plan.MetricNames)+len(plan.EntityIDs)+len(plan.Tags))
metricPlaceholders := appendPlaceholders(d, &args, plan.MetricNames)
parts := []string{"s.metric_name IN (" + strings.Join(metricPlaceholders, ", ") + ")"}
if len(plan.EntityIDs) > 0 {
entityPlaceholders := appendPlaceholders(d, &args, plan.EntityIDs)
parts = append(parts, "s.entity_id IN ("+strings.Join(entityPlaceholders, ", ")+")")
}
for _, key := range sortedKeys(plan.Tags) {
args = append(args, plan.Tags[key])
parts = append(parts, d.jsonExtractEquals("s.tags", key, d.placeholder(len(args))))
}
return renderedSQL{
Query: fmt.Sprintf("SELECT s.id, s.metric_name, s.entity_id, s.tags_hash, s.tags FROM %s s%s WHERE %s ORDER BY s.id ASC", tables.series, indexHint, strings.Join(parts, " AND ")),
Args: args,
}
}
func appendPlaceholders[T ~string](d dialect, args *[]any, values []T) []string {
placeholders := make([]string, 0, len(values))
for _, value := range values {
*args = append(*args, string(value))
placeholders = append(placeholders, d.placeholder(len(*args)))
}
return placeholders
}
func (d sqliteDialect) renderRollupRead(tables tables, indexName string, plan rollupReadPlan) renderedSQL {
seriesJSON, _ := json.Marshal(plan.SeriesIDs)
args := []any{plan.ResolutionID, plan.StartMilli, plan.EndMilli, string(seriesJSON)}
return renderedSQL{
Query: fmt.Sprintf("SELECT %s FROM %s r INDEXED BY %s WHERE r.resolution_id = %s AND r.bucket_milli >= %s AND r.bucket_milli <= %s AND r.series_id IN (SELECT CAST(value AS INTEGER) FROM json_each(%s)) ORDER BY r.bucket_milli ASC, r.series_id ASC, r.label_id ASC",
rollupReadColumns(plan.Fields), tables.rollups, indexName,
d.placeholder(1), d.placeholder(2), d.placeholder(3), d.placeholder(4)),
Args: args,
}
}
func (d mysqlDialect) renderRollupRead(tables tables, indexName string, plan rollupReadPlan) renderedSQL {
args := []any{plan.ResolutionID, plan.StartMilli, plan.EndMilli}
seriesPlaceholders := make([]string, 0, len(plan.SeriesIDs))
for _, seriesID := range plan.SeriesIDs {
args = append(args, seriesID)
seriesPlaceholders = append(seriesPlaceholders, d.placeholder(len(args)))
}
return renderedSQL{
Query: fmt.Sprintf("SELECT %s FROM %s r FORCE INDEX (%s) WHERE r.resolution_id = %s AND r.bucket_milli >= %s AND r.bucket_milli <= %s AND r.series_id IN (%s) ORDER BY r.bucket_milli ASC, r.series_id ASC, r.label_id ASC",
rollupReadColumns(plan.Fields), tables.rollups, indexName,
d.placeholder(1), d.placeholder(2), d.placeholder(3), strings.Join(seriesPlaceholders, ", ")),
Args: args,
}
}
func (d postgresDialect) renderRollupRead(tables tables, _ string, plan rollupReadPlan) renderedSQL {
args := []any{plan.ResolutionID, plan.StartMilli, plan.EndMilli, plan.SeriesIDs}
return renderedSQL{
Query: fmt.Sprintf("SELECT %s FROM %s r WHERE r.resolution_id = %s AND r.bucket_milli >= %s AND r.bucket_milli <= %s AND r.series_id = ANY(%s::bigint[]) ORDER BY r.bucket_milli ASC, r.series_id ASC, r.label_id ASC",
rollupReadColumns(plan.Fields), tables.rollups,
d.placeholder(1), d.placeholder(2), d.placeholder(3), d.placeholder(4)),
Args: args,
}
}
func rollupReadColumns(fields rollupReadFields) string {
columns := []string{"r.series_id", "r.bucket_milli", "r.count"}
if fields&rollupReadSum != 0 {
columns = append(columns, "r.sum")
}
if fields&rollupReadSumSq != 0 {
columns = append(columns, "r.sum_sq")
}
if fields&rollupReadMin != 0 {
columns = append(columns, "r.min_val")
}
if fields&rollupReadMax != 0 {
columns = append(columns, "r.max_val")
}
if fields&rollupReadFirst != 0 {
columns = append(columns, "r.first_val", "r.first_ts_milli")
}
if fields&rollupReadLast != 0 {
columns = append(columns, "r.last_val", "r.last_ts_milli")
}
if fields&rollupReadDigest != 0 {
columns = append(columns, "r.digest")
}
return strings.Join(columns, ", ")
}
+1 -1
View File
@@ -146,7 +146,7 @@ func (s *Store) rebuildRollupBucketTx(ctx context.Context, metricName string, fi
if err != nil {
return nil, err
}
stored, err := scanStoredRollups(rows, true, s.cfg.RollupPolicy.compression())
stored, err := scanStoredRollupsForMaintenance(rows, true, s.cfg.RollupPolicy.compression())
_ = rows.Close()
if err != nil {
return nil, err
+2 -2
View File
@@ -1104,8 +1104,8 @@ func TestCompactWithRawRetentionOnlyWritesChangedBuckets(t *testing.T) {
if err != nil {
t.Fatalf("rollup: %v", err)
}
if len(got) != 1 || got[0].Count != 2 || got[0].Value != 3 {
t.Fatalf("late delta was not merged exactly once: %#v", got)
if len(got) != 1 || got[0].Count != 1 || got[0].Value != 1 {
t.Fatalf("explicit 5m query read a finer tier: %#v", got)
}
}
+36 -5
View File
@@ -74,7 +74,7 @@ type Store struct {
// coarseMu protects in-memory parent buckets. Coarser tiers are only
// materialized once their source window has remained closed long enough for
// the raw late-arrival window to pass.
coarseMu sync.Mutex
coarseMu sync.RWMutex
coarse map[coarseRollupKey]*coarseRollup
// mu protects closed state.
//
@@ -247,16 +247,19 @@ func prepareSQLiteConfig(cfg Config) (Config, error) {
cfg.SQLite.BusyTimeout = 5 * time.Second
}
if cfg.SQLite.CacheSizeKB == 0 {
cfg.SQLite.CacheSizeKB = 16 * 1024
cfg.SQLite.CacheSizeKB = 4 * 1024
}
if cfg.SQLite.MMapSizeBytes == 0 {
cfg.SQLite.MMapSizeBytes = 32 * 1024 * 1024
// MMapSizeBytes == 0 表示显式禁用 mmap,而不是"未设置":DefaultConfig 始终
// 填充 SQLiteOptions,因此这里的 0 是"跳过 mmap"的明确选择,不能被覆盖为
// 32MB。负数视为非法输入,归一为 0(禁用)。
if cfg.SQLite.MMapSizeBytes < 0 {
cfg.SQLite.MMapSizeBytes = 0
}
if cfg.SQLite.WALAutoCheckpoint == 0 {
cfg.SQLite.WALAutoCheckpoint = 4000
}
if cfg.SQLite.JournalSizeLimitBytes == 0 {
cfg.SQLite.JournalSizeLimitBytes = 16 * 1024 * 1024
cfg.SQLite.JournalSizeLimitBytes = 4 * 1024 * 1024
}
if cfg.DB == nil {
@@ -410,6 +413,22 @@ func (s *Store) Ping(ctx context.Context) error {
return s.db.PingContext(ctx)
}
// QueryContext executes a raw read query against the Store's read pool.
func (s *Store) QueryContext(ctx context.Context, query string, args ...any) (*sql.Rows, error) {
if err := s.ensureOpen(); err != nil {
return nil, err
}
return s.reader().QueryContext(ctx, query, args...)
}
// ExecContext executes a raw statement against the Store's primary write pool.
func (s *Store) ExecContext(ctx context.Context, query string, args ...any) (sql.Result, error) {
if err := s.ensureOpen(); err != nil {
return nil, err
}
return s.db.ExecContext(ctx, query, args...)
}
// ensureOpen verifies that the Store is not closed.
//
// ensureOpen 检查 Store 是否仍处于打开状态。
@@ -931,6 +950,18 @@ func (s *Store) Query(ctx context.Context, query Query) ([]Point, error) {
return s.queryRawPoints(ctx, query)
}
// QueryBatch loads exact raw samples for multiple metrics with one scan of the
// in-memory raw store.
func (s *Store) QueryBatch(ctx context.Context, query BatchQuery) (map[string][]Point, error) {
if err := s.ensureOpen(); err != nil {
return nil, err
}
if err := query.Validate(); err != nil {
return nil, err
}
return s.queryRawPointsBatch(ctx, query.normalized())
}
// EntityIDs returns distinct entity ids that have exact in-memory samples or
// persisted/hot rollups.
func (s *Store) EntityIDs(ctx context.Context, query Query) ([]string, error) {
+2 -2
View File
@@ -213,8 +213,8 @@ func TestSQLiteInDirCreatesDirectoryAndAppliesPragmas(t *testing.T) {
if err := store.db.QueryRowContext(ctx, "PRAGMA journal_size_limit").Scan(&journalSizeLimit); err != nil {
t.Fatalf("query journal size limit: %v", err)
}
if journalSizeLimit != 16*1024*1024 {
t.Fatalf("journal size limit = %d, want %d", journalSizeLimit, 16*1024*1024)
if journalSizeLimit != 4*1024*1024 {
t.Fatalf("journal size limit = %d, want %d", journalSizeLimit, 4*1024*1024)
}
}
+10 -7
View File
@@ -139,11 +139,15 @@ func (t *TDigest) Add(x, w float64) {
t.processed = false
// Bound the unprocessed buffer so a long stream cannot grow memory without
// limit; process() collapses it back to ~compression centroids.
if len(t.centroids) > int(8*t.compression)+16 {
if len(t.centroids) > t.processThreshold() {
t.process()
}
}
func (t *TDigest) processThreshold() int {
return int(8*t.compression) + 16
}
// Merge folds every centroid of other into t. This is the operation rollup
// composition relies on: a coarse bucket merges the digests of the finer
// buckets it spans.
@@ -154,10 +158,7 @@ func (t *TDigest) Merge(other *TDigest) {
if other == nil || other.count == 0 {
return
}
other.process()
for _, c := range other.centroids {
t.centroids = append(t.centroids, c)
}
t.centroids = append(t.centroids, other.centroids...)
t.count += other.count
if other.min < t.min {
t.min = other.min
@@ -166,7 +167,9 @@ func (t *TDigest) Merge(other *TDigest) {
t.max = other.max
}
t.processed = false
t.process()
if len(t.centroids) > t.processThreshold() {
t.process()
}
}
// Count returns the total weight observed.
@@ -194,7 +197,7 @@ func (t *TDigest) process() {
return t.centroids[i].mean < t.centroids[j].mean
})
total := t.count
merged := t.centroids[:0:0] // fresh backing array; don't alias input mid-merge
merged := t.centroids[:0]
cur := t.centroids[0]
weightBefore := 0.0
for i := 1; i < len(t.centroids); i++ {
+89
View File
@@ -73,6 +73,95 @@ func TestTDigestMergeMatchesCombined(t *testing.T) {
}
}
func TestTDigestMergeGroupingAccuracy(t *testing.T) {
rng := rand.New(rand.NewSource(17))
all := make([]float64, 0, 24*2000)
fine := make([]*TDigest, 24)
for bucket := range fine {
fine[bucket] = NewTDigest(100)
for i := 0; i < 2000; i++ {
value := math.Abs(rng.NormFloat64())*70 + rng.ExpFloat64()*30
fine[bucket].Add(value, 1)
all = append(all, value)
}
}
flat := NewTDigest(100)
grouped := NewTDigest(100)
for _, digest := range fine {
flat.Merge(digest)
}
for groupStart := 0; groupStart < len(fine); groupStart += 4 {
group := NewTDigest(100)
for _, digest := range fine[groupStart : groupStart+4] {
group.Merge(digest)
}
grouped.Merge(group)
}
for name, digest := range map[string]*TDigest{"flat": flat, "grouped": grouped} {
if digest.Count() != float64(len(all)) || digest.min != exactQuantile(all, 0) || digest.max != exactQuantile(all, 1) {
t.Fatalf("%s exact aggregates changed: count=%v min=%v max=%v", name, digest.Count(), digest.min, digest.max)
}
for _, q := range []float64{0.5, 0.95, 0.99, 0.999} {
exact := exactQuantile(all, q)
relErr := math.Abs(digest.Quantile(q)-exact) / math.Abs(exact)
if relErr > 0.02 {
t.Fatalf("%s q=%v relative error %v exceeds 2%%", name, q, relErr)
}
}
}
}
func TestTDigestMergeDefersProcessingUntilThreshold(t *testing.T) {
source := NewTDigest(30)
source.Add(42, 1)
if source.processed {
t.Fatal("source should have buffered centroid before merge")
}
merged := NewTDigest(30)
for i := 0; i <= merged.processThreshold(); i++ {
merged.Merge(source)
}
if !merged.processed {
t.Fatal("merge did not process after exceeding threshold")
}
if source.processed {
t.Fatal("merge processed the source digest")
}
merged.Merge(source)
if merged.processed {
t.Fatal("merge below threshold should remain buffered")
}
if merged.Count() != float64(merged.processThreshold()+2) {
t.Fatalf("count = %v", merged.Count())
}
if merged.processed {
t.Fatal("Count processed buffered centroids")
}
if value := merged.Quantile(0.5); value != 42 || !merged.processed {
t.Fatalf("Quantile did not finalize digest: value=%v processed=%v", value, merged.processed)
}
merged.Merge(source)
if merged.processed {
t.Fatal("merge below threshold should remain buffered before Encode")
}
encoded := merged.Encode()
if len(encoded) == 0 || !merged.processed {
t.Fatal("Encode did not finalize digest")
}
decoded, err := DecodeTDigest(encoded)
if err != nil {
t.Fatal(err)
}
if decoded.Count() != merged.Count() || decoded.min != merged.min || decoded.max != merged.max {
t.Fatalf("round trip changed exact aggregates: %#v vs %#v", decoded, merged)
}
}
// TestTDigestEncodeRoundTrip verifies t-digest serialization.
//
// TestTDigestEncodeRoundTrip 验证 t-digest 编码和解码往返。
+107
View File
@@ -291,6 +291,43 @@ type Query struct {
Order Order `json:"order,omitempty"`
}
// BatchQuery loads exact raw samples for multiple metrics and entities from one
// shared time window.
type BatchQuery struct {
MetricNames []string `json:"metric_names"`
EntityIDs []string `json:"entity_ids,omitempty"`
Start time.Time `json:"start"`
End time.Time `json:"end"`
Tags map[string]string `json:"tags,omitempty"`
Order Order `json:"order,omitempty"`
}
// BatchSeriesSpec describes the rollup output requested for one metric.
type BatchSeriesSpec struct {
MetricName string
Aggregations []Aggregation
Interval time.Duration
PreserveSeries bool
}
// BatchSeriesQuery combines per-metric rollup specifications that share the
// same time, entity, tag, and ordering filters.
type BatchSeriesQuery struct {
Specs []BatchSeriesSpec
EntityIDs []string
Start time.Time
End time.Time
Tags map[string]string
Order Order
}
// BatchSeriesResult contains definitions and aggregate values keyed by metric
// and aggregation.
type BatchSeriesResult struct {
Definitions map[string]Definition
Values map[string]map[Aggregation][]AggregatePoint
}
// Validate checks whether the value is well formed.
//
// Validate 检查原始查询条件是否合法。
@@ -327,6 +364,76 @@ func (q Query) normalized() Query {
return q
}
// Validate checks whether a batch raw query is well formed.
func (q BatchQuery) Validate() error {
if len(q.MetricNames) == 0 {
return fmt.Errorf("%w: at least one metric name is required", ErrInvalidArgument)
}
for _, name := range q.MetricNames {
if strings.TrimSpace(name) == "" {
return fmt.Errorf("%w: metric name is required", ErrInvalidArgument)
}
}
if q.Start.IsZero() || q.End.IsZero() {
return fmt.Errorf("%w: start and end time are required", ErrInvalidArgument)
}
if q.End.Before(q.Start) {
return fmt.Errorf("%w: end time cannot be before start time", ErrInvalidArgument)
}
switch q.Order {
case "", OrderAsc, OrderDesc:
default:
return fmt.Errorf("%w: unsupported order %q", ErrInvalidArgument, q.Order)
}
return nil
}
func (q BatchQuery) normalized() BatchQuery {
q.Start = q.Start.UTC()
q.End = q.End.UTC()
if q.Order == "" {
q.Order = OrderAsc
}
return q
}
// Validate checks whether a batch rollup query is well formed.
func (q BatchSeriesQuery) Validate() error {
if len(q.Specs) == 0 {
return fmt.Errorf("%w: at least one series specification is required", ErrInvalidArgument)
}
seen := make(map[string]struct{}, len(q.Specs))
for _, spec := range q.Specs {
if _, ok := seen[spec.MetricName]; ok {
return fmt.Errorf("%w: duplicate series specification for metric %q", ErrInvalidArgument, spec.MetricName)
}
seen[spec.MetricName] = struct{}{}
if len(spec.Aggregations) == 0 {
return fmt.Errorf("%w: at least one aggregation is required for metric %q", ErrInvalidArgument, spec.MetricName)
}
for _, aggregation := range spec.Aggregations {
if err := (AggregateQuery{
Query: Query{MetricName: spec.MetricName, Start: q.Start, End: q.End, Tags: q.Tags, Order: q.Order},
Aggregation: aggregation,
Interval: spec.Interval,
PreserveSeries: spec.PreserveSeries,
}).Validate(); err != nil {
return err
}
}
}
return nil
}
func (q BatchSeriesQuery) normalized() BatchSeriesQuery {
q.Start = q.Start.UTC()
q.End = q.End.UTC()
if q.Order == "" {
q.Order = OrderAsc
}
return q
}
// AggregateQuery describes a bucketed aggregate query.
//
// AggregateQuery 描述按固定时间间隔分桶后的聚合查询。
+1 -1
View File
@@ -130,7 +130,7 @@ func sendTrafficReport(daily, weekly, monthly bool) {
return
}
message := strings.Join(lines, "\n")
message := strings.Join(lines, "\nKomari 内置的流量报告即将在 1.5.0 版本移除,请尽快迁移至插件“流量定期报告”。\nKomari's built-in traffic reports will be removed in version 1.5.0. Please migrate to the Scheduled Traffic Reports plugin as soon as possible.")
var emoji string
switch {
case daily:
+70
View File
@@ -0,0 +1,70 @@
package admin
import (
"fmt"
"os"
"path/filepath"
"time"
"github.com/komari-monitor/komari/internal/plugin"
logger "github.com/komari-monitor/komari/utils/log"
"github.com/komari-monitor/komari/web/backup"
"github.com/komari-monitor/komari/web/upload"
)
func NewArchiveUploadHandler() *upload.Handler {
return upload.NewHandler(upload.DefaultStore, map[upload.Purpose]upload.Finalizer{
upload.PurposeBackup: finalizeBackupUpload,
upload.PurposePlugin: finalizePluginUpload,
upload.PurposeTheme: finalizeThemeUpload,
})
}
func finalizeBackupUpload(session upload.Session) (upload.Result, error) {
restoreLock, err := backup.AcquireRestoreLock()
if err != nil {
return upload.Result{}, err
}
archive, err := os.Open(session.ArchivePath)
if err != nil {
restoreLock.Release()
return upload.Result{}, fmt.Errorf("open merged backup: %w", err)
}
if err := restoreLock.SaveUploadedBackup(archive, session.Metadata.Filename); err != nil {
_ = archive.Close()
restoreLock.Release()
return upload.Result{}, err
}
if err := archive.Close(); err != nil {
restoreLock.Release()
return upload.Result{}, fmt.Errorf("close merged backup: %w", err)
}
go func() {
logger.InfoArgs("admin-api", "Backup uploaded, restarting service in 2 seconds to apply on startup...")
time.Sleep(2 * time.Second)
restoreLock.Release()
os.Exit(0)
}()
return upload.Result{
Message: "Backup uploaded successfully. The service will restart and apply the backup.",
Data: map[string]string{"path": filepath.Join(".", "data", "backup.zip")},
}, nil
}
func finalizePluginUpload(session upload.Session) (upload.Result, error) {
info, err := plugin.InstallZip(session.ArchivePath)
if err != nil {
return upload.Result{}, err
}
return upload.Result{Message: "插件上传成功", Data: info}, nil
}
func finalizeThemeUpload(session upload.Session) (upload.Result, error) {
info, err := extractAndValidateTheme(session.ArchivePath)
if err != nil {
return upload.Result{}, err
}
return upload.Result{Message: "主题上传成功", Data: info}, nil
}
-111
View File
@@ -1,111 +0,0 @@
package admin
import (
"encoding/json"
"os"
"path/filepath"
"testing"
)
func TestIsValidUploadID(t *testing.T) {
for _, uploadID := range []string{chunkUploadID, "../backup", "", "another-upload"} {
want := uploadID == chunkUploadID
if got := isValidUploadID(uploadID); got != want {
t.Errorf("isValidUploadID(%q) = %v, want %v", uploadID, got, want)
}
}
}
func TestChunkUploadDirRejectsTraversal(t *testing.T) {
if _, err := chunkUploadDir(".."); err == nil {
t.Fatal("chunkUploadDir accepted traversal upload_id")
}
}
func TestClearChunkUploadCache(t *testing.T) {
root := t.TempDir()
if err := os.WriteFile(filepath.Join(root, "interrupted.part"), []byte("stale"), 0600); err != nil {
t.Fatalf("write cache: %v", err)
}
if err := clearChunkUploadCache(root); err != nil {
t.Fatalf("clearChunkUploadCache: %v", err)
}
entries, err := os.ReadDir(root)
if err != nil {
t.Fatalf("read cache: %v", err)
}
if len(entries) != 0 {
t.Fatalf("cache entries = %d, want 0", len(entries))
}
}
func TestCopyWhitelistedFilesIncludesFont(t *testing.T) {
dataDir := t.TempDir()
backupDir := t.TempDir()
fontPath := filepath.Join(dataDir, "font.ttf")
if err := os.WriteFile(fontPath, []byte("font-data"), 0644); err != nil {
t.Fatalf("write font: %v", err)
}
if err := copyWhitelistedFilesFrom(dataDir, backupDir); err != nil {
t.Fatalf("copyWhitelistedFilesFrom: %v", err)
}
content, err := os.ReadFile(filepath.Join(backupDir, "font.ttf"))
if err != nil {
t.Fatalf("read copied font: %v", err)
}
if string(content) != "font-data" {
t.Fatalf("font content = %q, want %q", content, "font-data")
}
}
func TestChunkUploadMetadataAndSize(t *testing.T) {
dir := t.TempDir()
metadata, err := json.Marshal(chunkUploadMetadata{Size: 7})
if err != nil {
t.Fatalf("marshal metadata: %v", err)
}
if err := os.WriteFile(filepath.Join(dir, uploadMetadataName), metadata, 0600); err != nil {
t.Fatalf("write metadata: %v", err)
}
for name, content := range map[string]string{"0.part": "abc", "1.part": "defg", ".1-staging.part": "staging", "ignored.txt": "ignored"} {
if err := os.WriteFile(filepath.Join(dir, name), []byte(content), 0600); err != nil {
t.Fatalf("write %s: %v", name, err)
}
}
gotMetadata, err := readChunkUploadMetadata(dir)
if err != nil {
t.Fatalf("read metadata: %v", err)
}
if gotMetadata.Size != 7 {
t.Fatalf("metadata size = %d, want 7", gotMetadata.Size)
}
gotSize, err := chunkUploadSize(dir)
if err != nil {
t.Fatalf("chunkUploadSize: %v", err)
}
if gotSize != 7 {
t.Fatalf("chunk upload size = %d, want 7", gotSize)
}
}
func TestMergeChunksIgnoresTemporaryParts(t *testing.T) {
dir := t.TempDir()
for name, content := range map[string]string{"0.part": "first", "1.part": "second", ".1-staging.part": "temporary"} {
if err := os.WriteFile(filepath.Join(dir, name), []byte(content), 0600); err != nil {
t.Fatalf("write %s: %v", name, err)
}
}
destination := filepath.Join(dir, "merged.zip")
if err := mergeChunks(dir, destination); err != nil {
t.Fatalf("mergeChunks: %v", err)
}
content, err := os.ReadFile(destination)
if err != nil {
t.Fatalf("read merged chunks: %v", err)
}
if string(content) != "firstsecond" {
t.Fatalf("merged chunks = %q, want %q", content, "firstsecond")
}
}
+2
View File
@@ -15,6 +15,8 @@ var backupWhitelist = []string{
"favicon.ico",
"font.ttf",
"theme/",
"plugin/",
"plguin-data/",
"metrics.db",
}
-381
View File
@@ -1,381 +0,0 @@
package admin
import (
"encoding/json"
"fmt"
"io"
"log"
"net/http"
"os"
"path/filepath"
"sort"
"strconv"
"strings"
"time"
"github.com/gin-gonic/gin"
"github.com/komari-monitor/komari/web/api"
"github.com/komari-monitor/komari/web/backup"
)
const (
defaultChunkSize = 5 * 1024 * 1024 // 5 MiB
maxChunkRequestSize = defaultChunkSize + 1*1024*1024
chunkUploadID = "backup"
maxChunkIndex = int(backup.MaxArchiveSize / defaultChunkSize)
)
var chunkUploadRoot = filepath.Join(".", "data", "backup", ".uploading")
const uploadMetadataName = "upload.json"
type chunkUploadMetadata struct {
Size int64 `json:"size"`
}
func isValidUploadID(uploadID string) bool {
return uploadID == chunkUploadID
}
func chunkUploadDir(uploadID string) (string, error) {
if !isValidUploadID(uploadID) {
return "", fmt.Errorf("invalid upload_id")
}
return filepath.Join(chunkUploadRoot, uploadID), nil
}
func readChunkUploadMetadata(chunkDir string) (chunkUploadMetadata, error) {
var metadata chunkUploadMetadata
data, err := os.ReadFile(filepath.Join(chunkDir, uploadMetadataName))
if err != nil {
return metadata, err
}
if err := json.Unmarshal(data, &metadata); err != nil {
return metadata, err
}
if metadata.Size <= 0 || metadata.Size > backup.MaxArchiveSize {
return metadata, fmt.Errorf("invalid upload size")
}
return metadata, nil
}
func chunkPartIndex(name string) (int, bool) {
index, err := strconv.Atoi(strings.TrimSuffix(name, ".part"))
return index, err == nil && index >= 0 && strings.HasSuffix(name, ".part")
}
func chunkUploadSize(chunkDir string) (int64, error) {
entries, err := os.ReadDir(chunkDir)
if err != nil {
return 0, err
}
var total int64
for _, entry := range entries {
if entry.IsDir() {
continue
}
if _, ok := chunkPartIndex(entry.Name()); !ok {
continue
}
info, err := entry.Info()
if err != nil {
return 0, err
}
if total > backup.MaxArchiveSize-info.Size() {
return 0, fmt.Errorf("uploaded chunks exceed the archive limit")
}
total += info.Size()
}
return total, nil
}
func clearChunkUploadCache(root string) error {
if err := os.RemoveAll(root); err != nil {
return err
}
return os.MkdirAll(root, 0755)
}
// InitChunkUpload 初始化分块上传,返回 upload_id 和 chunk_size。
// Komari 使用单一上传槽位,初始化时会清除上次中断留下的全部缓存。
func InitChunkUpload(c *gin.Context) {
var req struct {
Size int64 `json:"size"`
}
if err := c.ShouldBindJSON(&req); err != nil || req.Size <= 0 || req.Size > backup.MaxArchiveSize {
api.RespondError(c, http.StatusBadRequest, fmt.Sprintf("size must be between 1 and %d bytes", backup.MaxArchiveSize))
return
}
if err := clearChunkUploadCache(chunkUploadRoot); err != nil {
api.RespondError(c, http.StatusInternalServerError, fmt.Sprintf("Error clearing upload cache: %v", err))
return
}
chunkDir, err := chunkUploadDir(chunkUploadID)
if err != nil {
api.RespondError(c, http.StatusInternalServerError, "Error preparing upload directory")
return
}
if err := os.MkdirAll(chunkDir, 0755); err != nil {
api.RespondError(c, http.StatusInternalServerError, fmt.Sprintf("Error creating upload directory: %v", err))
return
}
metadata, err := json.Marshal(chunkUploadMetadata{Size: req.Size})
if err != nil {
clearChunkUploadCache(chunkUploadRoot)
api.RespondError(c, http.StatusInternalServerError, fmt.Sprintf("Error encoding upload metadata: %v", err))
return
}
if err := os.WriteFile(filepath.Join(chunkDir, uploadMetadataName), metadata, 0600); err != nil {
clearChunkUploadCache(chunkUploadRoot)
api.RespondError(c, http.StatusInternalServerError, fmt.Sprintf("Error writing upload metadata: %v", err))
return
}
c.JSON(http.StatusOK, gin.H{
"upload_id": chunkUploadID,
"chunk_size": defaultChunkSize,
})
}
// UploadChunk 接收单个分块,保存到临时目录。
func UploadChunk(c *gin.Context) {
defer func() {
if c.Request.Context().Err() != nil {
_ = clearChunkUploadCache(chunkUploadRoot)
}
}()
c.Request.Body = http.MaxBytesReader(c.Writer, c.Request.Body, maxChunkRequestSize)
uploadID := c.PostForm("upload_id")
chunkDir, err := chunkUploadDir(uploadID)
if err != nil {
api.RespondError(c, http.StatusBadRequest, "invalid upload_id")
return
}
chunkIndexStr := c.PostForm("chunk_index")
if chunkIndexStr == "" {
api.RespondError(c, http.StatusBadRequest, "chunk_index is required")
return
}
chunkIndex, err := strconv.Atoi(chunkIndexStr)
if err != nil || chunkIndex < 0 || chunkIndex > maxChunkIndex {
api.RespondError(c, http.StatusBadRequest, "chunk_index must be a non-negative integer")
return
}
if info, err := os.Stat(chunkDir); err != nil || !info.IsDir() {
api.RespondError(c, http.StatusNotFound, "upload_id not found or expired")
return
}
file, _, err := c.Request.FormFile("chunk_data")
if err != nil {
api.RespondError(c, http.StatusBadRequest, fmt.Sprintf("Error getting chunk data: %v", err))
return
}
defer file.Close()
metadata, err := readChunkUploadMetadata(chunkDir)
if err != nil {
api.RespondError(c, http.StatusNotFound, "upload_id not found or expired")
return
}
chunkPath := filepath.Join(chunkDir, fmt.Sprintf("%d.part", chunkIndex))
tempChunk, err := os.CreateTemp(chunkDir, fmt.Sprintf(".%d-*.part", chunkIndex))
if err != nil {
api.RespondError(c, http.StatusInternalServerError, fmt.Sprintf("Error saving chunk: %v", err))
return
}
tempChunkPath := tempChunk.Name()
defer os.Remove(tempChunkPath)
written, err := io.Copy(tempChunk, io.LimitReader(file, defaultChunkSize+1))
if closeErr := tempChunk.Close(); err == nil {
err = closeErr
}
if err != nil || written > defaultChunkSize {
if err != nil {
api.RespondError(c, http.StatusInternalServerError, fmt.Sprintf("Error writing chunk: %v", err))
} else {
api.RespondError(c, http.StatusBadRequest, fmt.Sprintf("Chunk too large: %d bytes (max %d)", written, defaultChunkSize))
}
return
}
currentSize, err := chunkUploadSize(chunkDir)
if err != nil {
api.RespondError(c, http.StatusInternalServerError, fmt.Sprintf("Error checking uploaded chunks: %v", err))
return
}
oldSize := int64(0)
if info, err := os.Stat(chunkPath); err == nil {
oldSize = info.Size()
} else if !os.IsNotExist(err) {
api.RespondError(c, http.StatusInternalServerError, fmt.Sprintf("Error checking existing chunk: %v", err))
return
}
if currentSize-oldSize+written > metadata.Size {
api.RespondError(c, http.StatusBadRequest, "uploaded chunks exceed the declared backup size")
return
}
if err := os.Remove(chunkPath); err != nil && !os.IsNotExist(err) {
api.RespondError(c, http.StatusInternalServerError, fmt.Sprintf("Error replacing chunk: %v", err))
return
}
if err := os.Rename(tempChunkPath, chunkPath); err != nil {
api.RespondError(c, http.StatusInternalServerError, fmt.Sprintf("Error publishing chunk: %v", err))
return
}
c.JSON(http.StatusOK, gin.H{
"received": true,
"chunk_index": chunkIndex,
})
}
// MergeChunkUpload 合并分块到 backup.zip 后触发恢复。
func MergeChunkUpload(c *gin.Context) {
restoreLock, err := backup.AcquireRestoreLock()
if err != nil {
api.RespondError(c, http.StatusConflict, err.Error())
return
}
keepRestoreLock := false
defer func() {
if !keepRestoreLock {
restoreLock.Release()
}
}()
var req struct {
UploadID string `json:"upload_id" binding:"required"`
}
if err := c.ShouldBindJSON(&req); err != nil {
api.RespondError(c, http.StatusBadRequest, fmt.Sprintf("Invalid request: %v", err))
return
}
chunkDir, err := chunkUploadDir(req.UploadID)
if err != nil {
api.RespondError(c, http.StatusBadRequest, "invalid upload_id")
return
}
if info, err := os.Stat(chunkDir); err != nil || !info.IsDir() {
api.RespondError(c, http.StatusNotFound, "upload_id not found or expired")
return
}
metadata, err := readChunkUploadMetadata(chunkDir)
if err != nil {
api.RespondError(c, http.StatusNotFound, "upload_id not found or expired")
return
}
if uploadedSize, err := chunkUploadSize(chunkDir); err != nil || uploadedSize != metadata.Size {
_ = clearChunkUploadCache(chunkUploadRoot)
if err != nil {
api.RespondError(c, http.StatusInternalServerError, fmt.Sprintf("Error checking uploaded chunks: %v", err))
} else {
api.RespondError(c, http.StatusBadRequest, "uploaded chunk size does not match the declared backup size")
}
return
}
mergedPath := filepath.Join(chunkDir, "merged.zip")
if err := mergeChunks(chunkDir, mergedPath); err != nil {
clearChunkUploadCache(chunkUploadRoot)
api.RespondError(c, http.StatusInternalServerError, fmt.Sprintf("Error merging chunks: %v", err))
return
}
if err := validateBackupZip(mergedPath); err != nil {
clearChunkUploadCache(chunkUploadRoot)
api.RespondError(c, http.StatusBadRequest, fmt.Sprintf("Invalid backup: %v", err))
return
}
archive, err := os.Open(mergedPath)
if err != nil {
clearChunkUploadCache(chunkUploadRoot)
api.RespondError(c, http.StatusInternalServerError, fmt.Sprintf("Error opening merged backup: %v", err))
return
}
if err := restoreLock.SaveUploadedBackup(archive, "backup.zip"); err != nil {
archive.Close()
clearChunkUploadCache(chunkUploadRoot)
api.RespondError(c, http.StatusBadRequest, fmt.Sprintf("Error preparing restore: %v", err))
return
}
if err := archive.Close(); err != nil {
clearChunkUploadCache(chunkUploadRoot)
api.RespondError(c, http.StatusInternalServerError, fmt.Sprintf("Error closing merged backup: %v", err))
return
}
clearChunkUploadCache(chunkUploadRoot)
c.JSON(http.StatusOK, gin.H{
"status": "success",
"message": "Backup uploaded successfully. The service will restart and apply the backup.",
"path": filepath.Join(".", "data", "backup.zip"),
})
keepRestoreLock = true
go func() {
log.Println("Backup uploaded (chunk), restarting service in 2 seconds to apply on startup...")
time.Sleep(2 * time.Second)
restoreLock.Release()
os.Exit(0)
}()
}
// mergeChunks 按分块索引数值排序后顺序合并所有 .part 文件。
func mergeChunks(chunkDir, destPath string) error {
parts, err := filepath.Glob(filepath.Join(chunkDir, "*.part"))
if err != nil {
return err
}
numericParts := parts[:0]
for _, partPath := range parts {
if _, ok := chunkPartIndex(filepath.Base(partPath)); ok {
numericParts = append(numericParts, partPath)
}
}
parts = numericParts
if len(parts) == 0 {
return fmt.Errorf("no chunks found")
}
// 按分块索引数值排序(非字典序),避免 10.part 排在 2.part 前面
sort.Slice(parts, func(i, j int) bool {
idxI, _ := chunkPartIndex(filepath.Base(parts[i]))
idxJ, _ := chunkPartIndex(filepath.Base(parts[j]))
return idxI < idxJ
})
out, err := os.Create(destPath)
if err != nil {
return err
}
defer out.Close()
for _, partPath := range parts {
f, err := os.Open(partPath)
if err != nil {
return fmt.Errorf("error opening chunk %s: %v", filepath.Base(partPath), err)
}
if _, err := io.Copy(out, f); err != nil {
f.Close()
return fmt.Errorf("error reading chunk %s: %v", filepath.Base(partPath), err)
}
f.Close()
}
return nil
}
// validateBackupZip 校验 ZIP 结构完整性及 komari-backup-markup 标记文件。
func validateBackupZip(zipPath string) error {
if err := backup.ValidateArchive(zipPath); err != nil {
return fmt.Errorf("not a valid backup archive: %w", err)
}
return nil
}
-37
View File
@@ -1,50 +1,13 @@
package admin
import (
"io"
"net/http"
"os"
"strings"
"github.com/gin-gonic/gin"
"github.com/komari-monitor/komari/internal/plugin"
"github.com/komari-monitor/komari/web/api"
)
// UploadPlugin installs a plugin ZIP package. The request body is the raw
// ZIP, mirroring UploadTheme.
func UploadPlugin(c *gin.Context) {
data, err := io.ReadAll(c.Request.Body)
if err != nil || len(data) == 0 {
api.RespondError(c, http.StatusBadRequest, "请选择要上传的插件文件")
return
}
tempFile, err := os.CreateTemp("", "komari-plugin-*.zip")
if err != nil {
api.RespondError(c, http.StatusInternalServerError, "保存文件失败: "+err.Error())
return
}
tempPath := tempFile.Name()
defer func() { _ = os.Remove(tempPath) }()
if _, err := tempFile.Write(data); err != nil {
_ = tempFile.Close()
api.RespondError(c, http.StatusInternalServerError, "保存文件失败: "+err.Error())
return
}
if err := tempFile.Close(); err != nil {
api.RespondError(c, http.StatusInternalServerError, "保存文件失败: "+err.Error())
return
}
info, err := plugin.InstallZip(tempPath)
if err != nil {
api.RespondError(c, http.StatusBadRequest, err.Error())
return
}
api.RespondSuccessMessage(c, "插件上传成功", info)
}
// ServePluginFile serves a static file from an installed plugin directory,
// used by injected plugin admin pages.
func ServePluginFile(c *gin.Context) {
+12 -54
View File
@@ -28,39 +28,6 @@ const (
maxThemeManifestSize = 1 << 20
)
// UploadTheme 上传主题
func UploadTheme(c *gin.Context) {
// 读取上传的文件内容
data, err := io.ReadAll(c.Request.Body)
if err != nil || len(data) == 0 {
api.RespondError(c, http.StatusBadRequest, "请选择要上传的主题文件")
return
}
// 临时文件名
tempFile := filepath.Join(os.TempDir(), "uploaded_theme.zip")
if err := os.WriteFile(tempFile, data, 0644); err != nil {
api.RespondError(c, http.StatusInternalServerError, "保存文件失败: "+err.Error())
return
}
defer os.Remove(tempFile)
// 检查文件扩展名(这里假定上传的就是zip)
if !strings.HasSuffix(strings.ToLower(tempFile), ".zip") {
api.RespondError(c, http.StatusBadRequest, "只支持ZIP格式的主题文件")
return
}
// 解压ZIP文件并验证
themeInfo, err := extractAndValidateTheme(tempFile)
if err != nil {
api.RespondError(c, http.StatusBadRequest, err.Error())
return
}
api.RespondSuccessMessage(c, "主题上传成功", themeInfo)
}
// ListThemes 列出所有主题
func ListThemes(c *gin.Context) {
dataDir := "./data/theme"
@@ -217,17 +184,7 @@ func extractAndValidateTheme(zipPath string) (models.Theme, error) {
return themeInfo, fmt.Errorf("主题配置格式错误: %v", err)
}
// 验证必填字段
if themeInfo.Name == "" || themeInfo.Short == "" {
return themeInfo, fmt.Errorf("主题配置缺少必填字段(name、short)")
}
// 验证Short字段格式(只允许字母、数字、下划线、连字符)
if !isValidMarketShort(themeInfo.Short) {
return themeInfo, fmt.Errorf("主题short字段格式无效,只允许字母、数字、下划线和连字符")
}
if err := themeInfo.ValidateConfiguration(); err != nil {
if err := validateThemeManifest(themeInfo); err != nil {
return themeInfo, err
}
@@ -324,6 +281,16 @@ func loadThemeConfig(configPath string) (models.Theme, error) {
return themeInfo, nil
}
func validateThemeManifest(themeInfo models.Theme) error {
if !models.IsLocalizedText(themeInfo.Name) || themeInfo.Short == "" {
return fmt.Errorf("主题配置缺少必填字段(name、short)")
}
if !isValidMarketShort(themeInfo.Short) {
return fmt.Errorf("主题short字段格式无效,只允许字母、数字、下划线和连字符")
}
return themeInfo.ValidateConfiguration()
}
// isValidMarketShort validates a market entry short name (shared by the
// theme and plugin markets).
func isValidMarketShort(short string) bool {
@@ -726,15 +693,7 @@ func peekThemeFromZip(zipPath string) (models.Theme, error) {
return themeInfo, fmt.Errorf("主题配置格式错误: %v", err)
}
if themeInfo.Name == "" || themeInfo.Short == "" {
return themeInfo, fmt.Errorf("主题配置缺少必填字段(name、short)")
}
if !isValidMarketShort(themeInfo.Short) {
return themeInfo, fmt.Errorf("主题short字段格式无效,只允许字母、数字、下划线和连字符")
}
if err := themeInfo.ValidateConfiguration(); err != nil {
if err := validateThemeManifest(themeInfo); err != nil {
return themeInfo, err
}
@@ -840,7 +799,6 @@ func UpdateThemeSettings(c *gin.Context) {
api.RespondError(c, http.StatusBadRequest, "主题名称不能为空")
return
}
var req map[string]any
err := c.ShouldBindJSON(&req)
+2 -11
View File
@@ -16,6 +16,7 @@ import (
"time"
"github.com/gin-gonic/gin"
"github.com/komari-monitor/komari/database/models"
"github.com/komari-monitor/komari/internal/config"
"github.com/komari-monitor/komari/web/api"
)
@@ -445,17 +446,7 @@ func validateThemeMarketTheme(theme ThemeMarketTheme) error {
// isMarketText reports whether a market entry field is usable: a non-empty
// string or an i18n object with at least one non-empty value.
func isMarketText(value any) bool {
switch text := value.(type) {
case string:
return strings.TrimSpace(text) != ""
case map[string]any:
for _, item := range text {
if itemText, ok := item.(string); ok && strings.TrimSpace(itemText) != "" {
return true
}
}
}
return false
return models.IsLocalizedText(value)
}
func validateMarketURLSyntax(rawURL string) error {
+87 -1
View File
@@ -1,6 +1,13 @@
package admin
import "testing"
import (
"archive/zip"
"os"
"path/filepath"
"testing"
"github.com/komari-monitor/komari/database/models"
)
// TestIsValidThemeShort_PathTraversal 防止 DeleteTheme/UpdateTheme/SetTheme
// 的路径穿越漏洞:req.Short 直接进入 filepath.Join("./data/theme", short),
@@ -45,3 +52,82 @@ func TestIsValidThemeShort_PathTraversal(t *testing.T) {
}
}
}
func TestValidateThemeManifestAcceptsLocalizedMetadata(t *testing.T) {
theme := models.Theme{
Name: map[string]any{
"zh-CN": "配置字段演示主题",
"en": "Managed Configuration Demo Theme",
},
Short: "managed-config-demo",
Description: map[string]any{
"zh-CN": "用于验证全部托管配置字段的测试主题。",
"en": "A test theme covering every managed configuration field type.",
},
Author: map[string]any{
"zh-CN": "Komari 团队",
"en": "Komari",
},
}
if err := validateThemeManifest(theme); err != nil {
t.Fatalf("localized theme metadata was rejected: %v", err)
}
}
func TestPeekThemeFromZipAcceptsLocalizedMetadata(t *testing.T) {
zipPath := filepath.Join(t.TempDir(), "localized-theme.zip")
archive, err := os.Create(zipPath)
if err != nil {
t.Fatalf("create zip: %v", err)
}
writer := zip.NewWriter(archive)
manifest, err := writer.Create("komari-theme.json")
if err != nil {
t.Fatalf("create manifest: %v", err)
}
_, err = manifest.Write([]byte(`{
"name": {"zh-CN": "本地化主题", "en": "Localized Theme"},
"short": "localized-theme",
"description": {"zh-CN": "描述", "en": "Description"},
"author": {"zh-CN": "作者", "en": "Author"},
"version": "1.0.0"
}`))
if err != nil {
t.Fatalf("write manifest: %v", err)
}
if err := writer.Close(); err != nil {
t.Fatalf("close zip writer: %v", err)
}
if err := archive.Close(); err != nil {
t.Fatalf("close zip file: %v", err)
}
theme, err := peekThemeFromZip(zipPath)
if err != nil {
t.Fatalf("localized theme package was rejected: %v", err)
}
if theme.Short != "localized-theme" {
t.Fatalf("short = %q, want localized-theme", theme.Short)
}
workDir, err := os.Getwd()
if err != nil {
t.Fatalf("get working directory: %v", err)
}
tempDir := t.TempDir()
if err := os.Chdir(tempDir); err != nil {
t.Fatalf("change working directory: %v", err)
}
t.Cleanup(func() {
_ = os.Chdir(workDir)
})
installed, err := extractAndValidateTheme(zipPath)
if err != nil {
t.Fatalf("localized theme upload package was rejected: %v", err)
}
if installed.Short != "localized-theme" {
t.Fatalf("installed short = %q, want localized-theme", installed.Short)
}
}
-45
View File
@@ -1,45 +0,0 @@
package admin
import (
"fmt"
"net/http"
"os"
"time"
"github.com/gin-gonic/gin"
logger "github.com/komari-monitor/komari/utils/log"
"github.com/komari-monitor/komari/web/api"
"github.com/komari-monitor/komari/web/backup"
)
// UploadBackup receives a backup and applies it after the service restarts.
func UploadBackup(c *gin.Context) {
file, header, err := c.Request.FormFile("backup")
if err != nil {
api.RespondError(c, http.StatusBadRequest, fmt.Sprintf("get uploaded backup: %v", err))
return
}
defer file.Close()
restoreLock, err := backup.AcquireRestoreLock()
if err != nil {
api.RespondError(c, http.StatusConflict, err.Error())
return
}
if err := restoreLock.SaveUploadedBackup(file, header.Filename); err != nil {
restoreLock.Release()
api.RespondError(c, http.StatusBadRequest, err.Error())
return
}
c.JSON(http.StatusOK, gin.H{
"status": "success",
"message": "Backup uploaded successfully. The service will restart and apply the backup.",
"path": "./data/backup.zip",
})
go func() {
logger.InfoArgs("admin-api", "Backup uploaded, restarting service in 2 seconds to apply on startup...")
time.Sleep(2 * time.Second)
restoreLock.Release()
os.Exit(0)
}()
}
+19 -13
View File
@@ -15,12 +15,13 @@ import (
"github.com/gin-gonic/gin"
"github.com/komari-monitor/komari/database/accounts"
"github.com/komari-monitor/komari/internal/metricstore"
"github.com/komari-monitor/komari/database/models"
appconfig "github.com/komari-monitor/komari/internal/config"
"github.com/komari-monitor/komari/internal/metricstore"
logger "github.com/komari-monitor/komari/utils/log"
"github.com/komari-monitor/komari/web/api"
"github.com/komari-monitor/komari/web/backup"
"github.com/komari-monitor/komari/web/upload"
"gorm.io/gorm"
)
@@ -64,7 +65,16 @@ func (c *Controller) Register(r *gin.Engine) {
g := r.Group(APIPath, c.requireActive)
g.GET("/status", c.status)
g.POST("/complete", c.complete)
g.POST("/restore", c.restore)
uploadHandler := upload.NewHandler(upload.DefaultStore, map[upload.Purpose]upload.Finalizer{
upload.PurposeBackup: c.finalizeBackupUpload,
})
uploadGroup := g.Group("/upload")
{
uploadGroup.POST("/init", uploadHandler.Init)
uploadGroup.POST("/chunk", uploadHandler.Chunk)
uploadGroup.POST("/merge", uploadHandler.Merge)
uploadGroup.POST("/cancel", uploadHandler.Cancel)
}
}
func (c *Controller) requireActive(ctx *gin.Context) {
@@ -82,25 +92,21 @@ func (c *Controller) status(ctx *gin.Context) {
api.RespondSuccess(ctx, Status{State: state, Required: state != "completed"})
}
// restore stages a verified backup before the guide creates any new data.
func (c *Controller) restore(ctx *gin.Context) {
file, header, err := ctx.Request.FormFile("backup")
func (c *Controller) finalizeBackupUpload(session upload.Session) (upload.Result, error) {
archive, err := os.Open(session.ArchivePath)
if err != nil {
api.RespondError(ctx, http.StatusBadRequest, fmt.Sprintf("get uploaded backup: %v", err))
return
return upload.Result{}, fmt.Errorf("open merged backup: %w", err)
}
defer file.Close()
if err := backup.SaveUploadedBackup(file, header.Filename); err != nil {
api.RespondError(ctx, http.StatusBadRequest, err.Error())
return
defer archive.Close()
if err := backup.SaveUploadedBackup(archive, session.Metadata.Filename); err != nil {
return upload.Result{}, err
}
api.RespondSuccessMessage(ctx, "backup uploaded; restarting to restore", gin.H{})
go func() {
logger.InfoArgs("install", "Backup uploaded, restarting service to restore it on startup...")
time.Sleep(2 * time.Second)
os.Exit(0)
}()
return upload.Result{Message: "backup uploaded; restarting to restore", Data: gin.H{}}, nil
}
func (c *Controller) complete(ctx *gin.Context) {
+5 -4
View File
@@ -30,6 +30,7 @@ const (
)
const largeDatasetThreshold int64 = 300_000
const structureProgressCeiling = 80.0
type Mode string
@@ -471,7 +472,7 @@ func (c *Controller) failTarget(err error, dsn, phase string) {
func structureProgressPercent(progress metricstore.RestructureProgress) float64 {
if progress.Phase == "reclaiming" {
return 95
return structureProgressCeiling
}
if progress.Phase == "discarding" && progress.RowsTotal == 0 {
return 1
@@ -479,12 +480,12 @@ func structureProgressPercent(progress metricstore.RestructureProgress) float64
if progress.RowsTotal <= 0 {
return 0
}
value := float64(progress.RowsDone) / float64(progress.RowsTotal) * 95
value := float64(progress.RowsDone) / float64(progress.RowsTotal) * structureProgressCeiling
if value < 0 {
return 0
}
if value > 95 {
return 95
if value > structureProgressCeiling {
return structureProgressCeiling
}
return value
}
+8 -2
View File
@@ -126,7 +126,13 @@ func TestUnifiedAuthIncludesMode(t *testing.T) {
}
func TestStructureProgressPercent(t *testing.T) {
if got := structureProgressPercent(metricstore.RestructureProgress{RowsDone: 50, RowsTotal: 100}); got != 47.5 {
t.Fatalf("copying percent = %v, want 47.5", got)
if got := structureProgressPercent(metricstore.RestructureProgress{RowsDone: 50, RowsTotal: 100}); got != 40 {
t.Fatalf("copying percent = %v, want 40", got)
}
if got := structureProgressPercent(metricstore.RestructureProgress{Phase: "reclaiming"}); got != 80 {
t.Fatalf("reclaiming percent = %v, want 80", got)
}
if got := structureProgressPercent(metricstore.RestructureProgress{RowsDone: 100, RowsTotal: 100}); got != 80 {
t.Fatalf("capped percent = %v, want 80", got)
}
}
+10 -9
View File
@@ -86,11 +86,14 @@ func registerAdminRoutes(r *gin.Engine) {
// --- 二进制/流/重定向类,保留 REST handler ---
g.GET("/download/backup", admin.DownloadBackup)
g.POST("/upload/backup", admin.UploadBackup)
// chunk upload 用于大备份文件分块上传,提高稳定性
g.POST("/upload/backup/init", admin.InitChunkUpload)
g.POST("/upload/backup/chunk", admin.UploadChunk)
g.POST("/upload/backup/merge", admin.MergeChunkUpload)
uploadHandler := admin.NewArchiveUploadHandler()
uploadGroup := g.Group("/upload")
{
uploadGroup.POST("/init", uploadHandler.Init)
uploadGroup.POST("/chunk", uploadHandler.Chunk)
uploadGroup.POST("/merge", uploadHandler.Merge)
uploadGroup.POST("/cancel", uploadHandler.Cancel)
}
g.GET("/test/geoip", jsonRpc.Bind("admin:testGeoip", jsonRpc.WithQuery("ip")))
g.POST("/test/sendMessage", jsonRpc.Bind("admin:testSendMessage"))
g.POST("/update/mmdb", admin.UpdateMmdbGeoIP)
@@ -98,10 +101,9 @@ func registerAdminRoutes(r *gin.Engine) {
g.PUT("/update/favicon", admin.UploadFavicon)
g.POST("/update/favicon", admin.DeleteFavicon)
// theme 含文件上传,保留 REST handler。
// theme 的安装流程通过统一的分片上传接口;其余主题接口保留 REST handler。
theme := g.Group("/theme")
{
theme.PUT("/upload", admin.UploadTheme)
theme.GET("/list", admin.ListThemes)
theme.POST("/delete", admin.DeleteTheme)
theme.GET("/set", admin.SetTheme)
@@ -205,13 +207,12 @@ func registerAdminRoutes(r *gin.Engine) {
clipboardGroup.POST("/:id/remove", jsonRpc.Bind("admin:deleteClipboard", jsonRpc.WithPath("id")))
}
// plugins: 上传走 REST(zip 二进制),启停/列表/日志走 RPC2,市场对齐主题市场。
// plugins: 安装流程通过统一的分片上传接口,启停/列表/日志走 RPC2,市场对齐主题市场。
pluginGroup := g.Group("/plugin")
{
pluginGroup.GET("/list", jsonRpc.Bind("admin:listPlugins"))
pluginGroup.POST("/enabled", jsonRpc.Bind("admin:setPluginEnabled"))
pluginGroup.GET("/logs", jsonRpc.Bind("admin:getPluginLogs", jsonRpc.WithQuery("short")))
pluginGroup.POST("/install", admin.UploadPlugin)
pluginGroup.GET("/market/sources", admin.ListPluginMarketSources)
pluginGroup.POST("/market/sources", admin.CreatePluginMarketSource)
pluginGroup.PUT("/market/sources/:id", admin.UpdatePluginMarketSource)
+349
View File
@@ -0,0 +1,349 @@
package jsonrpc
import (
"context"
"database/sql"
"fmt"
"sort"
"strings"
"time"
"github.com/komari-monitor/komari/database/auditlog"
"github.com/komari-monitor/komari/database/dbcore"
"github.com/komari-monitor/komari/internal/metricstore"
"github.com/komari-monitor/komari/pkg/metric"
"github.com/komari-monitor/komari/pkg/rpc"
)
const (
databaseTargetMain = "main"
databaseTargetMetrics = "metrics"
defaultDBQueryLimit = 1000
maxDBQueryLimit = 10000
)
type databaseQueryParams struct {
Database *string `json:"database"`
SQL string `json:"sql"`
Args []any `json:"args"`
Limit *int `json:"limit"`
}
type databaseExecParams struct {
Database *string `json:"database"`
SQL string `json:"sql"`
Args []any `json:"args"`
}
type databaseTablesParams struct {
Database *string `json:"database"`
}
type databaseQueryResponse struct {
Database string `json:"database"`
Driver string `json:"driver"`
Columns []string `json:"columns"`
Rows [][]any `json:"rows"`
RowCount int `json:"row_count"`
Truncated bool `json:"truncated"`
}
type databaseExecResponse struct {
Database string `json:"database"`
Driver string `json:"driver"`
RowsAffected int64 `json:"rows_affected"`
LastInsertID *int64 `json:"last_insert_id"`
}
type databaseTablesResponse struct {
Database string `json:"database"`
Driver string `json:"driver"`
Tables []string `json:"tables"`
}
func init() {
RegisterWithGroupAndMeta("dbQuery", rpc.RoleAdmin, adminDBQuery, &rpc.MethodMeta{
Name: "admin:dbQuery",
Summary: "Execute a SQL query against the main or metrics database",
Params: []rpc.ParamMeta{
{Name: "database", Type: "main | metrics", Description: "Database target (default main)"},
{Name: "sql", Type: "string", Required: true, Description: "SQL query to execute"},
{Name: "args", Type: "any[]", Description: "Positional SQL parameters"},
{Name: "limit", Type: "number", Description: "Maximum rows to return (default 1000, max 10000)"},
},
Returns: "{ database: string, driver: string, columns: string[], rows: any[][], row_count: number, truncated: boolean }",
})
RegisterWithGroupAndMeta("dbExec", rpc.RoleAdmin, adminDBExec, &rpc.MethodMeta{
Name: "admin:dbExec",
Summary: "Execute a SQL statement against the main or metrics database",
Params: []rpc.ParamMeta{
{Name: "database", Type: "main | metrics", Description: "Database target (default main)"},
{Name: "sql", Type: "string", Required: true, Description: "SQL statement to execute"},
{Name: "args", Type: "any[]", Description: "Positional SQL parameters"},
},
Returns: "{ database: string, driver: string, rows_affected: number, last_insert_id: number | null }",
})
RegisterWithGroupAndMeta("dbTables", rpc.RoleAdmin, adminDBTables, &rpc.MethodMeta{
Name: "admin:dbTables",
Summary: "List tables in the main or metrics database",
Params: []rpc.ParamMeta{
{Name: "database", Type: "main | metrics", Description: "Database target (default main)"},
},
Returns: "{ database: string, driver: string, tables: string[] }",
})
}
func adminDBQuery(ctx context.Context, req *rpc.JsonRpcRequest) (any, *rpc.JsonRpcError) {
params, target, statement, limit, rpcErr := parseDatabaseQueryParams(req)
if rpcErr != nil {
return nil, rpcErr
}
response, err := queryDatabase(ctx, target, statement, params.Args, limit)
if err != nil {
return nil, rpc.MakeError(rpc.InternalError, "Failed to execute database query: "+err.Error(), nil)
}
return response, nil
}
func adminDBExec(ctx context.Context, req *rpc.JsonRpcRequest) (any, *rpc.JsonRpcError) {
var params databaseExecParams
if err := req.BindParams(&params); err != nil {
return nil, rpc.MakeError(rpc.InvalidParams, "Invalid request body: "+err.Error(), nil)
}
target, rpcErr := parseDatabaseTarget(params.Database)
if rpcErr != nil {
return nil, rpcErr
}
statement := strings.TrimSpace(params.SQL)
if statement == "" {
return nil, rpc.MakeError(rpc.InvalidParams, "sql is required", nil)
}
actor, ip := auditActor(ctx)
auditlog.Log(ip, actor, "execute database SQL on "+target, "warn")
result, driver, err := executeDatabase(ctx, target, statement, params.Args)
if err != nil {
return nil, rpc.MakeError(rpc.InternalError, "Failed to execute database statement: "+err.Error(), nil)
}
rowsAffected, err := result.RowsAffected()
if err != nil {
return nil, rpc.MakeError(rpc.InternalError, "Failed to read affected row count: "+err.Error(), nil)
}
var lastInsertID *int64
if id, err := result.LastInsertId(); err == nil {
lastInsertID = &id
}
return databaseExecResponse{
Database: target,
Driver: string(driver),
RowsAffected: rowsAffected,
LastInsertID: lastInsertID,
}, nil
}
func adminDBTables(ctx context.Context, req *rpc.JsonRpcRequest) (any, *rpc.JsonRpcError) {
var params databaseTablesParams
if err := req.BindParams(&params); err != nil {
return nil, rpc.MakeError(rpc.InvalidParams, "Invalid request body: "+err.Error(), nil)
}
target, rpcErr := parseDatabaseTarget(params.Database)
if rpcErr != nil {
return nil, rpcErr
}
response, err := listDatabaseTables(ctx, target)
if err != nil {
return nil, rpc.MakeError(rpc.InternalError, "Failed to list database tables: "+err.Error(), nil)
}
return response, nil
}
func parseDatabaseQueryParams(req *rpc.JsonRpcRequest) (databaseQueryParams, string, string, int, *rpc.JsonRpcError) {
var params databaseQueryParams
if err := req.BindParams(&params); err != nil {
return databaseQueryParams{}, "", "", 0, rpc.MakeError(rpc.InvalidParams, "Invalid request body: "+err.Error(), nil)
}
target, rpcErr := parseDatabaseTarget(params.Database)
if rpcErr != nil {
return databaseQueryParams{}, "", "", 0, rpcErr
}
statement := strings.TrimSpace(params.SQL)
if statement == "" {
return databaseQueryParams{}, "", "", 0, rpc.MakeError(rpc.InvalidParams, "sql is required", nil)
}
limit := defaultDBQueryLimit
if params.Limit != nil {
limit = *params.Limit
if limit < 1 || limit > maxDBQueryLimit {
return databaseQueryParams{}, "", "", 0, rpc.MakeError(rpc.InvalidParams, fmt.Sprintf("limit must be between 1 and %d", maxDBQueryLimit), nil)
}
}
return params, target, statement, limit, nil
}
func parseDatabaseTarget(value *string) (string, *rpc.JsonRpcError) {
if value == nil {
return databaseTargetMain, nil
}
target := strings.TrimSpace(*value)
if target != databaseTargetMain && target != databaseTargetMetrics {
return "", rpc.MakeError(rpc.InvalidParams, "database must be main or metrics", nil)
}
return target, nil
}
func queryDatabase(ctx context.Context, target, statement string, args []any, limit int) (databaseQueryResponse, error) {
rows, driver, release, err := openDatabaseRows(ctx, target, statement, args...)
if err != nil {
return databaseQueryResponse{}, err
}
defer func() {
_ = rows.Close()
release()
}()
columns, resultRows, truncated, err := collectDatabaseRows(rows, limit)
if err != nil {
return databaseQueryResponse{}, err
}
return databaseQueryResponse{
Database: target,
Driver: string(driver),
Columns: columns,
Rows: resultRows,
RowCount: len(resultRows),
Truncated: truncated,
}, nil
}
func executeDatabase(ctx context.Context, target, statement string, args []any) (sql.Result, metric.Driver, error) {
switch target {
case databaseTargetMain:
db, err := dbcore.GetDBInstance().DB()
if err != nil {
return nil, "", err
}
result, err := db.ExecContext(ctx, statement, args...)
return result, metric.DriverSQLite, err
case databaseTargetMetrics:
return metricstore.ExecContext(ctx, statement, args...)
default:
return nil, "", fmt.Errorf("unsupported database target: %s", target)
}
}
func listDatabaseTables(ctx context.Context, target string) (databaseTablesResponse, error) {
var (
rows *sql.Rows
actualDriver metric.Driver
release func()
err error
)
switch target {
case databaseTargetMain:
statement, err := tableListSQL(metric.DriverSQLite)
if err != nil {
return databaseTablesResponse{}, err
}
rows, actualDriver, release, err = openDatabaseRows(ctx, target, statement)
case databaseTargetMetrics:
rows, actualDriver, release, err = metricstore.QueryForDriver(ctx, tableListSQL)
default:
return databaseTablesResponse{}, fmt.Errorf("unsupported database target: %s", target)
}
if err != nil {
return databaseTablesResponse{}, err
}
defer func() {
_ = rows.Close()
release()
}()
tables := make([]string, 0)
for rows.Next() {
var name string
if err := rows.Scan(&name); err != nil {
return databaseTablesResponse{}, err
}
tables = append(tables, name)
}
if err := rows.Err(); err != nil {
return databaseTablesResponse{}, err
}
sort.Strings(tables)
return databaseTablesResponse{Database: target, Driver: string(actualDriver), Tables: tables}, nil
}
func openDatabaseRows(ctx context.Context, target, statement string, args ...any) (*sql.Rows, metric.Driver, func(), error) {
switch target {
case databaseTargetMain:
db, err := dbcore.GetDBInstance().DB()
if err != nil {
return nil, "", nil, err
}
rows, err := db.QueryContext(ctx, statement, args...)
return rows, metric.DriverSQLite, func() {}, err
case databaseTargetMetrics:
return metricstore.QueryContext(ctx, statement, args...)
default:
return nil, "", nil, fmt.Errorf("unsupported database target: %s", target)
}
}
func collectDatabaseRows(rows *sql.Rows, limit int) ([]string, [][]any, bool, error) {
columns, err := rows.Columns()
if err != nil {
return nil, nil, false, err
}
resultRows := make([][]any, 0, limit)
values := make([]any, len(columns))
scans := make([]any, len(columns))
for i := range scans {
scans[i] = &values[i]
}
for rows.Next() {
if len(resultRows) == limit {
return columns, resultRows, true, nil
}
if err := rows.Scan(scans...); err != nil {
return nil, nil, false, err
}
row := make([]any, len(values))
for i, value := range values {
row[i] = normalizeDatabaseValue(value)
}
resultRows = append(resultRows, row)
}
if err := rows.Err(); err != nil {
return nil, nil, false, err
}
return columns, resultRows, false, nil
}
func normalizeDatabaseValue(value any) any {
switch value := value.(type) {
case []byte:
return string(value)
case time.Time:
return value.Format(time.RFC3339Nano)
default:
return value
}
}
func tableListSQL(driver metric.Driver) (string, error) {
switch driver {
case metric.DriverSQLite:
return "SELECT name FROM sqlite_master WHERE type = 'table' AND name NOT LIKE 'sqlite_%'", nil
case metric.DriverMySQL:
return "SELECT table_name FROM information_schema.tables WHERE table_schema = DATABASE() AND table_type = 'BASE TABLE'", nil
case metric.DriverPostgreSQL:
return "SELECT tablename FROM pg_catalog.pg_tables WHERE schemaname = current_schema()", nil
default:
return "", fmt.Errorf("unsupported database driver: %s", driver)
}
}
+3
View File
@@ -113,5 +113,8 @@ func adminSetPluginConfiguration(_ context.Context, req *rpc.JsonRpcRequest) (an
if err := plugin.SaveConfiguration(short, data); err != nil {
return nil, rpc.MakeError(rpc.InternalError, err.Error(), nil)
}
if err := plugin.Reload(short); err != nil {
return nil, rpc.MakeError(rpc.InternalError, "plugin configuration saved but reload failed: "+err.Error(), nil)
}
return nil, nil
}
+30
View File
@@ -4,6 +4,7 @@ import (
"archive/zip"
"os"
"path/filepath"
"strings"
"testing"
"github.com/komari-monitor/komari/cmd/flags"
@@ -93,6 +94,35 @@ func TestAdminPluginConfigurationRoundTrip(t *testing.T) {
}
}
func TestAdminPluginConfigurationKeepsSavedDataWhenReloadFails(t *testing.T) {
withTempPluginDir(t)
installTestPlugin(t, `{"name":"Cfg","short":"cfg","version":"1.0.0","configuration":{"type":"managed","data":[{"key":"greeting","name":"Greeting","type":"string","default":"hi"}]}}`, "function load() {}")
if err := plugin.SetEnabled("cfg", true, false); err != nil {
t.Fatalf("enable failed: %v", err)
}
if err := os.WriteFile(filepath.Join(plugin.DataDir, "cfg", "script.js"), []byte(`function load() { throw new Error("reload-boom"); }`), 0644); err != nil {
t.Fatal(err)
}
_, jerr := adminSetPluginConfiguration(nil, &rpc.JsonRpcRequest{
Version: rpc.RPC_VERSION,
Method: "admin:setPluginConfiguration",
Params: map[string]any{"short": "cfg", "data": map[string]any{"greeting": "saved-before-reload-error"}},
})
if jerr == nil || !strings.Contains(jerr.Message, "plugin configuration saved but reload failed") {
t.Fatalf("expected reload error after saving, got %v", jerr)
}
values, err := plugin.GetConfiguration("cfg")
if err != nil {
t.Fatal(err)
}
if values["greeting"] != "saved-before-reload-error" {
t.Fatalf("saved configuration was rolled back: %#v", values)
}
}
// withTempPluginDir redirects plugin.DataDir for the duration of a test.
func withTempPluginDir(t *testing.T) {
t.Helper()
+231 -176
View File
@@ -10,9 +10,9 @@ import (
"time"
"github.com/komari-monitor/komari/database/clients"
"github.com/komari-monitor/komari/internal/metricstore"
"github.com/komari-monitor/komari/database/models"
"github.com/komari-monitor/komari/database/tasks"
"github.com/komari-monitor/komari/internal/metricstore"
"github.com/komari-monitor/komari/pkg/metric"
"github.com/komari-monitor/komari/pkg/rpc"
)
@@ -54,11 +54,12 @@ type publicMetricQueryParams struct {
}
type publicMetricPoint struct {
Time time.Time `json:"time"`
Value *float64 `json:"value"`
Count int `json:"count,omitempty"`
Tags map[string]string `json:"tags,omitempty"`
Labels map[string]string `json:"labels,omitempty"`
entityID string
Time time.Time `json:"time"`
Value *float64 `json:"value"`
Count int `json:"count,omitempty"`
Tags map[string]string `json:"tags,omitempty"`
Labels map[string]string `json:"labels,omitempty"`
}
type publicMetricSeries struct {
@@ -167,7 +168,8 @@ func publicQueryMetrics(ctx context.Context, req *rpc.JsonRpcRequest) (any, *rpc
return nil, rpc.MakeError(rpc.InvalidParams, "end must be after start", nil)
}
entityIDs, rpcErr := publicMetricEntityIDs(ctx, normalizeStringList(params.EntityIDs, []string{params.EntityID}))
requestedEntityIDs := normalizeStringList(params.EntityIDs, []string{params.EntityID})
entityIDs, rpcErr := publicMetricEntityIDs(ctx, requestedEntityIDs)
if rpcErr != nil {
return nil, rpcErr
}
@@ -176,59 +178,154 @@ func publicQueryMetrics(ctx context.Context, req *rpc.JsonRpcRequest) (any, *rpc
if store == nil {
return nil, rpc.MakeError(rpc.InternalError, "metric store not initialized", nil)
}
defs, err := store.ListMetrics(ctx)
if err != nil {
return nil, rpc.MakeError(rpc.InternalError, "Failed to list metric definitions: "+err.Error(), nil)
}
defMap := make(map[string]metric.Definition, len(defs))
for _, def := range defs {
defMap[def.Name] = def
}
series := make([]publicMetricSeries, 0, len(metricKeys)*maxInt(1, len(entityIDs)))
type metricLoadSpec struct {
metricKey string
algorithm metric.Aggregation
maxPoints int
interval time.Duration
}
loadSpecs := make([]metricLoadSpec, 0, len(metricKeys))
for _, metricKey := range metricKeys {
def, ok := defMap[metricKey]
if !ok {
return nil, rpc.MakeError(rpc.InvalidParams, "unknown metric key: "+metricKey, nil)
}
maxPoints, err := resolveMetricMaxPoints(metricKey, params)
if err != nil {
return nil, rpc.MakeError(rpc.InvalidParams, err.Error(), nil)
}
algorithm := resolveMetricAggregation(metricKey, params)
metricFillEmpty := resolveMetricFillEmpty(params)
loadSpecs = append(loadSpecs, metricLoadSpec{
metricKey: metricKey,
algorithm: resolveMetricAggregation(metricKey, params),
maxPoints: maxPoints,
})
}
metricFillEmpty := resolveMetricFillEmpty(params)
useRaw := publicMetricUsesRawWindow(start, end, queryNow)
definitions := make(map[string]metric.Definition, len(metricKeys))
rawValues := make(map[string][]metric.Point)
rollupValues := make(map[string]map[metric.Aggregation][]metric.AggregatePoint)
if len(entityIDs) > 0 && useRaw {
var err error
definitions, err = store.GetMetrics(ctx, metricKeys)
if err != nil {
return nil, rpc.MakeError(rpc.InternalError, "Failed to query metric definitions: "+err.Error(), nil)
}
for _, spec := range loadSpecs {
if _, ok := definitions[spec.metricKey]; !ok {
return nil, rpc.MakeError(rpc.InvalidParams, "unknown metric key: "+spec.metricKey, nil)
}
}
rawValues, err = store.QueryBatch(ctx, metric.BatchQuery{
MetricNames: metricKeys,
EntityIDs: entityIDs,
Start: start,
End: end,
Tags: params.Tags,
Order: metric.OrderAsc,
})
if err != nil {
return nil, rpc.MakeError(rpc.InvalidParams, "Failed to query metrics: "+err.Error(), nil)
}
} else if len(entityIDs) > 0 {
batchSpecs := make([]metric.BatchSeriesSpec, 0, len(loadSpecs))
for i := range loadSpecs {
loadSpecs[i].interval = metricDownsampleInterval(end.Sub(start), loadSpecs[i].maxPoints)
loadSpecs[i].interval = store.CompatibleSeriesInterval(start, queryNow, loadSpecs[i].interval)
batchSpecs = append(batchSpecs, metric.BatchSeriesSpec{
MetricName: loadSpecs[i].metricKey,
Aggregations: []metric.Aggregation{loadSpecs[i].algorithm},
Interval: loadSpecs[i].interval,
PreserveSeries: true,
})
}
loaded, err := store.SeriesBatch(ctx, metric.BatchSeriesQuery{
Specs: batchSpecs,
EntityIDs: entityIDs,
Start: start,
End: end,
Tags: params.Tags,
Order: metric.OrderAsc,
}, queryNow)
if err != nil {
return nil, rpc.MakeError(rpc.InvalidParams, "Failed to query metrics: "+err.Error(), nil)
}
definitions = loaded.Definitions
rollupValues = loaded.Values
for _, spec := range loadSpecs {
if _, ok := definitions[spec.metricKey]; !ok {
return nil, rpc.MakeError(rpc.InvalidParams, "unknown metric key: "+spec.metricKey, nil)
}
}
} else {
var err error
definitions, err = store.GetMetrics(ctx, metricKeys)
if err != nil {
return nil, rpc.MakeError(rpc.InternalError, "Failed to query metric definitions: "+err.Error(), nil)
}
for _, spec := range loadSpecs {
if _, ok := definitions[spec.metricKey]; !ok {
return nil, rpc.MakeError(rpc.InvalidParams, "unknown metric key: "+spec.metricKey, nil)
}
}
}
series := make([]publicMetricSeries, 0, len(metricKeys)*maxInt(1, len(entityIDs)))
for _, spec := range loadSpecs {
def := definitions[spec.metricKey]
item := publicMetricSeries{
MetricKey: spec.metricKey,
Type: string(def.Type),
Unit: def.Unit,
RetentionDays: def.RetentionDays,
Tags: params.Tags,
FillEmpty: metricFillEmpty,
MaxPoints: spec.maxPoints,
Downsampled: !useRaw,
}
if useRaw {
points := rawValues[spec.metricKey]
item.Points = make([]publicMetricPoint, 0, len(points))
for _, point := range points {
item.Points = append(item.Points, publicMetricPoint{
entityID: point.EntityID,
Time: point.Timestamp.UTC(),
Value: publicRawMetricValue(point.MetricName, point.Value, metricFillEmpty),
Count: 1,
Tags: point.Tags,
Labels: point.Labels,
})
}
} else {
item.DownsampleAlgorithm = string(spec.algorithm)
item.IntervalSeconds = spec.interval.Seconds()
points := rollupValues[spec.metricKey][spec.algorithm]
item.Points = make([]publicMetricPoint, 0, len(points))
for _, point := range points {
item.Points = append(item.Points, publicMetricPoint{
entityID: point.EntityID,
Time: point.Bucket.UTC(),
Value: publicRawMetricValue(point.MetricName, point.Value, metricFillEmpty),
Count: point.Count,
Tags: point.Tags,
})
}
}
byEntity := make(map[string][]publicMetricSeries, len(entityIDs))
for _, split := range splitPublicMetricSeries(item) {
if split.EntityID != "" {
byEntity[split.EntityID] = append(byEntity[split.EntityID], split)
}
}
for _, entityID := range entityIDs {
query := metric.Query{
MetricName: metricKey,
EntityID: entityID,
Start: start,
End: end,
Tags: params.Tags,
Order: metric.OrderAsc,
entitySeries := byEntity[entityID]
if len(entitySeries) == 0 {
empty := item
empty.EntityID = entityID
empty.Count = 0
empty.Points = nil
entitySeries = []publicMetricSeries{empty}
}
item := publicMetricSeries{
MetricKey: metricKey,
EntityID: entityID,
Type: string(def.Type),
Unit: def.Unit,
RetentionDays: def.RetentionDays,
Tags: params.Tags,
FillEmpty: metricFillEmpty,
MaxPoints: maxPoints,
}
loaded, err := loadPublicMetricPoints(ctx, store, query, algorithm, maxPoints, metricFillEmpty, queryNow)
if err != nil {
return nil, rpc.MakeError(rpc.InvalidParams, "Failed to query metric "+metricKey+": "+err.Error(), nil)
}
item.Downsampled = loaded.downsampled
item.Points = loaded.points
if loaded.downsampled {
item.DownsampleAlgorithm = string(algorithm)
item.IntervalSeconds = loaded.interval.Seconds()
}
for _, split := range splitPublicMetricSeries(item) {
for _, split := range entitySeries {
if metricFillEmpty {
split = adaptiveFillPublicMetricSeries(split, start, end)
}
@@ -263,62 +360,21 @@ func loadPublicMetricPoints(
now time.Time,
) (publicMetricPointResult, error) {
if publicMetricUsesRawWindow(query.Start, query.End, now) {
fallbackInterval := store.CompatibleSeriesInterval(query.Start, now, time.Minute)
// Read rollups first so a concurrent write is observed by the later raw
// read and suppresses the older summary for that bucket.
fallback, err := store.Series(ctx, metric.AggregateQuery{
Query: query,
Aggregation: algorithm,
Interval: fallbackInterval,
PreserveSeries: true,
}, now)
if err != nil {
return publicMetricPointResult{}, err
}
points, err := store.Query(ctx, query)
if err != nil {
return publicMetricPointResult{}, err
}
result := publicMetricPointResult{points: make([]publicMetricPoint, 0, len(points))}
rawBuckets := make(map[string]struct{}, len(points))
for _, point := range points {
publicPoint := publicMetricPoint{
Time: point.Timestamp.UTC(),
Value: publicRawMetricValue(point.MetricName, point.Value, fillEmpty),
Count: 1,
Tags: point.Tags,
Labels: point.Labels,
}
result.points = append(result.points, publicPoint)
rawBuckets[publicMetricBucketKey(publicPoint, fallbackInterval)] = struct{}{}
result.points = append(result.points, publicMetricPoint{
entityID: point.EntityID,
Time: point.Timestamp.UTC(),
Value: publicRawMetricValue(point.MetricName, point.Value, fillEmpty),
Count: 1,
Tags: point.Tags,
Labels: point.Labels,
})
}
// Exact samples disappear on restart, while the minute summaries remain
// persisted. Fill only buckets that have no exact sample so a restart does
// not create a false gap or duplicate a bucket that is still in memory.
for _, point := range fallback {
publicPoint := publicMetricPoint{
Time: point.Bucket.UTC(),
Value: publicRawMetricValue(point.MetricName, point.Value, fillEmpty),
Count: point.Count,
Tags: point.Tags,
}
if publicPoint.Time.Before(query.Start) || publicPoint.Time.Add(fallbackInterval).After(query.End) {
continue
}
if _, exact := rawBuckets[publicMetricBucketKey(publicPoint, fallbackInterval)]; exact {
continue
}
result.points = append(result.points, publicPoint)
result.downsampled = true
result.interval = fallbackInterval
}
sort.SliceStable(result.points, func(i, j int) bool {
if result.points[i].Time.Equal(result.points[j].Time) {
return publicMetricTagsKey(result.points[i].Tags) < publicMetricTagsKey(result.points[j].Tags)
}
return result.points[i].Time.Before(result.points[j].Time)
})
return result, nil
}
@@ -340,10 +396,11 @@ func loadPublicMetricPoints(
}
for _, point := range points {
result.points = append(result.points, publicMetricPoint{
Time: point.Bucket.UTC(),
Value: publicRawMetricValue(point.MetricName, point.Value, fillEmpty),
Count: point.Count,
Tags: point.Tags,
entityID: point.EntityID,
Time: point.Bucket.UTC(),
Value: publicRawMetricValue(point.MetricName, point.Value, fillEmpty),
Count: point.Count,
Tags: point.Tags,
})
}
return result, nil
@@ -354,11 +411,6 @@ func publicMetricUsesRawWindow(start, end, now time.Time) bool {
return end.Sub(start) <= retention && end.After(now.UTC().Add(-retention))
}
func publicMetricBucketKey(point publicMetricPoint, interval time.Duration) string {
bucket := point.Time.UTC().Truncate(interval).UnixMilli()
return strconv.FormatInt(bucket, 10) + "\x00" + publicMetricTagsKey(point.Tags)
}
func publicGetPingMetricStats(ctx context.Context, req *rpc.JsonRpcRequest) (any, *rpc.JsonRpcError) {
var params publicPingMetricStatsParams
if err := req.BindParams(&params); err != nil {
@@ -409,14 +461,13 @@ func publicGetPingMetricStats(ctx context.Context, req *rpc.JsonRpcRequest) (any
interval := metricDownsampleInterval(end.Sub(start), maxPoints)
interval = store.CompatibleSeriesInterval(start, now, interval)
groupsByEntity, err := loadPublicPingMetricAggregateGroups(ctx, store, entityIDs, start, end, interval, now)
if err != nil {
return nil, rpc.MakeError(rpc.InternalError, "Failed to query ping stats: "+err.Error(), nil)
}
stats := make([]publicPingMetricTaskStats, 0)
for _, entityID := range entityIDs {
groups, err := loadPublicPingMetricAggregateGroups(ctx, store, entityID, start, end, interval, now)
if err != nil {
return nil, rpc.MakeError(rpc.InternalError, "Failed to query ping stats: "+err.Error(), nil)
}
entityStats := publicPingStatsFromAggregateGroups(entityID, groups, taskMap, taskFilter)
stats = append(stats, entityStats...)
stats = append(stats, publicPingStatsFromAggregateGroups(entityID, groupsByEntity[entityID], taskMap, taskFilter)...)
}
sort.Slice(stats, func(i, j int) bool {
@@ -452,6 +503,9 @@ func splitPublicMetricSeries(base publicMetricSeries) []publicMetricSeries {
order := make([]string, 0)
for _, point := range base.Points {
entityID := base.EntityID
if point.entityID != "" {
entityID = point.entityID
}
tags := point.Tags
point.Tags = tags
tagsKey := publicMetricTagsKey(tags)
@@ -724,77 +778,78 @@ type publicPingMetricAggregateGroups struct {
LossAvailable bool
}
func loadPublicPingMetricAggregateGroups(ctx context.Context, store *metric.Store, entityID string, start, end time.Time, interval time.Duration, now time.Time) (publicPingMetricAggregateGroups, error) {
query := func(metricName string, aggregation metric.Aggregation) (map[string][]metric.AggregatePoint, error) {
points, err := store.Series(ctx, metric.AggregateQuery{
Query: metric.Query{
MetricName: metricName,
EntityID: entityID,
Start: start,
End: end,
Order: metric.OrderAsc,
},
Aggregation: aggregation,
Interval: interval,
PreserveSeries: true,
}, now)
if err != nil {
return nil, err
func loadPublicPingMetricAggregateGroups(ctx context.Context, store *metric.Store, entityIDs []string, start, end time.Time, interval time.Duration, now time.Time) (map[string]publicPingMetricAggregateGroups, error) {
latencyAggregations := []metric.Aggregation{
metric.AggAvg,
metric.AggMin,
metric.AggMax,
metric.AggLast,
metric.AggP50,
metric.AggP99,
metric.AggStdDev,
}
loaded, err := store.SeriesBatch(ctx, metric.BatchSeriesQuery{
Specs: []metric.BatchSeriesSpec{
{MetricName: metricstore.MetricPingLatency, Aggregations: latencyAggregations, Interval: interval, PreserveSeries: true},
{MetricName: metricstore.MetricPingLoss, Aggregations: []metric.Aggregation{metric.AggAvg}, Interval: interval, PreserveSeries: true},
},
EntityIDs: entityIDs,
Start: start,
End: end,
Order: metric.OrderAsc,
}, now)
if err != nil {
return nil, err
}
latency := loaded.Values[metricstore.MetricPingLatency]
lossPoints := loaded.Values[metricstore.MetricPingLoss][metric.AggAvg]
avg := groupPingMetricAggregatePointsByEntity(latency[metric.AggAvg])
minimum := groupPingMetricAggregatePointsByEntity(latency[metric.AggMin])
maximum := groupPingMetricAggregatePointsByEntity(latency[metric.AggMax])
last := groupPingMetricAggregatePointsByEntity(latency[metric.AggLast])
p50 := groupPingMetricAggregatePointsByEntity(latency[metric.AggP50])
p99 := groupPingMetricAggregatePointsByEntity(latency[metric.AggP99])
stddev := groupPingMetricAggregatePointsByEntity(latency[metric.AggStdDev])
loss := groupPingMetricAggregatePointsByEntity(lossPoints)
entitySet := make(map[string]struct{})
for _, groups := range []map[string]map[string][]metric.AggregatePoint{avg, minimum, maximum, last, p50, p99, stddev, loss} {
for currentEntityID := range groups {
entitySet[currentEntityID] = struct{}{}
}
return groupPingMetricAggregatePointsByTask(points), nil
}
avg, err := query(metricstore.MetricPingLatency, metric.AggAvg)
if err != nil {
return publicPingMetricAggregateGroups{}, err
result := make(map[string]publicPingMetricAggregateGroups, len(entitySet))
for currentEntityID := range entitySet {
entityLoss := loss[currentEntityID]
result[currentEntityID] = publicPingMetricAggregateGroups{
Avg: avg[currentEntityID],
Min: minimum[currentEntityID],
Max: maximum[currentEntityID],
Last: last[currentEntityID],
P50: p50[currentEntityID],
P99: p99[currentEntityID],
StdDev: stddev[currentEntityID],
Loss: entityLoss,
LossAvailable: pingMetricGroupsHaveData(entityLoss),
}
}
minimum, err := query(metricstore.MetricPingLatency, metric.AggMin)
if err != nil {
return publicPingMetricAggregateGroups{}, err
}
maximum, err := query(metricstore.MetricPingLatency, metric.AggMax)
if err != nil {
return publicPingMetricAggregateGroups{}, err
}
last, err := query(metricstore.MetricPingLatency, metric.AggLast)
if err != nil {
return publicPingMetricAggregateGroups{}, err
}
p50, err := query(metricstore.MetricPingLatency, metric.AggP50)
if err != nil {
return publicPingMetricAggregateGroups{}, err
}
p99, err := query(metricstore.MetricPingLatency, metric.AggP99)
if err != nil {
return publicPingMetricAggregateGroups{}, err
}
stddev, err := query(metricstore.MetricPingLatency, metric.AggStdDev)
if err != nil {
return publicPingMetricAggregateGroups{}, err
}
loss, lossErr := query(metricstore.MetricPingLoss, metric.AggAvg)
return publicPingMetricAggregateGroups{
Avg: avg,
Min: minimum,
Max: maximum,
Last: last,
P50: p50,
P99: p99,
StdDev: stddev,
Loss: loss,
LossAvailable: lossErr == nil && pingMetricGroupsHaveData(loss),
}, nil
return result, nil
}
func groupPingMetricAggregatePointsByTask(points []metric.AggregatePoint) map[string][]metric.AggregatePoint {
out := make(map[string][]metric.AggregatePoint)
func groupPingMetricAggregatePointsByEntity(points []metric.AggregatePoint) map[string]map[string][]metric.AggregatePoint {
out := make(map[string]map[string][]metric.AggregatePoint)
for _, point := range points {
taskID := strings.TrimSpace(point.Tags["task_id"])
if taskID == "" {
continue
}
out[taskID] = append(out[taskID], point)
byTask := out[point.EntityID]
if byTask == nil {
byTask = make(map[string][]metric.AggregatePoint)
out[point.EntityID] = byTask
}
byTask[taskID] = append(byTask[taskID], point)
}
return out
}
+10 -22
View File
@@ -9,8 +9,8 @@ import (
"testing"
"time"
"github.com/komari-monitor/komari/internal/metricstore"
"github.com/komari-monitor/komari/database/models"
"github.com/komari-monitor/komari/internal/metricstore"
"github.com/komari-monitor/komari/pkg/metric"
"github.com/komari-monitor/komari/pkg/rpc"
)
@@ -411,7 +411,7 @@ func TestPublicMetricUsesRawWindowOnlyForCurrentlyRetainedRange(t *testing.T) {
}
}
func TestLoadPublicMetricPointsUsesRollupsForRestartGapWithoutDuplicatingRaw(t *testing.T) {
func TestLoadPublicMetricPointsReturnsOnlyRawAfterRestart(t *testing.T) {
ctx := context.Background()
dsn := filepath.Join(t.TempDir(), "metrics.db")
policy := metric.RollupPolicy{
@@ -473,28 +473,16 @@ func TestLoadPublicMetricPointsUsesRollupsForRestartGapWithoutDuplicatingRaw(t *
if err != nil {
t.Fatalf("load mixed restart window: %v", err)
}
if !got.downsampled || got.interval != time.Minute {
t.Fatalf("restart fallback metadata = %#v", got)
if got.downsampled || got.interval != 0 {
t.Fatalf("raw query metadata = %#v", got)
}
if len(got.points) != 3 {
t.Fatalf("restart window returned %d points, want two rollups and one raw: %#v", len(got.points), got.points)
if len(got.points) != 1 {
t.Fatalf("restart window returned %d points, want one raw point: %#v", len(got.points), got.points)
}
values := []float64{1, 2, 3}
for i, point := range got.points {
if point.Value == nil || *point.Value != values[i] {
t.Fatalf("point %d = %#v, want value %v", i, point, values[i])
}
if got.points[0].Value == nil || *got.points[0].Value != 3 {
t.Fatalf("raw point = %#v, want value 3", got.points[0])
}
if got.points[2].Count != 1 || got.points[2].Labels["source"] != "raw" {
t.Fatalf("post-restart exact point changed: %#v", got.points[2])
}
// The current hot rollup covers the same minute as the exact point. It must
// be omitted rather than returned as a fourth, duplicate observation.
currentBucket := afterRestart.Timestamp.Truncate(time.Minute)
for _, point := range got.points {
if point.Time.Equal(currentBucket) {
t.Fatalf("current raw minute also returned its rollup: %#v", got.points)
}
if got.points[0].Count != 1 || got.points[0].Labels["source"] != "raw" {
t.Fatalf("post-restart exact point changed: %#v", got.points[0])
}
}
+228
View File
@@ -0,0 +1,228 @@
// Package upload provides the shared chunked archive upload flow.
package upload
import (
"encoding/json"
"errors"
"fmt"
"io"
"os"
"path/filepath"
"strconv"
"github.com/google/uuid"
"github.com/komari-monitor/komari/web/backup"
)
const ChunkSize int64 = 5 * 1024 * 1024
type Purpose string
const (
PurposeBackup Purpose = "backup"
PurposePlugin Purpose = "plugin"
PurposeTheme Purpose = "theme"
)
var ErrNotFound = errors.New("upload not found")
type Metadata struct {
Purpose Purpose `json:"purpose"`
Size int64 `json:"size"`
Filename string `json:"filename"`
}
type Session struct {
ID string
Metadata Metadata
Directory string
ArchivePath string
}
type Store struct {
Root string
MaxSize int64
}
var DefaultStore = &Store{
Root: filepath.Join(".", "data", ".uploading"),
MaxSize: backup.MaxArchiveSize,
}
func (s *Store) Init(purpose Purpose, filename string, size int64) (Session, error) {
if !isKnownPurpose(purpose) {
return Session{}, fmt.Errorf("invalid upload purpose")
}
if size <= 0 || size > s.MaxSize {
return Session{}, fmt.Errorf("size must be between 1 and %d bytes", s.MaxSize)
}
id := uuid.NewString()
directory := filepath.Join(s.Root, id)
if err := os.MkdirAll(directory, 0755); err != nil {
return Session{}, fmt.Errorf("create upload directory: %w", err)
}
metadata := Metadata{Purpose: purpose, Size: size, Filename: filename}
data, err := json.Marshal(metadata)
if err != nil {
_ = os.RemoveAll(directory)
return Session{}, fmt.Errorf("encode upload metadata: %w", err)
}
if err := os.WriteFile(filepath.Join(directory, "upload.json"), data, 0600); err != nil {
_ = os.RemoveAll(directory)
return Session{}, fmt.Errorf("write upload metadata: %w", err)
}
return Session{ID: id, Metadata: metadata, Directory: directory}, nil
}
func (s *Store) SaveChunk(uploadID string, index int64, source io.Reader) error {
session, err := s.load(uploadID)
if err != nil {
return err
}
expectedSize, err := expectedChunkSize(session.Metadata.Size, index)
if err != nil {
return err
}
temporary, err := os.CreateTemp(session.Directory, fmt.Sprintf(".%d-*.part", index))
if err != nil {
return fmt.Errorf("create chunk: %w", err)
}
temporaryPath := temporary.Name()
defer os.Remove(temporaryPath)
written, copyErr := io.Copy(temporary, io.LimitReader(source, ChunkSize+1))
closeErr := temporary.Close()
if copyErr != nil {
return fmt.Errorf("write chunk: %w", copyErr)
}
if closeErr != nil {
return fmt.Errorf("close chunk: %w", closeErr)
}
if written != expectedSize {
return fmt.Errorf("chunk %d has size %d, want %d", index, written, expectedSize)
}
chunkPath := filepath.Join(session.Directory, chunkFilename(index))
if err := os.Remove(chunkPath); err != nil && !os.IsNotExist(err) {
return fmt.Errorf("replace chunk: %w", err)
}
if err := os.Rename(temporaryPath, chunkPath); err != nil {
return fmt.Errorf("publish chunk: %w", err)
}
return nil
}
func (s *Store) Merge(uploadID string) (Session, error) {
session, err := s.load(uploadID)
if err != nil {
return Session{}, err
}
temporary, err := os.CreateTemp(session.Directory, ".merged-*.zip")
if err != nil {
return Session{}, fmt.Errorf("create merged archive: %w", err)
}
temporaryPath := temporary.Name()
defer func() {
_ = temporary.Close()
_ = os.Remove(temporaryPath)
}()
for index := int64(0); index < chunkCount(session.Metadata.Size); index++ {
chunkPath := filepath.Join(session.Directory, chunkFilename(index))
info, err := os.Stat(chunkPath)
if err != nil {
if os.IsNotExist(err) {
return Session{}, fmt.Errorf("chunk %d is missing", index)
}
return Session{}, fmt.Errorf("read chunk %d: %w", index, err)
}
expectedSize, _ := expectedChunkSize(session.Metadata.Size, index)
if info.Size() != expectedSize {
return Session{}, fmt.Errorf("chunk %d has size %d, want %d", index, info.Size(), expectedSize)
}
chunk, err := os.Open(chunkPath)
if err != nil {
return Session{}, fmt.Errorf("open chunk %d: %w", index, err)
}
_, copyErr := io.Copy(temporary, chunk)
closeErr := chunk.Close()
if copyErr != nil {
return Session{}, fmt.Errorf("merge chunk %d: %w", index, copyErr)
}
if closeErr != nil {
return Session{}, fmt.Errorf("close chunk %d: %w", index, closeErr)
}
}
if err := temporary.Close(); err != nil {
return Session{}, fmt.Errorf("close merged archive: %w", err)
}
archivePath := filepath.Join(session.Directory, "archive.zip")
if err := os.Rename(temporaryPath, archivePath); err != nil {
return Session{}, fmt.Errorf("publish merged archive: %w", err)
}
session.ArchivePath = archivePath
return session, nil
}
func (s *Store) Cancel(uploadID string) error {
if !validUploadID(uploadID) {
return fmt.Errorf("invalid upload id")
}
if err := os.RemoveAll(filepath.Join(s.Root, uploadID)); err != nil {
return fmt.Errorf("remove upload: %w", err)
}
return nil
}
func (s *Store) load(uploadID string) (Session, error) {
if !validUploadID(uploadID) {
return Session{}, fmt.Errorf("invalid upload id")
}
directory := filepath.Join(s.Root, uploadID)
data, err := os.ReadFile(filepath.Join(directory, "upload.json"))
if err != nil {
if os.IsNotExist(err) {
return Session{}, ErrNotFound
}
return Session{}, fmt.Errorf("read upload metadata: %w", err)
}
var metadata Metadata
if err := json.Unmarshal(data, &metadata); err != nil {
return Session{}, fmt.Errorf("read upload metadata: %w", err)
}
if !isKnownPurpose(metadata.Purpose) || metadata.Size <= 0 || metadata.Size > s.MaxSize {
return Session{}, fmt.Errorf("invalid upload metadata")
}
return Session{ID: uploadID, Metadata: metadata, Directory: directory}, nil
}
func isKnownPurpose(purpose Purpose) bool {
return purpose == PurposeBackup || purpose == PurposePlugin || purpose == PurposeTheme
}
func validUploadID(uploadID string) bool {
_, err := uuid.Parse(uploadID)
return err == nil
}
func chunkCount(size int64) int64 {
return (size + ChunkSize - 1) / ChunkSize
}
func expectedChunkSize(size, index int64) (int64, error) {
if index < 0 || index >= chunkCount(size) {
return 0, fmt.Errorf("invalid chunk index")
}
if index == chunkCount(size)-1 {
return size - index*ChunkSize, nil
}
return ChunkSize, nil
}
func chunkFilename(index int64) string {
return strconv.FormatInt(index, 10) + ".part"
}
+127
View File
@@ -0,0 +1,127 @@
package upload
import (
"errors"
"fmt"
"net/http"
"strconv"
"github.com/gin-gonic/gin"
"github.com/komari-monitor/komari/web/api"
)
const maxChunkRequestSize = ChunkSize + 1*1024*1024
type Result struct {
Message string
Data any
}
type Finalizer func(Session) (Result, error)
type Handler struct {
Store *Store
Finalizers map[Purpose]Finalizer
}
func NewHandler(store *Store, finalizers map[Purpose]Finalizer) *Handler {
return &Handler{Store: store, Finalizers: finalizers}
}
func (h *Handler) Init(c *gin.Context) {
var request struct {
Purpose Purpose `json:"purpose" binding:"required"`
Size int64 `json:"size" binding:"required"`
Filename string `json:"filename"`
}
if err := c.ShouldBindJSON(&request); err != nil {
api.RespondError(c, http.StatusBadRequest, fmt.Sprintf("invalid request: %v", err))
return
}
if _, ok := h.Finalizers[request.Purpose]; !ok {
api.RespondError(c, http.StatusBadRequest, "unsupported upload purpose")
return
}
session, err := h.Store.Init(request.Purpose, request.Filename, request.Size)
if err != nil {
api.RespondError(c, http.StatusBadRequest, err.Error())
return
}
api.RespondSuccess(c, gin.H{
"upload_id": session.ID,
"chunk_size": ChunkSize,
})
}
func (h *Handler) Chunk(c *gin.Context) {
c.Request.Body = http.MaxBytesReader(c.Writer, c.Request.Body, maxChunkRequestSize)
uploadID := c.PostForm("upload_id")
index, err := strconv.ParseInt(c.PostForm("chunk_index"), 10, 64)
if err != nil {
api.RespondError(c, http.StatusBadRequest, "chunk_index must be an integer")
return
}
chunk, _, err := c.Request.FormFile("chunk_data")
if err != nil {
api.RespondError(c, http.StatusBadRequest, fmt.Sprintf("get chunk data: %v", err))
return
}
defer chunk.Close()
if err := h.Store.SaveChunk(uploadID, index, chunk); err != nil {
h.respondUploadError(c, err)
return
}
api.RespondSuccess(c, gin.H{"received": true, "chunk_index": index})
}
func (h *Handler) Merge(c *gin.Context) {
var request struct {
UploadID string `json:"upload_id" binding:"required"`
}
if err := c.ShouldBindJSON(&request); err != nil {
api.RespondError(c, http.StatusBadRequest, fmt.Sprintf("invalid request: %v", err))
return
}
session, err := h.Store.Merge(request.UploadID)
if err != nil {
_ = h.Store.Cancel(request.UploadID)
h.respondUploadError(c, err)
return
}
defer h.Store.Cancel(session.ID)
finalize, ok := h.Finalizers[session.Metadata.Purpose]
if !ok {
api.RespondError(c, http.StatusBadRequest, "unsupported upload purpose")
return
}
result, err := finalize(session)
if err != nil {
api.RespondError(c, http.StatusBadRequest, err.Error())
return
}
api.RespondSuccessMessage(c, result.Message, result.Data)
}
func (h *Handler) Cancel(c *gin.Context) {
var request struct {
UploadID string `json:"upload_id" binding:"required"`
}
if err := c.ShouldBindJSON(&request); err != nil {
api.RespondError(c, http.StatusBadRequest, fmt.Sprintf("invalid request: %v", err))
return
}
if err := h.Store.Cancel(request.UploadID); err != nil {
api.RespondError(c, http.StatusBadRequest, err.Error())
return
}
api.RespondSuccess(c, gin.H{})
}
func (h *Handler) respondUploadError(c *gin.Context, err error) {
if errors.Is(err, ErrNotFound) {
api.RespondError(c, http.StatusNotFound, "upload not found or expired")
return
}
api.RespondError(c, http.StatusBadRequest, err.Error())
}