mirror of
https://github.com/komari-monitor/komari.git
synced 2026-11-11 08:42:39 +00:00
Compare commits
59
Commits
1.4.0
...
1.6.0-pre1
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
39cb3f59b2 | ||
|
|
9812acfd11 | ||
|
|
70df32096f | ||
|
|
29d309f4a1 | ||
|
|
7d692d2faf | ||
|
|
68c284d043 | ||
|
|
14bc87d1e9 | ||
|
|
154d2b8af2 | ||
|
|
2c3f2281fa | ||
|
|
7e91794f24 | ||
|
|
f57461f87f | ||
|
|
d15f0512c7 | ||
|
|
4ec22edf1f | ||
|
|
0ca87aafd1 | ||
|
|
f18ad72c17 | ||
|
|
8e4a3f8709 | ||
|
|
80add06a12 | ||
|
|
1333d63ce0 | ||
|
|
44ed3035f1 | ||
|
|
7ec36157d5 | ||
|
|
360fd5f50b | ||
|
|
b11ffd3aa7 | ||
|
|
d7d6135c49 | ||
|
|
ac037f2177 | ||
|
|
5405bdf377 | ||
|
|
f754f01f53 | ||
|
|
434ba108e1 | ||
|
|
24254749b5 | ||
|
|
c6b02af000 | ||
|
|
b28ac9d42c | ||
|
|
2ff7c11db0 | ||
|
|
46fd2af44c | ||
|
|
4d2eb70c62 | ||
|
|
7a6692b79a | ||
|
|
63d78b6ed4 | ||
|
|
2319ea77f6 | ||
|
|
0c39a52e0b | ||
|
|
e31a032d89 | ||
|
|
da4d5187c1 | ||
|
|
bf6b45ec3a | ||
|
|
4a3400bee6 | ||
|
|
4caa067556 | ||
|
|
2f3959adde | ||
|
|
63a7ffddcf | ||
|
|
92d679ffdb | ||
|
|
7c30fc4dc9 | ||
|
|
773a3715dc | ||
|
|
75d7b6c7a3 | ||
|
|
c294c17642 | ||
|
|
2883bf6295 | ||
|
|
14bf8bde7c | ||
|
|
f043f2890c | ||
|
|
057108a2f3 | ||
|
|
5879d4b3da | ||
|
|
16769d5687 | ||
|
|
5ea32c48b6 | ||
|
|
a06d922d78 | ||
|
|
0ed14fc73d | ||
|
|
61bd0c8bdd |
No files matched your search
@@ -23,6 +23,15 @@ runs:
|
||||
with:
|
||||
node-version: ${{ inputs.node-version }}
|
||||
|
||||
- name: Install zstd
|
||||
shell: bash
|
||||
run: |
|
||||
set -euxo pipefail
|
||||
if ! command -v zstd >/dev/null 2>&1; then
|
||||
sudo apt-get update
|
||||
sudo apt-get install -y zstd
|
||||
fi
|
||||
|
||||
- name: Clone and build frontend
|
||||
shell: bash
|
||||
run: |
|
||||
@@ -38,9 +47,12 @@ runs:
|
||||
npm run build
|
||||
cd ..
|
||||
|
||||
mkdir -p web/public/defaultTheme/dist
|
||||
rm -rf web/public/defaultTheme/dist/*
|
||||
cp -r komari-web/dist/* web/public/defaultTheme/dist/
|
||||
mkdir -p web/public/defaultTheme
|
||||
rm -rf web/public/defaultTheme/dist web/public/defaultTheme/dist.tar.zst
|
||||
test -f komari-web/dist/index.html
|
||||
tar -cf "$RUNNER_TEMP/komari-default-dist.tar" -C komari-web/dist .
|
||||
zstd -19 -T0 -q -f "$RUNNER_TEMP/komari-default-dist.tar" -o web/public/defaultTheme/dist.tar.zst
|
||||
rm -f "$RUNNER_TEMP/komari-default-dist.tar"
|
||||
cp -f komari-web/komari-theme.json web/public/defaultTheme/
|
||||
if [ -f komari-web/preview.png ]; then cp -f komari-web/preview.png web/public/defaultTheme/; fi
|
||||
if [ -f komari-web/perview.png ]; then cp -f komari-web/perview.png web/public/defaultTheme/; fi
|
||||
|
||||
@@ -22,3 +22,4 @@ runs:
|
||||
mkdir -p web/public/defaultTheme
|
||||
rm -rf web/public/defaultTheme/*
|
||||
tar -xzf "$RUNNER_TEMP/frontend-artifact/${{ inputs.artifact-name }}.tar.gz" -C web/public/defaultTheme
|
||||
test -s web/public/defaultTheme/dist.tar.zst
|
||||
@@ -98,7 +98,11 @@ jobs:
|
||||
fi
|
||||
VERSION="${{ github.sha }}"
|
||||
VERSION_HASH="${{ github.sha }}"
|
||||
go build -trimpath -ldflags="-s -w -X github.com/komari-monitor/komari/utils.CurrentVersion=${VERSION} -X github.com/komari-monitor/komari/utils.VersionHash=${VERSION_HASH}" -o $BINARY_NAME
|
||||
LDFLAGS="-s -w -X github.com/komari-monitor/komari/utils.CurrentVersion=${VERSION} -X github.com/komari-monitor/komari/utils.VersionHash=${VERSION_HASH}"
|
||||
if [ "${{ matrix.goos }}" = "linux" ]; then
|
||||
LDFLAGS="$LDFLAGS -extldflags=-Wl,--strip-all"
|
||||
fi
|
||||
go build -trimpath -ldflags="$LDFLAGS" -o "$BINARY_NAME"
|
||||
|
||||
- name: Upload binary as artifact
|
||||
uses: actions/upload-artifact@v4
|
||||
|
||||
@@ -14,11 +14,6 @@ jobs:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v4
|
||||
|
||||
#- name: Create dummy index.html
|
||||
# run: |
|
||||
# mkdir -p web/public/defaultTheme/dist
|
||||
# echo "<html><body><h1>This is komari development server.</h1></body></html>" > web/public/defaultTheme/dist/index.html
|
||||
|
||||
- name: Clone and build frontend
|
||||
uses: ./.github/actions/build-frontend
|
||||
|
||||
@@ -33,7 +28,7 @@ jobs:
|
||||
GOARCH: amd64
|
||||
run: |
|
||||
VERSION_SHA="${{ github.sha }}"
|
||||
go build -ldflags "-X github.com/komari-monitor/komari/utils.CurrentVersion=dev -X github.com/komari-monitor/komari/utils.VersionHash=${VERSION_SHA}" -o komari .
|
||||
go build -ldflags "-s -w -extldflags=-Wl,--strip-all -X github.com/komari-monitor/komari/utils.CurrentVersion=dev -X github.com/komari-monitor/komari/utils.VersionHash=${VERSION_SHA}" -o komari .
|
||||
|
||||
- name: Upload binary as artifact
|
||||
uses: actions/upload-artifact@v4
|
||||
|
||||
@@ -50,7 +50,7 @@ jobs:
|
||||
run: |
|
||||
VERSION="${{ github.ref_name }}"
|
||||
VERSION_HASH="${{ github.sha }}"
|
||||
LDFLAGS="-s -w -X github.com/komari-monitor/komari/utils.CurrentVersion=${VERSION} -X github.com/komari-monitor/komari/utils.VersionHash=${VERSION_HASH}"
|
||||
LDFLAGS="-s -w -extldflags=-Wl,--strip-all -X github.com/komari-monitor/komari/utils.CurrentVersion=${VERSION} -X github.com/komari-monitor/komari/utils.VersionHash=${VERSION_HASH}"
|
||||
|
||||
echo "Building for linux/amd64..."
|
||||
GOARCH=amd64 CC="zig cc -target x86_64-linux-musl" go build -trimpath -ldflags="$LDFLAGS" -o komari-linux-amd64
|
||||
|
||||
@@ -102,7 +102,11 @@ jobs:
|
||||
run: |
|
||||
BINARY_NAME=komari-${{ matrix.goos }}-${{ matrix.goarch }}${{ matrix.binary_extension }}
|
||||
VERSION_HASH="$(git rev-parse HEAD)"
|
||||
go build -trimpath -ldflags="-s -w -X github.com/komari-monitor/komari/utils.CurrentVersion=${RELEASE_TAG} -X github.com/komari-monitor/komari/utils.VersionHash=${VERSION_HASH}" -o "$BINARY_NAME"
|
||||
LDFLAGS="-s -w -X github.com/komari-monitor/komari/utils.CurrentVersion=${RELEASE_TAG} -X github.com/komari-monitor/komari/utils.VersionHash=${VERSION_HASH}"
|
||||
if [ "${{ matrix.goos }}" = "linux" ]; then
|
||||
LDFLAGS="$LDFLAGS -extldflags=-Wl,--strip-all"
|
||||
fi
|
||||
go build -trimpath -ldflags="$LDFLAGS" -o "$BINARY_NAME"
|
||||
|
||||
- name: Upload binary to release
|
||||
env:
|
||||
|
||||
@@ -56,7 +56,7 @@ jobs:
|
||||
VERSION: ${{ github.event.inputs.version }}
|
||||
run: |
|
||||
VERSION_HASH="${{ github.sha }}"
|
||||
LDFLAGS="-s -w -X github.com/komari-monitor/komari/utils.CurrentVersion=${VERSION} -X github.com/komari-monitor/komari/utils.VersionHash=${VERSION_HASH}"
|
||||
LDFLAGS="-s -w -extldflags=-Wl,--strip-all -X github.com/komari-monitor/komari/utils.CurrentVersion=${VERSION} -X github.com/komari-monitor/komari/utils.VersionHash=${VERSION_HASH}"
|
||||
|
||||
echo "Building for linux/amd64..."
|
||||
GOARCH=amd64 CC="zig cc -target x86_64-linux-musl" go build -trimpath -ldflags="$LDFLAGS" -o komari-linux-amd64
|
||||
|
||||
@@ -121,7 +121,11 @@ jobs:
|
||||
fi
|
||||
VERSION="${{ github.event.release.tag_name }}"
|
||||
VERSION_HASH="${{ github.sha }}" # Use commit SHA for hash
|
||||
go build -trimpath -ldflags="-s -w -X github.com/komari-monitor/komari/utils.CurrentVersion=${VERSION} -X github.com/komari-monitor/komari/utils.VersionHash=${VERSION_HASH}" -o $BINARY_NAME
|
||||
LDFLAGS="-s -w -X github.com/komari-monitor/komari/utils.CurrentVersion=${VERSION} -X github.com/komari-monitor/komari/utils.VersionHash=${VERSION_HASH}"
|
||||
if [ "${{ matrix.goos }}" = "linux" ]; then
|
||||
LDFLAGS="$LDFLAGS -extldflags=-Wl,--strip-all"
|
||||
fi
|
||||
go build -trimpath -ldflags="$LDFLAGS" -o "$BINARY_NAME"
|
||||
|
||||
- name: Upload binary as artifact
|
||||
uses: actions/upload-artifact@v4
|
||||
|
||||
@@ -150,7 +150,11 @@ jobs:
|
||||
fi
|
||||
VERSION="${{ needs.prepare.outputs.version }}"
|
||||
VERSION_HASH="${{ github.sha }}" # Use commit SHA for hash
|
||||
go build -trimpath -ldflags="-s -w -X github.com/komari-monitor/komari/utils.CurrentVersion=${VERSION} -X github.com/komari-monitor/komari/utils.VersionHash=${VERSION_HASH}" -o $BINARY_NAME
|
||||
LDFLAGS="-s -w -X github.com/komari-monitor/komari/utils.CurrentVersion=${VERSION} -X github.com/komari-monitor/komari/utils.VersionHash=${VERSION_HASH}"
|
||||
if [ "${{ matrix.goos }}" = "linux" ]; then
|
||||
LDFLAGS="$LDFLAGS -extldflags=-Wl,--strip-all"
|
||||
fi
|
||||
go build -trimpath -ldflags="$LDFLAGS" -o "$BINARY_NAME"
|
||||
|
||||
- name: Upload binary as artifact
|
||||
uses: actions/upload-artifact@v4
|
||||
|
||||
@@ -15,6 +15,7 @@ komari.db
|
||||
/utils/geoip/data
|
||||
.vscode/
|
||||
/backup
|
||||
/cache
|
||||
|
||||
# Test binary, built with `go test -c`
|
||||
*.test
|
||||
@@ -36,3 +37,5 @@ go.work.sum
|
||||
komari
|
||||
/dist/
|
||||
.codex
|
||||
|
||||
/web/filemanager/data
|
||||
+2
-4
@@ -8,13 +8,11 @@ ARG TARGETARCH
|
||||
|
||||
RUN apk add --no-cache ca-certificates curl tzdata
|
||||
|
||||
COPY komari-${TARGETOS}-${TARGETARCH} /app/komari
|
||||
|
||||
|
||||
RUN chmod +x /app/komari
|
||||
COPY --chmod=755 komari-${TARGETOS}-${TARGETARCH} /app/komari
|
||||
|
||||
ENV GIN_MODE=release
|
||||
ENV KOMARI_LISTEN=0.0.0.0:25774
|
||||
ENV GODEBUG=disablethp=1
|
||||
|
||||
EXPOSE 25774
|
||||
|
||||
|
||||
@@ -5,43 +5,63 @@
|
||||
|
||||

|
||||
|
||||
[简体中文](./README_zh-cn.md)
|
||||
[English](./README.md) | [简体中文](./README_zh-cn.md)
|
||||
|
||||
Komari is a lightweight, self-hosted server monitoring tool designed to provide a simple and efficient solution for monitoring server performance. It supports viewing server status through a web interface and collects data through a lightweight agent.
|
||||
Komari is a lightweight, self-hosted server monitoring solution. It provides a simple and efficient way to track server performance through a web interface, with metrics collected by a lightweight agent.
|
||||
|
||||
> [!WARNING]
|
||||
> Komari is a self-hosted monitoring and control program intended only for systems you own or are authorized to administer. Do not weaponize Komari, deploy it without consent, or use it for unauthorized access, persistence, command execution, or other abusive activity. For context on real-world abuse risks, see Huntress' analysis: [Komari C2 agent abuse](https://www.huntress.com/blog/komari-c2-agent-abuse).
|
||||
> Users are solely responsible for how they deploy and operate Komari. The developers do not accept responsibility for unauthorized or abusive use, or for any resulting consequences.
|
||||
> On Windows, when remote control is enabled, the client displays a Windows notification at each user login to remind the user that Komari is remote control software.
|
||||
> Komari is a self-hosted monitoring and control application. Deploy it only on systems you own or are authorized to manage. You are solely responsible for how you deploy and use Komari. The developers accept no liability for unauthorized access, persistence, command execution, other misuse, or any resulting consequences.
|
||||
|
||||
[文档/Documentation](https://komari-document.pages.dev/) | [文档(镜像站 By Geekertao)](https://www.komari.wiki) | [Telegram Group](https://t.me/komari_monitor)
|
||||
[Documentation](https://www.komari.wiki/) | [Telegram Group](https://t.me/komari_monitor)
|
||||
|
||||
## Features
|
||||
|
||||
- **Lightweight and Efficient**: Low resource consumption, suitable for servers of all sizes.
|
||||
- **Self-hosted**: Complete control over data privacy, easy to deploy.
|
||||
- **Web Interface**: Intuitive monitoring dashboard, easy to use.
|
||||
- **Real-time monitoring**: Displays monitoring data at one-second intervals.
|
||||
- **Lightweight and efficient**: Uses minimal system resources and works well on servers of any size.
|
||||
- **Self-hosted**: Keeps you in control of your data and privacy.
|
||||
- **Web interface**: Provides an intuitive, easy-to-use monitoring dashboard.
|
||||
- **Extensible**: Supports custom themes and plugins.
|
||||
|
||||
## One-click Deployment
|
||||
## Quick Start
|
||||
|
||||
- Rainyun - CNY 4.5/month
|
||||
| Platform | Description |
|
||||
| --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
|
||||
| <a href="https://app.rainyun.com/apps/rca/store/6780/NzYxNzAz_"><img src="https://rainyun-apps.cn-nb1.rains3.com/materials/deploy-on-rainyun-cn.svg" alt="Rainyun" width="180"></a> | Deploy websites, databases, and hundreds of popular apps in seconds with flexible hourly billing. [Get started for just ¥5/month. Deploy now!](https://app.rainyun.com/apps/rca/store/6780/NzYxNzAz_) |
|
||||
| <a href="https://apps.fit2cloud.com/1panel/komari"><img src="https://raw.githubusercontent.com/komari-monitor/public/refs/heads/main/images/1panel-logo-blue.png" alt="1Panel App Store" width="180"></a> | A modern, open-source Linux server management panel for websites, databases, containers, files, backups, security, and AI, with one-click deployment from its app store. |
|
||||
|
||||
[](https://app.rainyun.com/apps/rca/store/6780/NzYxNzAz_)
|
||||
For instructions on Docker deployment, binary installation, building from source, and updates, see the [installation guide](https://www.komari.wiki/en/install/quick-start).
|
||||
|
||||
- 1Panel App Store
|
||||
## Screenshots
|
||||
|
||||
Available on 1Panel App Store. Install via **App Store > Utilities > Komari**.
|
||||
| Page | Screenshot |
|
||||
| ------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
|
||||
| Home Dashboard | <img src="https://b2.akz.moe/awesome-pictures/komari-screenshot/%E4%B8%BB%E9%A1%B5%E4%BB%AA%E8%A1%A8%E7%9B%98-en.webp" width="800" alt="Home Dashboard"> |
|
||||
| Admin Dashboard | <img src="https://b2.akz.moe/awesome-pictures/komari-screenshot/%E5%90%8E%E5%8F%B0%E4%BB%AA%E8%A1%A8%E7%9B%98-en.webp" width="800" alt="Admin Dashboard"> |
|
||||
| History Charts | <img src="https://b2.akz.moe/awesome-pictures/komari-screenshot/%E5%8E%86%E5%8F%B2%E5%9B%BE%E8%A1%A8-en.webp" width="800" alt="History Charts"> |
|
||||
| Web Terminal | <img src="https://b2.akz.moe/awesome-pictures/komari-screenshot/%E7%BD%91%E9%A1%B5%E7%BB%88%E7%AB%AF.webp" width="800" alt="Web Terminal"> |
|
||||
| Customizable Themes | <img src="https://b2.akz.moe/awesome-pictures/komari-screenshot/%E4%B8%BB%E9%A2%98%E5%8F%AF%E8%87%AA%E5%AE%9A%E4%B9%89-en.webp" width="800" alt="Customizable Themes"> |
|
||||
| Theme Market | <img src="https://b2.akz.moe/awesome-pictures/komari-screenshot/%E4%B8%BB%E9%A2%98%E5%B8%82%E5%9C%BA-en.webp" width="800" alt="Theme Market"> |
|
||||
|
||||
For Docker, binary, source-build, and update instructions, see the [installation guide](https://komari-document.pages.dev/en/install/quick-start).
|
||||
## Sponsors
|
||||
|
||||
## Contributing
|
||||
Interested in sponsoring Komari? Contact the developer via [email](mailto:komari@akz.moe) or [Telegram](https://t.me/mamomoe).
|
||||
|
||||
Issues and Pull Requests are welcome!
|
||||
| Sponsor | Description |
|
||||
| ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
|
||||
| <a href="https://axisnow.io/zh?utm=komari"><img src="https://raw.githubusercontent.com/komari-monitor/public/refs/heads/main/images/AxisNow.jpg" alt="AxisNow" width="180"></a> | [Self-Hosted Private CDN \| Subscription-Based CDN-Like Service \| A Fully Controlled, Flexible, Modular CDN Network](https://axisnow.io/zh?utm=komari) |
|
||||
| <a href="https://whmcs.as211392.com/aff.php?aff=110"><img src="https://raw.githubusercontent.com/komari-monitor/public/refs/heads/main/images/dreamcloud.png" alt="Dream Cloud" width="180"></a> | Cost-effective Asia-Pacific hosting with direct connectivity and robust DDoS protection, backed by transparent capacity claims. |
|
||||
| <a href="https://sharon.io"><img src="https://raw.githubusercontent.com/komari-monitor/public/refs/heads/main/images/sharon-networks.webp" alt="Sharon Networks" width="180"></a> | Premium China-optimized connectivity from Asia-Pacific data centers, featuring low latency, high bandwidth, and Tbps-scale local DDoS mitigation. Join the [Telegram community](https://t.me/SharonNetwork) to participate in charitable initiatives and community giveaways. |
|
||||
|
||||
## Acknowledgements
|
||||
## Contributors
|
||||
|
||||
| Acknowledgement | Details |
|
||||
| ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
|
||||
| <a href="https://whmcs.as211392.com/aff.php?aff=110"><img src="https://raw.githubusercontent.com/komari-monitor/public/refs/heads/main/images/dreamcloud.png" alt="Dream Cloud" width="180"></a> | 极高性价比解锁直连亚太高防,真高防,不虚标,打死退款 |
|
||||
| <a href="https://sharon.io"><img src="https://raw.githubusercontent.com/komari-monitor/public/refs/heads/main/images/sharon-networks.webp" alt="Sharon Networks" width="180"></a> | 亚太数据中心提供顶级的中国优化网络接入 · 低延时&高带宽&提供Tbps级本地清洗高防服务, 为您的业务保驾护航, 为您的客户提供极致体验. 加入社区 [Telegram群组](https://t.me/SharonNetwork) 可参与公益募捐或群内抽奖免费使用 |
|
||||
| Open-source community | Everyone who has submitted pull requests or created themes. |
|
||||
Thanks to everyone who has contributed code, themes, plugins, documentation, translations, bug reports, or feedback to Komari.
|
||||
|
||||
<a href="https://github.com/komari-monitor/komari/graphs/contributors"><img src="https://contributors-img.web.app/image?repo=komari-monitor/komari" alt="Komari contributors" width="600"></a>
|
||||
|
||||
## Support the Project
|
||||
|
||||
If Komari has been useful to you, consider buying me a coffee. Thank you for your support!
|
||||
|
||||
| WeChat Pay | TRON Network |
|
||||
| ------------------------------------------------------------------------------------------------------------ | ------------------------------------------------------------------------------------------- |
|
||||
|  |  |
|
||||
+36
-18
@@ -5,45 +5,63 @@
|
||||
|
||||

|
||||
|
||||
[简体中文](./README_zh-cn.md)
|
||||
[English](./README.md) | [简体中文](./README_zh-cn.md)
|
||||
|
||||
Komari 是一款轻量级的自托管服务器监控工具,旨在提供简单、高效的服务器性能监控解决方案。它支持通过 Web 界面查看服务器状态,并通过轻量级 Agent 收集数据。
|
||||
|
||||
> [!WARNING]
|
||||
> Komari 是一款自托管的监控/控制程序,仅应部署在你拥有或已获得授权管理的系统上。请勿将 Komari 武器化,或在未获授权的情况下部署、访问、持久化、执行命令及从事其他滥用行为。关于现实中的滥用风险,可参考 Huntress 的分析:[Komari C2 agent abuse](https://www.huntress.com/blog/komari-c2-agent-abuse)。
|
||||
> 用户需要自行承担部署和使用 Komari 的责任。开发者不对未经授权或滥用行为及其后果承担责任。
|
||||
> 在 Windows 端开启远程控制后,客户端会在每次用户登录时通过 Windows 通知提醒用户 Komari 是一款远程控制软件。
|
||||
> Komari 是一款自托管的监控/控制程序,仅应部署在你拥有或已获得授权管理的系统上。在未获授权的情况下部署、访问、持久化、执行命令及从事其他滥用行为,用户需要自行承担部署和使用 Komari 的责任。开发者不对未经授权或滥用行为及其后果承担责任。
|
||||
|
||||
[文档/Documentation](https://www.komari.wiki/) | [Telegram 群](https://t.me/komari_monitor)
|
||||
[文档](https://www.komari.wiki/) | [Telegram 群](https://t.me/komari_monitor)
|
||||
|
||||
## 特性
|
||||
|
||||
- **实时监控**: 秒级实时数据展示。
|
||||
- **轻量高效**:低资源占用,适合各种规模的服务器。
|
||||
- **自托管**:完全掌控数据隐私,部署简单。
|
||||
- **Web 界面**:直观的监控仪表盘,易于使用。
|
||||
- **极强的可扩展性**: 支持自定义主题和插件。
|
||||
|
||||
## 快速开始
|
||||
|
||||
### 0. 容器云一键部署
|
||||
|
||||
- 雨云云应用 - CNY 4.5/月
|
||||
|
||||
[](https://app.rainyun.com/apps/rca/store/6780/NzYxNzAz_)
|
||||
|
||||
- 1Panel 应用商店
|
||||
|
||||
已上架1Panel应用商店,应用商店-实用工具-Komari 即可安装
|
||||
| 平台 | 介绍 |
|
||||
| -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------- |
|
||||
| <a href="https://app.rainyun.com/apps/rca/store/6780/NzYxNzAz_"><img src="https://rainyun-apps.cn-nb1.rains3.com/materials/deploy-on-rainyun-cn.svg" alt="Rainyun" width="180"></a> | 秒级部署网站、数据库及数百款热门 App,并采用按小时灵活计费。[每月5元,立即部署](https://app.rainyun.com/apps/rca/store/6780/NzYxNzAz_) |
|
||||
| <a href="https://apps.fit2cloud.com/1panel/komari"><img src="https://raw.githubusercontent.com/komari-monitor/public/refs/heads/main/images/1panel-logo-blue.png" alt="1Panel Appstore" width="180"></a> | 现代化、开源的Linux 服务器运维管理面板,提供网站、数据库、容器、文件、备份、安全与AI 管理能力,支持应用商店一键部署。 |
|
||||
|
||||
Docker、二进制文件、源码构建和更新说明,请参阅 [安装指南](https://www.komari.wiki/install/quick-start).
|
||||
|
||||
## 贡献
|
||||
## 截图
|
||||
|
||||
欢迎提交 Issue 或 Pull Request!
|
||||
| 页面 | 截图 |
|
||||
| ------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------ |
|
||||
| 主页仪表盘 | <img src="https://b2.akz.moe/awesome-pictures/komari-screenshot/%E4%B8%BB%E9%A1%B5%E4%BB%AA%E8%A1%A8%E7%9B%98.webp" width="800" alt="主页仪表盘"> |
|
||||
| 后台仪表盘 | <img src="https://b2.akz.moe/awesome-pictures/komari-screenshot/%E5%90%8E%E5%8F%B0%E4%BB%AA%E8%A1%A8%E7%9B%98.webp" width="800" alt="后台仪表盘"> |
|
||||
| 历史图表 | <img src="https://b2.akz.moe/awesome-pictures/komari-screenshot/%E5%8E%86%E5%8F%B2%E5%9B%BE%E8%A1%A8.webp" width="800" alt="历史图表"> |
|
||||
| 网页终端 | <img src="https://b2.akz.moe/awesome-pictures/komari-screenshot/%E7%BD%91%E9%A1%B5%E7%BB%88%E7%AB%AF.webp" width="800" alt="网页终端"> |
|
||||
| 主题可自定义 | <img src="https://b2.akz.moe/awesome-pictures/komari-screenshot/%E4%B8%BB%E9%A2%98%E5%8F%AF%E8%87%AA%E5%AE%9A%E4%B9%89.webp" width="800" alt="主题可自定义"> |
|
||||
| 主题市场 | <img src="https://b2.akz.moe/awesome-pictures/komari-screenshot/%E4%B8%BB%E9%A2%98%E5%B8%82%E5%9C%BA.webp" width="800" alt="主题市场"> |
|
||||
|
||||
## 鸣谢
|
||||
## 赞助商
|
||||
|
||||
有意赞助 Komari?请通过 [电子邮箱](mailto:komari@akz.moe) 或 [Telegram](https://t.me/mamomoe) 联系开发者。
|
||||
|
||||
| 赞助商 | 描述 |
|
||||
| ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
|
||||
| <a href="https://axisnow.io/zh?utm=komari"><img src="https://raw.githubusercontent.com/komari-monitor/public/refs/heads/main/images/AxisNow.jpg" alt="AxisNow" width="180"></a> | [自建私有部署CDN \| 订阅式高仿CDN \| 自主可控、灵活组合的CDN网络](https://axisnow.io/zh?utm=komari) |
|
||||
| <a href="https://whmcs.as211392.com/aff.php?aff=110"><img src="https://raw.githubusercontent.com/komari-monitor/public/refs/heads/main/images/dreamcloud.png" alt="Dream Cloud" width="180"></a> | 极高性价比解锁直连亚太高防,真高防,不虚标,打死退款 |
|
||||
| <a href="https://sharon.io"><img src="https://raw.githubusercontent.com/komari-monitor/public/refs/heads/main/images/sharon-networks.webp" alt="Sharon Networks" width="180"></a> | 亚太数据中心提供顶级的中国优化网络接入 · 低延时&高带宽&提供Tbps级本地清洗高防服务, 为您的业务保驾护航, 为您的客户提供极致体验. 加入社区 [Telegram群组](https://t.me/SharonNetwork) 可参与公益募捐或群内抽奖免费使用 |
|
||||
| 开源社区 | 提交 PR、制作主题的各位开发者 |
|
||||
|
||||
## 贡献者
|
||||
|
||||
感谢所有为 Komari 贡献代码、主题、插件、文档、翻译、问题报告或反馈的朋友。
|
||||
|
||||
<a href="https://github.com/komari-monitor/komari/graphs/contributors"><img src="https://contributors-img.web.app/image?repo=komari-monitor/komari" alt="Komari 贡献者" width="600"></a>
|
||||
|
||||
## 支持项目
|
||||
|
||||
如果 Komari 对你有所帮助,欢迎请作者喝一杯奶茶。感谢你的支持!
|
||||
|
||||
| 微信赞赏码 | TRON Network |
|
||||
| ------------------------------------------------------------------------------------------------ | --------------------------------------------------------------------------- |
|
||||
|  |  |
|
||||
@@ -6,6 +6,7 @@ import (
|
||||
|
||||
appserver "github.com/komari-monitor/komari/internal/server"
|
||||
logger "github.com/komari-monitor/komari/utils/log"
|
||||
frontendpublic "github.com/komari-monitor/komari/web/public"
|
||||
|
||||
"github.com/komari-monitor/komari/cmd/flags"
|
||||
"github.com/spf13/cobra"
|
||||
@@ -32,6 +33,10 @@ func init() {
|
||||
// 具体各阶段的职责与顺序见 internal/server.App。这里只负责串联:
|
||||
// 任一初始化阶段失败即中止启动,避免在半初始化状态下对外提供服务。
|
||||
func RunServer() {
|
||||
if err := frontendpublic.PrepareDefaultDist(); err != nil {
|
||||
logger.Fatalf("server", "prepare default frontend failed: %v", err)
|
||||
}
|
||||
|
||||
app := appserver.New(appserver.Options{ListenAddr: flags.Listen})
|
||||
if err := app.Bootstrap(); err != nil {
|
||||
_ = app.Shutdown()
|
||||
|
||||
@@ -6,7 +6,7 @@ import (
|
||||
|
||||
"github.com/komari-monitor/komari/database/dbcore"
|
||||
"github.com/komari-monitor/komari/database/models"
|
||||
v1 "github.com/komari-monitor/komari/protocol/v1"
|
||||
v2 "github.com/komari-monitor/komari/protocol/v2"
|
||||
)
|
||||
|
||||
func GetClientUUIDByToken(token string) (clientUUID string, err error) {
|
||||
@@ -20,7 +20,7 @@ func GetClientUUIDByToken(token string) (clientUUID string, err error) {
|
||||
}
|
||||
|
||||
// 检查数据防止异常数据导致数据库损坏
|
||||
func ReportVerify(report v1.Report) error {
|
||||
func ReportVerify(report v2.Report) error {
|
||||
// 防止输入不合理范围
|
||||
if report.CPU.Usage < 0 || report.CPU.Usage > 100 {
|
||||
return fmt.Errorf("CPU.Usage must be between 0 and 100")
|
||||
|
||||
@@ -1,64 +0,0 @@
|
||||
package clipboard
|
||||
|
||||
import (
|
||||
"time"
|
||||
|
||||
"github.com/komari-monitor/komari/database/dbcore"
|
||||
"github.com/komari-monitor/komari/database/models"
|
||||
)
|
||||
|
||||
// CreateClipboard 创建剪贴板记录
|
||||
func CreateClipboard(cb *models.Clipboard) error {
|
||||
db := dbcore.GetDBInstance()
|
||||
cb.CreatedAt = time.Now().UTC()
|
||||
cb.UpdatedAt = cb.CreatedAt
|
||||
return db.Create(cb).Error
|
||||
}
|
||||
|
||||
// GetClipboardByID 根据ID获取剪贴板记录
|
||||
func GetClipboardByID(id int) (*models.Clipboard, error) {
|
||||
var cb models.Clipboard
|
||||
db := dbcore.GetDBInstance()
|
||||
if err := db.First(&cb, id).Error; err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &cb, nil
|
||||
}
|
||||
|
||||
// UpdateClipboardFields 更新剪贴板记录
|
||||
func UpdateClipboardFields(id int, fields map[string]interface{}) error {
|
||||
db := dbcore.GetDBInstance()
|
||||
delete(fields, "created_at")
|
||||
fields["updated_at"] = time.Now().UTC()
|
||||
return db.Model(&models.Clipboard{}).Where("id = ?", id).Updates(fields).Error
|
||||
}
|
||||
|
||||
// DeleteClipboard 删除剪贴板记录
|
||||
func DeleteClipboard(id int) error {
|
||||
db := dbcore.GetDBInstance()
|
||||
// Check if record exists first
|
||||
var cb models.Clipboard
|
||||
if err := db.First(&cb, id).Error; err != nil {
|
||||
return err // Record not found or other error
|
||||
}
|
||||
return db.Delete(&cb).Error
|
||||
}
|
||||
|
||||
// DeleteClipboardBatch 批量删除剪贴板记录
|
||||
func DeleteClipboardBatch(ids []int) error {
|
||||
if len(ids) == 0 {
|
||||
return nil
|
||||
}
|
||||
db := dbcore.GetDBInstance()
|
||||
return db.Where("id IN ?", ids).Delete(&models.Clipboard{}).Error
|
||||
}
|
||||
|
||||
// ListClipboard 列出所有剪贴板记录
|
||||
func ListClipboard() ([]models.Clipboard, error) {
|
||||
var list []models.Clipboard
|
||||
db := dbcore.GetDBInstance()
|
||||
if err := db.Find(&list).Error; err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return list, nil
|
||||
}
|
||||
@@ -444,15 +444,13 @@ func doInitialize() error {
|
||||
// 若升级时旧表仍存在,管理员可通过升级向导显式导入并清理。
|
||||
// models.Record / models.PingRecord / models.GPURecord 结构体仍作为
|
||||
// metric store 的读写 DTO 和旧表导入 DTO 保留在 models 包中。
|
||||
// models.TrafficReportNotification 同样仅为旧数据库兼容保留,不再建表。
|
||||
|
||||
err = instance.AutoMigrate(
|
||||
&models.User{},
|
||||
&models.Client{},
|
||||
&models.Log{},
|
||||
&models.Clipboard{},
|
||||
&models.LoadNotification{},
|
||||
&models.OfflineNotification{},
|
||||
&models.TrafficReportNotification{},
|
||||
&models.PingTask{},
|
||||
&models.OidcProvider{},
|
||||
&models.MessageSenderProvider{},
|
||||
|
||||
@@ -1,13 +0,0 @@
|
||||
package models
|
||||
|
||||
import "time"
|
||||
|
||||
type Clipboard struct {
|
||||
Id int `json:"id" gorm:"primaryKey;autoIncrement;unique"`
|
||||
Text string `json:"text" gorm:"type:longtext"`
|
||||
Name string `json:"name" gorm:"type:varchar(255)"`
|
||||
Weight int `json:"weight" gorm:"type:int"`
|
||||
Remark string `json:"remark" gorm:"type:text"`
|
||||
CreatedAt time.Time `json:"created_at"`
|
||||
UpdatedAt time.Time `json:"updated_at"`
|
||||
}
|
||||
@@ -8,7 +8,4 @@ const (
|
||||
Login = "Login"
|
||||
Alert = "Alert"
|
||||
Traffic = "Traffic"
|
||||
DReport = "DReport" // 日报
|
||||
WReport = "WReport" // 周报
|
||||
MReport = "MReport" // 月报
|
||||
)
|
||||
@@ -12,18 +12,6 @@ type OfflineNotification struct {
|
||||
LastNotified *time.Time `json:"last_notified"` // 上次通知时间
|
||||
}
|
||||
|
||||
// LoadNotification 定义了基于资源占用达标时间比的负载通知规则
|
||||
type LoadNotification struct {
|
||||
Id uint `json:"id,omitempty" gorm:"primaryKey;autoIncrement"`
|
||||
Name string `json:"name" gorm:"type:varchar(255)"`
|
||||
Clients StringArray `json:"clients" gorm:"type:longtext"`
|
||||
Metric string `json:"metric" gorm:"type:varchar(50);not null;default:'cpu'"` // 监控指标,如 cpu, ram, load
|
||||
Threshold float32 `json:"threshold" gorm:"type:decimal(5,2);not null;default:80.00"` // 阈值百分比
|
||||
Ratio float32 `json:"ratio" gorm:"type:decimal(5,2);not null;default:0.80"` // 达标时间比
|
||||
Interval int `json:"interval" gorm:"type:int;not null;default:15"` // 监测间隔(分钟)
|
||||
LastNotified *time.Time `json:"last_notified"` // 上次通知时间
|
||||
}
|
||||
|
||||
// TrafficReportNotification 定义了流量定时报告的数据库模型
|
||||
type TrafficReportNotification struct {
|
||||
Client string `json:"client" gorm:"type:varchar(36);not null;index;unique;constraint:OnDelete:CASCADE,OnUpdate:CASCADE;foreignKey:client;references:UUID"`
|
||||
|
||||
@@ -15,16 +15,37 @@ const (
|
||||
|
||||
// Theme represents a komari theme information
|
||||
type Theme struct {
|
||||
Name string `json:"name"` // 主题名称
|
||||
Name any `json:"name"` // 主题名称,支持字符串或多语言对象
|
||||
Short string `json:"short"` // 短名称,用作文件夹名
|
||||
Description string `json:"description"` // 主题描述
|
||||
Description any `json:"description"` // 主题描述,支持字符串或多语言对象
|
||||
Version string `json:"version"` // 版本号
|
||||
Author string `json:"author"` // 作者
|
||||
Author any `json:"author"` // 作者,支持字符串或多语言对象
|
||||
URL string `json:"url"` // 主题URL
|
||||
Preview string `json:"preview"` // 预览图片相对路径
|
||||
Configuration Configuration `json:"configuration"` // 声明配置项
|
||||
}
|
||||
|
||||
// IsLocalizedText reports whether a manifest text field has a usable value.
|
||||
func IsLocalizedText(value any) bool {
|
||||
switch text := value.(type) {
|
||||
case string:
|
||||
return strings.TrimSpace(text) != ""
|
||||
case map[string]any:
|
||||
for _, item := range text {
|
||||
if itemText, ok := item.(string); ok && strings.TrimSpace(itemText) != "" {
|
||||
return true
|
||||
}
|
||||
}
|
||||
case map[string]string:
|
||||
for _, item := range text {
|
||||
if strings.TrimSpace(item) != "" {
|
||||
return true
|
||||
}
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
type Configuration struct {
|
||||
Type string `json:"type"` // managed raw redirect
|
||||
Icon string `json:"icon"` // 图标
|
||||
@@ -36,7 +57,7 @@ type ManagedThemeConfigurationItem struct {
|
||||
Key string `json:"key"`
|
||||
Name any `json:"name"`
|
||||
Required bool `json:"required"`
|
||||
Type string `json:"type"` // string number select switch title
|
||||
Type string `json:"type"` // string number select switch title textbox richtext nodes pingtasks
|
||||
Options string `json:"options"`
|
||||
Default any `json:"default"`
|
||||
Help any `json:"help"`
|
||||
|
||||
@@ -1,63 +0,0 @@
|
||||
package notification
|
||||
|
||||
import (
|
||||
"github.com/komari-monitor/komari/database/dbcore"
|
||||
"github.com/komari-monitor/komari/database/models"
|
||||
"github.com/komari-monitor/komari/utils/notifier"
|
||||
"gorm.io/gorm"
|
||||
)
|
||||
|
||||
func AddLoadNotification(clients []string, name string, metric string, threshold float32, ratio float32, interval int) (uint, error) {
|
||||
db := dbcore.GetDBInstance()
|
||||
notification := models.LoadNotification{
|
||||
Clients: clients,
|
||||
Name: name,
|
||||
Metric: metric,
|
||||
Threshold: threshold,
|
||||
Ratio: ratio,
|
||||
Interval: interval,
|
||||
}
|
||||
if err := db.Create(¬ification).Error; err != nil {
|
||||
return 0, err
|
||||
}
|
||||
|
||||
return notification.Id, ReloadLoadNotificationSchedule()
|
||||
}
|
||||
func DeleteLoadNotification(id []uint) error {
|
||||
db := dbcore.GetDBInstance()
|
||||
result := db.Where("id IN ?", id).Delete(&models.LoadNotification{})
|
||||
if result.RowsAffected == 0 {
|
||||
return gorm.ErrRecordNotFound
|
||||
}
|
||||
return ReloadLoadNotificationSchedule()
|
||||
}
|
||||
|
||||
func EditLoadNotification(notifications []*models.LoadNotification) error {
|
||||
db := dbcore.GetDBInstance()
|
||||
for _, notification := range notifications {
|
||||
result := db.Model(&models.LoadNotification{}).Where("id = ?", notification.Id).Updates(notification)
|
||||
if result.RowsAffected == 0 {
|
||||
return gorm.ErrRecordNotFound
|
||||
}
|
||||
}
|
||||
|
||||
return ReloadLoadNotificationSchedule()
|
||||
}
|
||||
|
||||
func GetAllLoadNotifications() ([]models.LoadNotification, error) {
|
||||
db := dbcore.GetDBInstance()
|
||||
var notifications []models.LoadNotification
|
||||
if err := db.Find(¬ifications).Error; err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return notifications, nil
|
||||
}
|
||||
|
||||
func ReloadLoadNotificationSchedule() error {
|
||||
db := dbcore.GetDBInstance()
|
||||
var loadNotifications []models.LoadNotification
|
||||
if err := db.Find(&loadNotifications).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
return notifier.ReloadLoadNotificationSchedule(loadNotifications)
|
||||
}
|
||||
@@ -1,128 +0,0 @@
|
||||
package notification
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
|
||||
"github.com/komari-monitor/komari/database/dbcore"
|
||||
"github.com/komari-monitor/komari/database/models"
|
||||
"gorm.io/gorm/clause"
|
||||
)
|
||||
|
||||
func validateTrafficReportNotification(notification models.TrafficReportNotification) error {
|
||||
if notification.Client == "" {
|
||||
return fmt.Errorf("client UUID cannot be empty")
|
||||
}
|
||||
if notification.Enable && !notification.Daily && !notification.Weekly && !notification.Monthly {
|
||||
return fmt.Errorf("at least one cadence must be selected when enabling traffic reports")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func ValidateTrafficReportNotifications(notifications []models.TrafficReportNotification) error {
|
||||
for _, notification := range notifications {
|
||||
if err := validateTrafficReportNotification(notification); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func buildEnabledTrafficReportNotifications(uuids []string, existing []models.TrafficReportNotification) ([]models.TrafficReportNotification, error) {
|
||||
if len(uuids) == 0 {
|
||||
return nil, fmt.Errorf("at least one client UUID is required")
|
||||
}
|
||||
|
||||
existingByClient := make(map[string]models.TrafficReportNotification, len(existing))
|
||||
for _, notification := range existing {
|
||||
existingByClient[notification.Client] = notification
|
||||
}
|
||||
|
||||
notifications := make([]models.TrafficReportNotification, 0, len(uuids))
|
||||
for _, uuid := range uuids {
|
||||
if uuid == "" {
|
||||
return nil, fmt.Errorf("client UUID cannot be empty")
|
||||
}
|
||||
existingNotification, ok := existingByClient[uuid]
|
||||
if !ok || (!existingNotification.Daily && !existingNotification.Weekly && !existingNotification.Monthly) {
|
||||
return nil, fmt.Errorf("at least one cadence must be selected when enabling traffic reports")
|
||||
}
|
||||
notifications = append(notifications, models.TrafficReportNotification{
|
||||
Client: uuid,
|
||||
Enable: true,
|
||||
})
|
||||
}
|
||||
|
||||
return notifications, nil
|
||||
}
|
||||
|
||||
// ListTrafficReportNotifications 获取所有流量定时报告配置(关联客户端信息)
|
||||
func ListTrafficReportNotifications() ([]models.TrafficReportNotification, error) {
|
||||
db := dbcore.GetDBInstance()
|
||||
var notifications []models.TrafficReportNotification
|
||||
err := db.Model(&models.TrafficReportNotification{}).Preload("ClientInfo").Find(¬ifications).Error
|
||||
return notifications, err
|
||||
}
|
||||
|
||||
// EditTrafficReportNotifications 批量更新流量定时报告配置
|
||||
func EditTrafficReportNotifications(notifications []models.TrafficReportNotification) error {
|
||||
if err := ValidateTrafficReportNotifications(notifications); err != nil {
|
||||
return err
|
||||
}
|
||||
db := dbcore.GetDBInstance()
|
||||
return db.Model(&models.TrafficReportNotification{}).
|
||||
Clauses(clause.OnConflict{
|
||||
Columns: []clause.Column{{Name: "client"}},
|
||||
DoUpdates: clause.AssignmentColumns([]string{"enable", "daily", "weekly", "monthly"}),
|
||||
}).
|
||||
Select("client", "enable", "daily", "weekly", "monthly").
|
||||
Create(notifications).Error
|
||||
}
|
||||
|
||||
// EnableTrafficReportNotifications 批量启用(仅更新 enable 字段)
|
||||
func EnableTrafficReportNotifications(uuids []string) error {
|
||||
if len(uuids) == 0 {
|
||||
return fmt.Errorf("at least one client UUID is required")
|
||||
}
|
||||
|
||||
db := dbcore.GetDBInstance()
|
||||
var existing []models.TrafficReportNotification
|
||||
if err := db.Where("client IN ?", uuids).Find(&existing).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
notifications, err := buildEnabledTrafficReportNotifications(uuids, existing)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return db.Model(&models.TrafficReportNotification{}).
|
||||
Clauses(clause.OnConflict{
|
||||
Columns: []clause.Column{{Name: "client"}},
|
||||
DoUpdates: clause.AssignmentColumns([]string{"enable"}),
|
||||
}).
|
||||
Select("client", "enable").
|
||||
Create(notifications).Error
|
||||
}
|
||||
|
||||
// DisableTrafficReportNotifications 批量禁用
|
||||
func DisableTrafficReportNotifications(uuids []string) error {
|
||||
if len(uuids) == 0 {
|
||||
return fmt.Errorf("at least one client UUID is required")
|
||||
}
|
||||
db := dbcore.GetDBInstance()
|
||||
var notifications []models.TrafficReportNotification
|
||||
for _, uuid := range uuids {
|
||||
if uuid == "" {
|
||||
return fmt.Errorf("client UUID cannot be empty")
|
||||
}
|
||||
notifications = append(notifications, models.TrafficReportNotification{
|
||||
Client: uuid,
|
||||
Enable: false,
|
||||
})
|
||||
}
|
||||
return db.Model(&models.TrafficReportNotification{}).
|
||||
Clauses(clause.OnConflict{
|
||||
Columns: []clause.Column{{Name: "client"}},
|
||||
DoUpdates: clause.AssignmentColumns([]string{"enable"}),
|
||||
}).
|
||||
Select("client", "enable").
|
||||
Create(notifications).Error
|
||||
}
|
||||
@@ -1,36 +0,0 @@
|
||||
package notification
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"github.com/komari-monitor/komari/database/models"
|
||||
)
|
||||
|
||||
func TestValidateTrafficReportNotificationsRejectsEnabledWithoutCadence(t *testing.T) {
|
||||
err := ValidateTrafficReportNotifications([]models.TrafficReportNotification{{
|
||||
Client: "client-a",
|
||||
Enable: true,
|
||||
}})
|
||||
|
||||
assert.Error(t, err)
|
||||
}
|
||||
|
||||
func TestBuildEnabledTrafficReportNotificationsRequiresExistingCadence(t *testing.T) {
|
||||
_, err := buildEnabledTrafficReportNotifications([]string{"client-a"}, nil)
|
||||
assert.Error(t, err)
|
||||
|
||||
_, err = buildEnabledTrafficReportNotifications([]string{"client-a"}, []models.TrafficReportNotification{{Client: "client-a"}})
|
||||
assert.Error(t, err)
|
||||
|
||||
notifications, err := buildEnabledTrafficReportNotifications([]string{"client-a"}, []models.TrafficReportNotification{{
|
||||
Client: "client-a",
|
||||
Daily: true,
|
||||
}})
|
||||
require.NoError(t, err)
|
||||
require.Len(t, notifications, 1)
|
||||
assert.Equal(t, "client-a", notifications[0].Client)
|
||||
assert.True(t, notifications[0].Enable)
|
||||
}
|
||||
+31
-53
@@ -3,16 +3,17 @@ package database
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
logger "github.com/komari-monitor/komari/utils/log"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
|
||||
"github.com/gin-gonic/gin"
|
||||
"github.com/komari-monitor/komari/database/dbcore"
|
||||
"github.com/komari-monitor/komari/internal/metricstore"
|
||||
"github.com/komari-monitor/komari/database/models"
|
||||
"github.com/komari-monitor/komari/internal/config"
|
||||
"github.com/komari-monitor/komari/internal/managedconfig"
|
||||
"github.com/komari-monitor/komari/internal/metricstore"
|
||||
logger "github.com/komari-monitor/komari/utils/log"
|
||||
"github.com/komari-monitor/komari/web/public"
|
||||
)
|
||||
|
||||
func GetPublicInfo() (map[string]interface{}, error) {
|
||||
@@ -69,58 +70,20 @@ func GetPublicInfo() (map[string]interface{}, error) {
|
||||
if err != nil {
|
||||
logger.Infof("database", "%v", err)
|
||||
}
|
||||
// Try to load theme declaration file and merge defaults for managed configuration
|
||||
// Theme declarations live in ./data/theme/<short>/komari-theme.json
|
||||
if cst.Theme != "" && cst.Theme != "default" {
|
||||
themeConfigPath := filepath.Join("./data/theme", cst.Theme, "komari-theme.json")
|
||||
if _, err := os.Stat(themeConfigPath); err == nil {
|
||||
b, err := os.ReadFile(themeConfigPath)
|
||||
if err == nil {
|
||||
var themeDecl struct {
|
||||
Configuration struct {
|
||||
Type string `json:"type"`
|
||||
Data []models.ManagedThemeConfigurationItem `json:"data"`
|
||||
} `json:"configuration"`
|
||||
}
|
||||
if err := json.Unmarshal(b, &themeDecl); err == nil {
|
||||
if themeDecl.Configuration.Type == "managed" {
|
||||
for _, item := range themeDecl.Configuration.Data {
|
||||
if item.Key == "" {
|
||||
continue
|
||||
}
|
||||
// missing
|
||||
if _, exists := tc_data[item.Key]; !exists {
|
||||
var def any = item.Default
|
||||
// select
|
||||
if item.Type == "select" {
|
||||
if def == nil || def == "" {
|
||||
if item.Options != "" {
|
||||
opts := strings.Split(item.Options, ",")
|
||||
if len(opts) > 0 {
|
||||
def = strings.TrimSpace(opts[0])
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
// number->0, string->"", switch->false
|
||||
if def == nil {
|
||||
switch item.Type {
|
||||
case "number":
|
||||
def = 0
|
||||
case "switch":
|
||||
def = false
|
||||
default:
|
||||
def = ""
|
||||
}
|
||||
}
|
||||
tc_data[item.Key] = def
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
items := themeConfigurationItems(cst.Theme)
|
||||
if cst.Theme != "default" {
|
||||
for _, item := range items {
|
||||
if item.Key == "" {
|
||||
continue
|
||||
}
|
||||
if _, exists := tc_data[item.Key]; !exists {
|
||||
tc_data[item.Key] = managedconfig.DefaultValue(item)
|
||||
}
|
||||
}
|
||||
}
|
||||
if err := managedconfig.ResolveForOutput(tc_data, items); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return gin.H{
|
||||
"sitename": cst.Sitename,
|
||||
@@ -135,8 +98,23 @@ func GetPublicInfo() (map[string]interface{}, error) {
|
||||
"record_preserve_time": retention.MaxDays * 24,
|
||||
"ping_record_preserve_time": retention.MaxDays * 24,
|
||||
"private_site": cst.PrivateSite,
|
||||
"visitor_audit_enabled": cst.VisitorAuditEnabled,
|
||||
"theme": cst.Theme,
|
||||
"theme_settings": tc_data,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func themeConfigurationItems(short string) []models.ManagedThemeConfigurationItem {
|
||||
var manifest models.Theme
|
||||
if short == "default" {
|
||||
data, err := public.PublicFS.ReadFile("defaultTheme/komari-theme.json")
|
||||
if err != nil || json.Unmarshal(data, &manifest) != nil {
|
||||
return nil
|
||||
}
|
||||
} else {
|
||||
data, err := os.ReadFile(filepath.Join("./data/theme", short, "komari-theme.json"))
|
||||
if err != nil || json.Unmarshal(data, &manifest) != nil {
|
||||
return nil
|
||||
}
|
||||
}
|
||||
return managedconfig.Items(manifest.Configuration)
|
||||
}
|
||||
+1060
-224
File diff suppressed because it is too large.
Load diff
@@ -16,7 +16,7 @@ type Settings struct {
|
||||
AutoDiscoveryKey string `json:"auto_discovery_key" default:""` // 自动发现密钥
|
||||
ScriptDomain string `json:"script_domain" default:""` // 自定义脚本域名
|
||||
SendIpAddrToGuest bool `json:"send_ip_addr_to_guest" default:"false"` // 是否向访客页面发送 IP 地址,默认 false
|
||||
VisitorAuditEnabled bool `json:"visitor_audit_enabled" default:"false"` // 是否允许公开访客事件写入审计日志,默认 false
|
||||
SSRFProtectionEnabled bool `json:"ssrf_protection_enabled" default:"false"` // 是否启用 SSRF 防护,默认关闭
|
||||
EulaAccepted bool `json:"eula_accepted" default:"false"`
|
||||
BaseScriptsURLKey string `json:"base_scripts_url" default:""`
|
||||
// GeoIP 配置
|
||||
@@ -54,7 +54,7 @@ const (
|
||||
AutoDiscoveryKeyKey = "auto_discovery_key"
|
||||
ScriptDomainKey = "script_domain"
|
||||
SendIpAddrToGuestKey = "send_ip_addr_to_guest"
|
||||
VisitorAuditEnabledKey = "visitor_audit_enabled"
|
||||
SSRFProtectionEnabledKey = "ssrf_protection_enabled"
|
||||
EulaAcceptedKey = "eula_accepted"
|
||||
BaseScriptsURLKey = "base_scripts_url"
|
||||
GeoIpEnabledKey = "geo_ip_enabled"
|
||||
@@ -73,7 +73,6 @@ const (
|
||||
LoginNotificationKey = "login_notification"
|
||||
TrafficLimitPercentageKey = "traffic_limit_percentage"
|
||||
UpdatedAtKey = "updated_at"
|
||||
XtermjsSettingsKey = "xtermjs_settings"
|
||||
ThemeMarketSourcesKey = "theme_market_sources"
|
||||
PluginMarketSourcesKey = "plugin_market_sources"
|
||||
)
|
||||
@@ -0,0 +1,144 @@
|
||||
// Package managedconfig implements the shared managed-configuration behavior
|
||||
// used by themes and plugins.
|
||||
package managedconfig
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"strings"
|
||||
|
||||
"github.com/komari-monitor/komari/database/dbcore"
|
||||
"github.com/komari-monitor/komari/database/models"
|
||||
)
|
||||
|
||||
const (
|
||||
TypeNodes = "nodes"
|
||||
TypePingTasks = "pingtasks"
|
||||
)
|
||||
|
||||
// Items returns the declared managed configuration items. An omitted type is
|
||||
// managed, matching the theme configuration default.
|
||||
func Items(configuration models.Configuration) []models.ManagedThemeConfigurationItem {
|
||||
if configuration.Type != "" && !strings.EqualFold(configuration.Type, models.ThemeConfigurationManaged) {
|
||||
return nil
|
||||
}
|
||||
raw, err := json.Marshal(configuration.Data)
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
var items []models.ManagedThemeConfigurationItem
|
||||
if err := json.Unmarshal(raw, &items); err != nil {
|
||||
return nil
|
||||
}
|
||||
return items
|
||||
}
|
||||
|
||||
func DefaultValue(item models.ManagedThemeConfigurationItem) any {
|
||||
value := item.Default
|
||||
if item.Type == "select" && (value == nil || value == "") && item.Options != "" {
|
||||
options := strings.Split(item.Options, ",")
|
||||
if len(options) > 0 {
|
||||
return strings.TrimSpace(options[0])
|
||||
}
|
||||
}
|
||||
if value != nil {
|
||||
return value
|
||||
}
|
||||
switch item.Type {
|
||||
case "number":
|
||||
return float64(0)
|
||||
case "switch":
|
||||
return false
|
||||
case TypeNodes, TypePingTasks:
|
||||
return "[]"
|
||||
default:
|
||||
return ""
|
||||
}
|
||||
}
|
||||
|
||||
// ResolveForOutput decodes declared selectors, drops deleted references, and
|
||||
// returns typed arrays suitable for public theme settings and plugin config.
|
||||
func ResolveForOutput(values map[string]any, items []models.ManagedThemeConfigurationItem) error {
|
||||
hasNodes := false
|
||||
hasPingTasks := false
|
||||
for _, item := range items {
|
||||
switch item.Type {
|
||||
case TypeNodes:
|
||||
hasNodes = true
|
||||
case TypePingTasks:
|
||||
hasPingTasks = true
|
||||
}
|
||||
}
|
||||
|
||||
db := dbcore.GetDBInstance()
|
||||
liveNodes := map[string]struct{}{}
|
||||
if hasNodes {
|
||||
var nodes []models.Client
|
||||
if err := db.Select("uuid").Find(&nodes).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
for _, node := range nodes {
|
||||
liveNodes[node.UUID] = struct{}{}
|
||||
}
|
||||
}
|
||||
livePingTasks := map[uint]struct{}{}
|
||||
if hasPingTasks {
|
||||
var tasks []models.PingTask
|
||||
if err := db.Select("id").Find(&tasks).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
for _, task := range tasks {
|
||||
livePingTasks[task.Id] = struct{}{}
|
||||
}
|
||||
}
|
||||
|
||||
for _, item := range items {
|
||||
if item.Key == "" {
|
||||
continue
|
||||
}
|
||||
switch item.Type {
|
||||
case TypeNodes:
|
||||
selected := NodeIDs(values[item.Key])
|
||||
filtered := make([]string, 0, len(selected))
|
||||
for _, id := range selected {
|
||||
if _, ok := liveNodes[id]; ok {
|
||||
filtered = append(filtered, id)
|
||||
}
|
||||
}
|
||||
values[item.Key] = filtered
|
||||
case TypePingTasks:
|
||||
selected := PingTaskIDs(values[item.Key])
|
||||
filtered := make([]uint, 0, len(selected))
|
||||
for _, id := range selected {
|
||||
if _, ok := livePingTasks[id]; ok {
|
||||
filtered = append(filtered, id)
|
||||
}
|
||||
}
|
||||
values[item.Key] = filtered
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func NodeIDs(value any) []string {
|
||||
raw, ok := value.(string)
|
||||
if !ok {
|
||||
return nil
|
||||
}
|
||||
var ids []string
|
||||
if json.Unmarshal([]byte(raw), &ids) != nil {
|
||||
return nil
|
||||
}
|
||||
return ids
|
||||
}
|
||||
|
||||
func PingTaskIDs(value any) []uint {
|
||||
raw, ok := value.(string)
|
||||
if !ok {
|
||||
return nil
|
||||
}
|
||||
var ids []uint
|
||||
if json.Unmarshal([]byte(raw), &ids) != nil {
|
||||
return nil
|
||||
}
|
||||
return ids
|
||||
}
|
||||
@@ -92,9 +92,6 @@ func createMetricDefinitionsWithDefaultRetention(ctx context.Context, s *metric.
|
||||
}
|
||||
if err == nil {
|
||||
if existing.RetentionDays == 0 {
|
||||
if _, err := s.SetMetricRetention(ctx, def.Name, 0); err != nil {
|
||||
return fmt.Errorf("failed to preserve disabled metric %s: %w", def.Name, err)
|
||||
}
|
||||
continue
|
||||
}
|
||||
def.RetentionDays = existing.RetentionDays
|
||||
@@ -103,11 +100,5 @@ func createMetricDefinitionsWithDefaultRetention(ctx context.Context, s *metric.
|
||||
return fmt.Errorf("failed to create metric %s: %w", def.Name, err)
|
||||
}
|
||||
}
|
||||
for _, name := range obsoleteBuiltinMetricNames {
|
||||
if err := s.DeleteMetric(ctx, name); err != nil {
|
||||
return fmt.Errorf("failed to remove obsolete metric %s: %w", name, err)
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
@@ -84,7 +84,31 @@ func ReclaimSpace(ctx context.Context) (MaintenanceResult, error) {
|
||||
}
|
||||
maintenanceCtx := context.Background()
|
||||
result.Before, result.BeforeSizeError = activeStore.StorageSize(maintenanceCtx)
|
||||
maintenanceErr := activeStore.ReclaimSpace(maintenanceCtx)
|
||||
maintenanceErr := deleteUndefinedMetrics(maintenanceCtx, activeStore)
|
||||
if maintenanceErr == nil {
|
||||
maintenanceErr = activeStore.ReclaimSpace(maintenanceCtx)
|
||||
}
|
||||
result.After, result.AfterSizeError = activeStore.StorageSize(maintenanceCtx)
|
||||
return result, maintenanceErr
|
||||
}
|
||||
|
||||
func deleteUndefinedMetrics(ctx context.Context, s *metric.Store) error {
|
||||
definitions, err := s.ListMetrics(ctx)
|
||||
if err != nil {
|
||||
return fmt.Errorf("list metric definitions before reclaim: %w", err)
|
||||
}
|
||||
|
||||
builtin := make(map[string]struct{}, len(builtinMetricNames))
|
||||
for _, name := range builtinMetricNames {
|
||||
builtin[name] = struct{}{}
|
||||
}
|
||||
for _, definition := range definitions {
|
||||
if _, ok := builtin[definition.Name]; ok {
|
||||
continue
|
||||
}
|
||||
if err := s.DeleteMetric(ctx, definition.Name); err != nil {
|
||||
return fmt.Errorf("delete undefined metric %q before reclaim: %w", definition.Name, err)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -32,13 +32,6 @@ var loadRecordMetricNames = []string{
|
||||
MetricProcess, MetricConnections, MetricConnectionsUDP,
|
||||
}
|
||||
|
||||
var obsoleteBuiltinMetricNames = []string{
|
||||
"memory.total",
|
||||
"swap.total",
|
||||
"temperature",
|
||||
"disk.total",
|
||||
}
|
||||
|
||||
// gpuDeviceRecordMetricNames are stored separately from the entity-level GPU
|
||||
// average and are included when deleting all system records.
|
||||
var gpuDeviceRecordMetricNames = []string{
|
||||
@@ -50,6 +43,8 @@ var recordMetricNames = joinMetricNames(loadRecordMetricNames, gpuDeviceRecordMe
|
||||
// Ping has an independent retention and cleanup boundary.
|
||||
var pingMetricNames = []string{MetricPingLatency, MetricPingLoss}
|
||||
|
||||
var builtinMetricNames = joinMetricNames(recordMetricNames, pingMetricNames)
|
||||
|
||||
func metricNameForRecordField(name string) (string, bool) {
|
||||
switch name {
|
||||
case "cpu":
|
||||
|
||||
@@ -3,14 +3,13 @@ package metricstore
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"errors"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/komari-monitor/komari/database/models"
|
||||
"github.com/komari-monitor/komari/pkg/metric"
|
||||
v1 "github.com/komari-monitor/komari/protocol/v1"
|
||||
v2 "github.com/komari-monitor/komari/protocol/v2"
|
||||
)
|
||||
|
||||
func TestDefaultRollupPolicy(t *testing.T) {
|
||||
@@ -283,7 +282,7 @@ func TestCreateMetricDefinitionsUsesExplicitRetentionAndPreservesOverrides(t *te
|
||||
}
|
||||
}
|
||||
|
||||
func TestCreateMetricDefinitionsRemovesObsoleteMetrics(t *testing.T) {
|
||||
func TestCreateMetricDefinitionsKeepsExistingMetrics(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
s, err := metric.Open(ctx, metric.SQLite(":memory:", metric.WithMaxOpenConns(1)))
|
||||
if err != nil {
|
||||
@@ -299,15 +298,19 @@ func TestCreateMetricDefinitionsRemovesObsoleteMetrics(t *testing.T) {
|
||||
if err := createMetricDefinitions(ctx, s); err != nil {
|
||||
t.Fatalf("refresh built-in definitions: %v", err)
|
||||
}
|
||||
if _, err := s.GetMetric(ctx, "memory.total"); !errors.Is(err, metric.ErrNotFound) {
|
||||
t.Fatalf("obsolete definition error = %v, want ErrNotFound", err)
|
||||
definition, err := s.GetMetric(ctx, "memory.total")
|
||||
if err != nil {
|
||||
t.Fatalf("existing definition was removed: %v", err)
|
||||
}
|
||||
if definition.RetentionDays != 1 {
|
||||
t.Fatalf("existing retention = %d, want 1", definition.RetentionDays)
|
||||
}
|
||||
points, err := s.Query(ctx, metric.Query{MetricName: "memory.total", EntityID: "node-a", Start: time.Now().UTC().Add(-time.Hour), End: time.Now().UTC().Add(time.Hour)})
|
||||
if err != nil {
|
||||
t.Fatalf("query obsolete points: %v", err)
|
||||
t.Fatalf("query existing points: %v", err)
|
||||
}
|
||||
if len(points) != 0 {
|
||||
t.Fatalf("obsolete points remain: %#v", points)
|
||||
if len(points) != 1 || points[0].Value != 1024 {
|
||||
t.Fatalf("existing points were removed: %#v", points)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -569,15 +572,15 @@ func TestGetRecordsByClientAndTimeReadsRollupsAfterRawCompaction(t *testing.T) {
|
||||
Connections: 321,
|
||||
ConnectionsUdp: 12,
|
||||
}
|
||||
if _, err := WriteReport(ctx, v1.Report{
|
||||
if _, err := WriteReport(ctx, v2.Report{
|
||||
UUID: rec.Client,
|
||||
UpdatedAt: ts,
|
||||
CPU: v1.CPUReport{Usage: float64(rec.Cpu)},
|
||||
Ram: v1.RamReport{Used: rec.Ram, Total: rec.RamTotal},
|
||||
Load: v1.LoadReport{Load1: float64(rec.Load)},
|
||||
Disk: v1.DiskReport{Used: rec.Disk, Total: rec.DiskTotal},
|
||||
CPU: v2.CPUReport{Usage: float64(rec.Cpu)},
|
||||
Ram: v2.RamReport{Used: rec.Ram, Total: rec.RamTotal},
|
||||
Load: v2.LoadReport{Load1: float64(rec.Load)},
|
||||
Disk: v2.DiskReport{Used: rec.Disk, Total: rec.DiskTotal},
|
||||
Process: rec.Process,
|
||||
Connections: v1.ConnectionsReport{
|
||||
Connections: v2.ConnectionsReport{
|
||||
TCP: rec.Connections,
|
||||
UDP: rec.ConnectionsUdp,
|
||||
},
|
||||
|
||||
@@ -0,0 +1,63 @@
|
||||
package metricstore
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"fmt"
|
||||
|
||||
"github.com/komari-monitor/komari/pkg/metric"
|
||||
)
|
||||
|
||||
// QueryContext runs a raw read query against the active metric store. The
|
||||
// caller must invoke the returned release function after closing rows.
|
||||
func QueryContext(ctx context.Context, query string, args ...any) (*sql.Rows, metric.Driver, func(), error) {
|
||||
return QueryForDriver(ctx, func(metric.Driver) (string, error) {
|
||||
return query, nil
|
||||
}, args...)
|
||||
}
|
||||
|
||||
// QueryForDriver runs a raw read query whose SQL depends on the active store's
|
||||
// driver. The caller must invoke the returned release function after closing rows.
|
||||
func QueryForDriver(ctx context.Context, query func(metric.Driver) (string, error), args ...any) (*sql.Rows, metric.Driver, func(), error) {
|
||||
if err := storeOperations.AcquireShared(ctx); err != nil {
|
||||
return nil, "", nil, fmt.Errorf("wait for metric store operations before query: %w", err)
|
||||
}
|
||||
|
||||
storeMu.RLock()
|
||||
activeStore := store
|
||||
storeMu.RUnlock()
|
||||
if activeStore == nil {
|
||||
storeOperations.ReleaseShared()
|
||||
return nil, "", nil, ErrStoreNotInitialized
|
||||
}
|
||||
|
||||
statement, err := query(activeStore.Driver())
|
||||
if err != nil {
|
||||
storeOperations.ReleaseShared()
|
||||
return nil, "", nil, err
|
||||
}
|
||||
rows, err := activeStore.QueryContext(ctx, statement, args...)
|
||||
if err != nil {
|
||||
storeOperations.ReleaseShared()
|
||||
return nil, "", nil, err
|
||||
}
|
||||
return rows, activeStore.Driver(), storeOperations.ReleaseShared, nil
|
||||
}
|
||||
|
||||
// ExecContext runs a raw statement against the active metric store.
|
||||
func ExecContext(ctx context.Context, query string, args ...any) (sql.Result, metric.Driver, error) {
|
||||
if err := storeOperations.Acquire(ctx); err != nil {
|
||||
return nil, "", fmt.Errorf("wait for metric store operations before execution: %w", err)
|
||||
}
|
||||
defer storeOperations.Release()
|
||||
|
||||
storeMu.RLock()
|
||||
activeStore := store
|
||||
storeMu.RUnlock()
|
||||
if activeStore == nil {
|
||||
return nil, "", ErrStoreNotInitialized
|
||||
}
|
||||
|
||||
result, err := activeStore.ExecContext(ctx, query, args...)
|
||||
return result, activeStore.Driver(), err
|
||||
}
|
||||
@@ -11,7 +11,7 @@ import (
|
||||
|
||||
"github.com/komari-monitor/komari/database/models"
|
||||
"github.com/komari-monitor/komari/pkg/metric"
|
||||
v1 "github.com/komari-monitor/komari/protocol/v1"
|
||||
v2 "github.com/komari-monitor/komari/protocol/v2"
|
||||
)
|
||||
|
||||
type reportTrafficState struct {
|
||||
@@ -25,17 +25,24 @@ type reportTrafficValues struct {
|
||||
timestamp time.Time
|
||||
hasUp bool
|
||||
totalUp int64
|
||||
totalUpAt time.Time
|
||||
hasDown bool
|
||||
totalDown int64
|
||||
totalDownAt time.Time
|
||||
}
|
||||
|
||||
var reportTrafficStates sync.Map
|
||||
var (
|
||||
reportTrafficStates sync.Map
|
||||
reportWriteMu sync.Mutex
|
||||
)
|
||||
|
||||
const (
|
||||
reportBatchInterval = 3 * time.Second
|
||||
reportBatchQueueSize = 4096
|
||||
reportBatchQueueSize = 512
|
||||
pingBatchMaxRecords = 512
|
||||
reportBatchWriteTimeout = 10 * time.Second
|
||||
// Longer gaps have no reliable per-bucket traffic attribution.
|
||||
maxTrafficReportGap = 15 * time.Minute
|
||||
)
|
||||
|
||||
var (
|
||||
@@ -57,7 +64,7 @@ type reportBatchRequest struct {
|
||||
|
||||
type reportBatchWorker struct {
|
||||
mu sync.Mutex
|
||||
queue chan v1.Report
|
||||
queue chan v2.Report
|
||||
pingQueue chan models.PingRecord
|
||||
requests chan reportBatchRequest
|
||||
done chan struct{}
|
||||
@@ -73,7 +80,7 @@ func StartReportBatcher() {
|
||||
return
|
||||
}
|
||||
worker := &reportBatchWorker{
|
||||
queue: make(chan v1.Report, reportBatchQueueSize),
|
||||
queue: make(chan v2.Report, reportBatchQueueSize),
|
||||
pingQueue: make(chan models.PingRecord, reportBatchQueueSize),
|
||||
requests: make(chan reportBatchRequest, 1),
|
||||
done: make(chan struct{}),
|
||||
@@ -150,16 +157,16 @@ func FlushReportBatch(ctx context.Context) error {
|
||||
// WriteReport persists one agent report and adds it to in-memory minute
|
||||
// summaries using the same server receive time. Traffic deltas remain summable
|
||||
// after rollup.
|
||||
func WriteReport(ctx context.Context, report v1.Report) (v1.Report, error) {
|
||||
func WriteReport(ctx context.Context, report v2.Report) (v2.Report, error) {
|
||||
if report.UUID == "" {
|
||||
return v1.Report{}, fmt.Errorf("report UUID is required")
|
||||
return v2.Report{}, fmt.Errorf("report UUID is required")
|
||||
}
|
||||
if report.UpdatedAt.IsZero() {
|
||||
return v1.Report{}, fmt.Errorf("report receive time is required")
|
||||
return v2.Report{}, fmt.Errorf("report receive time is required")
|
||||
}
|
||||
report.UpdatedAt = report.UpdatedAt.UTC()
|
||||
if GetStore() == nil {
|
||||
return v1.Report{}, fmt.Errorf("metric store not enabled")
|
||||
return v2.Report{}, fmt.Errorf("metric store not enabled")
|
||||
}
|
||||
|
||||
reportBatcherMu.Lock()
|
||||
@@ -167,19 +174,19 @@ func WriteReport(ctx context.Context, report v1.Report) (v1.Report, error) {
|
||||
reportBatcherMu.Unlock()
|
||||
if worker != nil {
|
||||
if err := worker.enqueue(ctx, report); err != nil {
|
||||
return v1.Report{}, err
|
||||
return v2.Report{}, err
|
||||
}
|
||||
return report, nil
|
||||
}
|
||||
|
||||
saved, err := writeReportBatch(ctx, []v1.Report{report})
|
||||
saved, err := writeReportBatch(ctx, []v2.Report{report})
|
||||
if err != nil {
|
||||
return v1.Report{}, err
|
||||
return v2.Report{}, err
|
||||
}
|
||||
return saved[0], nil
|
||||
}
|
||||
|
||||
func (w *reportBatchWorker) enqueue(ctx context.Context, report v1.Report) error {
|
||||
func (w *reportBatchWorker) enqueue(ctx context.Context, report v2.Report) error {
|
||||
w.mu.Lock()
|
||||
defer w.mu.Unlock()
|
||||
if w.stopping {
|
||||
@@ -200,7 +207,7 @@ func (w *reportBatchWorker) run() {
|
||||
ticker := time.NewTicker(reportBatchInterval)
|
||||
defer ticker.Stop()
|
||||
|
||||
var pending []v1.Report
|
||||
var pending []v2.Report
|
||||
var pendingPings []models.PingRecord
|
||||
for {
|
||||
select {
|
||||
@@ -250,7 +257,7 @@ func (w *reportBatchWorker) enqueuePing(ctx context.Context, record models.PingR
|
||||
}
|
||||
}
|
||||
|
||||
func drainReportQueue(queue <-chan v1.Report, limit int) []v1.Report {
|
||||
func drainReportQueue(queue <-chan v2.Report, limit int) []v2.Report {
|
||||
if limit <= 0 {
|
||||
return nil
|
||||
}
|
||||
@@ -261,7 +268,7 @@ func drainReportQueue(queue <-chan v1.Report, limit int) []v1.Report {
|
||||
if capacity == 0 {
|
||||
return nil
|
||||
}
|
||||
reports := make([]v1.Report, 0, capacity)
|
||||
reports := make([]v2.Report, 0, capacity)
|
||||
for len(reports) < limit {
|
||||
select {
|
||||
case report := <-queue:
|
||||
@@ -296,7 +303,7 @@ func drainPingQueue(queue <-chan models.PingRecord, limit int) []models.PingReco
|
||||
return records
|
||||
}
|
||||
|
||||
func writePendingReports(ctx context.Context, pending *[]v1.Report) error {
|
||||
func writePendingReports(ctx context.Context, pending *[]v2.Report) error {
|
||||
if len(*pending) == 0 {
|
||||
return nil
|
||||
}
|
||||
@@ -333,7 +340,7 @@ func writePendingPingRecords(ctx context.Context, pending *[]models.PingRecord)
|
||||
return nil
|
||||
}
|
||||
|
||||
func writeReportBatch(ctx context.Context, reports []v1.Report) ([]v1.Report, error) {
|
||||
func writeReportBatch(ctx context.Context, reports []v2.Report) ([]v2.Report, error) {
|
||||
if len(reports) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
@@ -342,12 +349,17 @@ func writeReportBatch(ctx context.Context, reports []v1.Report) ([]v1.Report, er
|
||||
}
|
||||
defer storeOperations.ReleaseShared()
|
||||
|
||||
// Serialize baseline reads and commits, including direct writes without
|
||||
// the background batcher.
|
||||
reportWriteMu.Lock()
|
||||
defer reportWriteMu.Unlock()
|
||||
|
||||
s := GetStore()
|
||||
if s == nil {
|
||||
return nil, fmt.Errorf("metric store not enabled")
|
||||
}
|
||||
|
||||
prepared := make([]v1.Report, len(reports))
|
||||
prepared := make([]v2.Report, len(reports))
|
||||
copy(prepared, reports)
|
||||
points := make([]metric.Point, 0, len(reports)*20)
|
||||
pendingStates := make(map[*reportTrafficState]reportTrafficValues)
|
||||
@@ -361,43 +373,50 @@ func writeReportBatch(ctx context.Context, reports []v1.Report) ([]v1.Report, er
|
||||
state.mu.Unlock()
|
||||
}
|
||||
if !values.initialized {
|
||||
totalUp, hasUp, err := latestReportCounter(ctx, s, MetricNetTotalUp, report.UUID, report.UpdatedAt)
|
||||
if err == nil {
|
||||
values.totalUp = totalUp
|
||||
values.hasUp = hasUp
|
||||
} else if ctx.Err() == nil {
|
||||
totalUp, hasUp, err := s.LatestBefore(ctx, MetricNetTotalUp, report.UUID, report.UpdatedAt)
|
||||
if err != nil {
|
||||
logger.Errorf("metricstore", "failed to restore previous upload counter for %s: %v", report.UUID, err)
|
||||
} else {
|
||||
values.totalUp = int64(totalUp.Value)
|
||||
values.totalUpAt = totalUp.Timestamp
|
||||
values.hasUp = hasUp
|
||||
}
|
||||
totalDown, hasDown, err := latestReportCounter(ctx, s, MetricNetTotalDown, report.UUID, report.UpdatedAt)
|
||||
if err == nil {
|
||||
values.totalDown = totalDown
|
||||
values.hasDown = hasDown
|
||||
} else if ctx.Err() == nil {
|
||||
totalDown, hasDown, err := s.LatestBefore(ctx, MetricNetTotalDown, report.UUID, report.UpdatedAt)
|
||||
if err != nil {
|
||||
logger.Errorf("metricstore", "failed to restore previous download counter for %s: %v", report.UUID, err)
|
||||
}
|
||||
if err := ctx.Err(); err != nil {
|
||||
return nil, err
|
||||
} else {
|
||||
values.totalDown = int64(totalDown.Value)
|
||||
values.totalDownAt = totalDown.Timestamp
|
||||
values.hasDown = hasDown
|
||||
}
|
||||
values.initialized = true
|
||||
// Keep only the restored baseline on failure. WriteBatch may have
|
||||
// accepted raw samples before a rollup flush fails, so restoring
|
||||
// again on retry could otherwise read this uncommitted batch.
|
||||
state.mu.Lock()
|
||||
state.reportTrafficValues = values
|
||||
state.mu.Unlock()
|
||||
}
|
||||
|
||||
if !values.timestamp.IsZero() && !report.UpdatedAt.After(values.timestamp) {
|
||||
report.UpdatedAt = values.timestamp.Add(time.Millisecond)
|
||||
}
|
||||
trafficUp := int64(0)
|
||||
if values.hasUp {
|
||||
if values.hasUp && !values.totalUpAt.IsZero() && report.UpdatedAt.Sub(values.totalUpAt) <= maxTrafficReportGap {
|
||||
trafficUp = TrafficCounterDelta(report.Network.TotalUp, values.totalUp)
|
||||
}
|
||||
trafficDown := int64(0)
|
||||
if values.hasDown {
|
||||
if values.hasDown && !values.totalDownAt.IsZero() && report.UpdatedAt.Sub(values.totalDownAt) <= maxTrafficReportGap {
|
||||
trafficDown = TrafficCounterDelta(report.Network.TotalDown, values.totalDown)
|
||||
}
|
||||
points = append(points, reportMetricPoints(report, trafficUp, trafficDown)...)
|
||||
values.timestamp = report.UpdatedAt
|
||||
values.hasUp = true
|
||||
values.totalUp = report.Network.TotalUp
|
||||
values.totalUpAt = report.UpdatedAt
|
||||
values.hasDown = true
|
||||
values.totalDown = report.Network.TotalDown
|
||||
values.totalDownAt = report.UpdatedAt
|
||||
pendingStates[state] = values
|
||||
prepared[i] = report
|
||||
}
|
||||
@@ -405,10 +424,14 @@ func writeReportBatch(ctx context.Context, reports []v1.Report) ([]v1.Report, er
|
||||
if err := s.WriteBatch(ctx, points); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Advance counters and timestamps only after success. Retried batches
|
||||
// must replace the same samples, not manufacture resets or new timestamps.
|
||||
for state, values := range pendingStates {
|
||||
state.mu.Lock()
|
||||
state.reportTrafficValues = values
|
||||
state.mu.Unlock()
|
||||
}
|
||||
|
||||
return prepared, nil
|
||||
}
|
||||
@@ -8,10 +8,10 @@ import (
|
||||
|
||||
"github.com/komari-monitor/komari/database/models"
|
||||
"github.com/komari-monitor/komari/pkg/metric"
|
||||
v1 "github.com/komari-monitor/komari/protocol/v1"
|
||||
v2 "github.com/komari-monitor/komari/protocol/v2"
|
||||
)
|
||||
|
||||
func reportMetricPoints(report v1.Report, trafficUp, trafficDown int64) []metric.Point {
|
||||
func reportMetricPoints(report v2.Report, trafficUp, trafficDown int64) []metric.Point {
|
||||
entityID := report.UUID
|
||||
ts := report.UpdatedAt
|
||||
points := []metric.Point{
|
||||
@@ -93,8 +93,15 @@ func GetLatestTrafficBefore(ctx context.Context, entityIDs []string, before time
|
||||
return result, nil
|
||||
}
|
||||
|
||||
// maxResetAwareDelta is the largest per-sample increase accepted when a
|
||||
// counter appears to reset. A "reset" whose current value is larger than this
|
||||
// is treated as jitter of a still-huge counter, not a wrap or reboot leftover.
|
||||
const maxResetAwareDelta int64 = 64 << 30
|
||||
|
||||
// TrafficCounterDelta returns a reset-aware increase between two cumulative
|
||||
// traffic counters. After a reset, the current counter is the new increase.
|
||||
// traffic counters. After a wrap or reboot, the current counter is the new
|
||||
// increase. A tiny dip of a TB-scale counter is not a reset: counting the
|
||||
// still-huge current value would inject a multi-terabyte spike.
|
||||
func TrafficCounterDelta(current, previous int64) int64 {
|
||||
if current < 0 || previous < 0 {
|
||||
return 0
|
||||
@@ -102,6 +109,9 @@ func TrafficCounterDelta(current, previous int64) int64 {
|
||||
if current >= previous {
|
||||
return current - previous
|
||||
}
|
||||
if current > maxResetAwareDelta {
|
||||
return 0
|
||||
}
|
||||
return current
|
||||
}
|
||||
|
||||
|
||||
@@ -2,14 +2,18 @@ package metricstore
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"database/sql/driver"
|
||||
"errors"
|
||||
"fmt"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/komari-monitor/komari/database/models"
|
||||
"github.com/komari-monitor/komari/pkg/metric"
|
||||
v1 "github.com/komari-monitor/komari/protocol/v1"
|
||||
v2 "github.com/komari-monitor/komari/protocol/v2"
|
||||
sqlite3 "github.com/mattn/go-sqlite3"
|
||||
)
|
||||
|
||||
func useReportTestStore(t *testing.T, policy *metric.RollupPolicy) *metric.Store {
|
||||
@@ -44,6 +48,198 @@ func useReportTestStore(t *testing.T, policy *metric.RollupPolicy) *metric.Store
|
||||
return s
|
||||
}
|
||||
|
||||
type reportCounterFault struct {
|
||||
remaining atomic.Int32
|
||||
denied atomic.Int32
|
||||
failWrite atomic.Bool
|
||||
}
|
||||
|
||||
func (f *reportCounterFault) denyRollupRead() bool {
|
||||
for {
|
||||
remaining := f.remaining.Load()
|
||||
if remaining <= 0 {
|
||||
return false
|
||||
}
|
||||
if f.remaining.CompareAndSwap(remaining, remaining-1) {
|
||||
f.denied.Add(1)
|
||||
return true
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
type reportSQLiteConnector struct {
|
||||
driver *sqlite3.SQLiteDriver
|
||||
dsn string
|
||||
}
|
||||
|
||||
func (c *reportSQLiteConnector) Connect(context.Context) (driver.Conn, error) {
|
||||
return c.driver.Open(c.dsn)
|
||||
}
|
||||
|
||||
func (c *reportSQLiteConnector) Driver() driver.Driver {
|
||||
return c.driver
|
||||
}
|
||||
|
||||
// useReportCounterFailureStore denies exactly two rollup reads after schema
|
||||
// setup, exercising failed counter restoration while leaving writes usable.
|
||||
func useReportCounterFailureStore(t *testing.T) (*metric.Store, *reportCounterFault) {
|
||||
t.Helper()
|
||||
fault := &reportCounterFault{}
|
||||
dsn := fmt.Sprintf("file:report-counter-fault-%d?mode=memory&cache=shared", time.Now().UnixNano())
|
||||
driver := &sqlite3.SQLiteDriver{
|
||||
ConnectHook: func(conn *sqlite3.SQLiteConn) error {
|
||||
conn.RegisterAuthorizer(func(op int, arg1, _, _ string) int {
|
||||
if op == sqlite3.SQLITE_TRANSACTION && arg1 == "BEGIN" && fault.failWrite.Load() {
|
||||
return sqlite3.SQLITE_DENY
|
||||
}
|
||||
if op == sqlite3.SQLITE_READ && arg1 == "metric_rollups" && fault.denyRollupRead() {
|
||||
return sqlite3.SQLITE_DENY
|
||||
}
|
||||
return sqlite3.SQLITE_OK
|
||||
})
|
||||
return nil
|
||||
},
|
||||
}
|
||||
db := sql.OpenDB(&reportSQLiteConnector{driver: driver, dsn: dsn})
|
||||
s, err := metric.Open(context.Background(), metric.SQLite("", metric.WithDB(db), metric.WithMaxOpenConns(1)))
|
||||
if err != nil {
|
||||
_ = db.Close()
|
||||
t.Fatalf("open metric store: %v", err)
|
||||
}
|
||||
if err := createMetricDefinitions(context.Background(), s); err != nil {
|
||||
_ = s.Close()
|
||||
_ = db.Close()
|
||||
t.Fatalf("create metric definitions: %v", err)
|
||||
}
|
||||
fault.remaining.Store(2)
|
||||
storeMu.Lock()
|
||||
previous := store
|
||||
store = s
|
||||
storeMu.Unlock()
|
||||
t.Cleanup(func() {
|
||||
clearReportTrafficStates()
|
||||
storeMu.Lock()
|
||||
store = previous
|
||||
storeMu.Unlock()
|
||||
_ = s.Close()
|
||||
_ = db.Close()
|
||||
})
|
||||
return s, fault
|
||||
}
|
||||
|
||||
func TestReportBatchCounterRestoreFailureInitializesStateOnce(t *testing.T) {
|
||||
s, fault := useReportCounterFailureStore(t)
|
||||
ctx := context.Background()
|
||||
base := time.Now().UTC().Truncate(time.Second)
|
||||
first := v2.Report{
|
||||
UUID: "counter-restore-failure",
|
||||
UpdatedAt: base,
|
||||
CPU: v2.CPUReport{Usage: 10},
|
||||
Network: v2.NetworkReport{TotalUp: 100, TotalDown: 200},
|
||||
}
|
||||
|
||||
if _, err := writeReportBatch(ctx, []v2.Report{first}); err != nil {
|
||||
t.Fatalf("write first report after counter restore failure: %v", err)
|
||||
}
|
||||
if fault.denied.Load() != 2 {
|
||||
t.Fatalf("counter restore queries = %d, want 2", fault.denied.Load())
|
||||
}
|
||||
stateValue, ok := reportTrafficStates.Load(first.UUID)
|
||||
if !ok {
|
||||
t.Fatal("report traffic state was not persisted")
|
||||
}
|
||||
state := stateValue.(*reportTrafficState)
|
||||
state.mu.Lock()
|
||||
initialized := state.initialized
|
||||
state.mu.Unlock()
|
||||
if !initialized {
|
||||
t.Fatal("report traffic state was not initialized after restore failure")
|
||||
}
|
||||
|
||||
second := first
|
||||
second.UpdatedAt = base.Add(time.Second)
|
||||
second.Network.TotalUp = 150
|
||||
second.Network.TotalDown = 260
|
||||
if _, err := writeReportBatch(ctx, []v2.Report{second}); err != nil {
|
||||
t.Fatalf("write second report: %v", err)
|
||||
}
|
||||
if fault.denied.Load() != 2 {
|
||||
t.Fatalf("counter restore queries after second batch = %d, want no repeat", fault.denied.Load())
|
||||
}
|
||||
assertMetricValues(t, s, MetricTrafficUp, first.UUID, base.Add(-time.Second), base.Add(2*time.Second), []float64{0, 50})
|
||||
assertMetricValues(t, s, MetricTrafficDown, first.UUID, base.Add(-time.Second), base.Add(2*time.Second), []float64{0, 60})
|
||||
}
|
||||
|
||||
func TestReportBatchRetryDoesNotCountHistoricalTraffic(t *testing.T) {
|
||||
for _, existingBaseline := range []bool{false, true} {
|
||||
t.Run(fmt.Sprintf("existing-baseline-%v", existingBaseline), func(t *testing.T) {
|
||||
s, fault := useReportCounterFailureStore(t)
|
||||
fault.remaining.Store(0)
|
||||
ctx := context.Background()
|
||||
base := time.Now().UTC().Truncate(time.Minute).Add(-2 * time.Minute)
|
||||
report := v2.Report{
|
||||
UUID: "retry-node", UpdatedAt: base,
|
||||
Network: v2.NetworkReport{TotalUp: 12 << 30, TotalDown: 30 << 30},
|
||||
}
|
||||
if existingBaseline {
|
||||
if _, err := writeReportBatch(ctx, []v2.Report{report}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
first := report
|
||||
first.UpdatedAt = base.Add(3 * time.Second)
|
||||
first.Network.TotalUp += 100
|
||||
first.Network.TotalDown += 200
|
||||
second := first
|
||||
second.UpdatedAt = base.Add(6 * time.Second)
|
||||
second.Network.TotalUp += 50
|
||||
second.Network.TotalDown += 60
|
||||
pending := []v2.Report{first, second}
|
||||
|
||||
fault.failWrite.Store(true)
|
||||
for attempt := 0; attempt < 2; attempt++ {
|
||||
if err := writePendingReports(ctx, &pending); err == nil {
|
||||
t.Fatal("expected injected rollup write failure")
|
||||
}
|
||||
if len(pending) != 2 {
|
||||
t.Fatal("failed reports were removed from pending batch")
|
||||
}
|
||||
value, _ := reportTrafficStates.Load(report.UUID)
|
||||
state := value.(*reportTrafficState)
|
||||
state.mu.Lock()
|
||||
baseline := state.reportTrafficValues
|
||||
state.mu.Unlock()
|
||||
if baseline.hasUp != existingBaseline || baseline.hasDown != existingBaseline ||
|
||||
(existingBaseline && (baseline.totalUp != report.Network.TotalUp ||
|
||||
baseline.totalDown != report.Network.TotalDown || !baseline.timestamp.Equal(base))) {
|
||||
t.Fatal("failed write advanced the committed counter baseline")
|
||||
}
|
||||
}
|
||||
fault.failWrite.Store(false)
|
||||
if err := writePendingReports(ctx, &pending); err != nil {
|
||||
t.Fatalf("retry failed: %v", err)
|
||||
}
|
||||
if len(pending) != 0 {
|
||||
t.Fatal("successful reports remained pending")
|
||||
}
|
||||
|
||||
wantUp, wantDown := []float64{0, 50}, []float64{0, 60}
|
||||
if existingBaseline {
|
||||
wantUp, wantDown = []float64{0, 100, 50}, []float64{0, 200, 60}
|
||||
}
|
||||
assertMetricValues(t, s, MetricTrafficUp, report.UUID, base.Add(-time.Second), base.Add(time.Minute), wantUp)
|
||||
assertMetricValues(t, s, MetricTrafficDown, report.UUID, base.Add(-time.Second), base.Add(time.Minute), wantDown)
|
||||
var sumUp, sumDown float64
|
||||
for i := range wantUp {
|
||||
sumUp += wantUp[i]
|
||||
sumDown += wantDown[i]
|
||||
}
|
||||
assertMetricAggregate(t, s, MetricTrafficUp, report.UUID, base.Add(-time.Second), base.Add(time.Minute), metric.AggSum, sumUp, len(wantUp))
|
||||
assertMetricAggregate(t, s, MetricTrafficDown, report.UUID, base.Add(-time.Second), base.Add(time.Minute), metric.AggSum, sumDown, len(wantDown))
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestWriteReportStoresMinuteMetricsAndResetAwareTraffic(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
policy := defaultRollupPolicy()
|
||||
@@ -51,20 +247,20 @@ func TestWriteReportStoresMinuteMetricsAndResetAwareTraffic(t *testing.T) {
|
||||
base := time.Now().UTC().Truncate(time.Minute).Add(5 * time.Second)
|
||||
now := base.Add(45 * time.Second)
|
||||
|
||||
report := v1.Report{
|
||||
report := v2.Report{
|
||||
UUID: "node-a",
|
||||
UpdatedAt: base,
|
||||
CPU: v1.CPUReport{Usage: 12.5},
|
||||
Ram: v1.RamReport{Used: 100, Total: 1000},
|
||||
Swap: v1.RamReport{Used: 20, Total: 200},
|
||||
Load: v1.LoadReport{Load1: 0.5},
|
||||
Disk: v1.DiskReport{Used: 300, Total: 3000},
|
||||
Network: v1.NetworkReport{Up: 3, Down: 4, TotalUp: 100, TotalDown: 200},
|
||||
CPU: v2.CPUReport{Usage: 12.5},
|
||||
Ram: v2.RamReport{Used: 100, Total: 1000},
|
||||
Swap: v2.RamReport{Used: 20, Total: 200},
|
||||
Load: v2.LoadReport{Load1: 0.5},
|
||||
Disk: v2.DiskReport{Used: 300, Total: 3000},
|
||||
Network: v2.NetworkReport{Up: 3, Down: 4, TotalUp: 100, TotalDown: 200},
|
||||
Process: 7,
|
||||
Connections: v1.ConnectionsReport{TCP: 8, UDP: 9},
|
||||
GPU: &v1.GPUDetailReport{
|
||||
Connections: v2.ConnectionsReport{TCP: 8, UDP: 9},
|
||||
GPU: &v2.GPUDetailReport{
|
||||
AverageUsage: 25,
|
||||
DetailedInfo: []v1.GPUDeviceInfo{{
|
||||
DetailedInfo: []v2.GPUDeviceInfo{{
|
||||
Name: "GPU 0", MemoryUsed: 400, MemoryTotal: 800, Utilization: 30, Temperature: 55,
|
||||
}},
|
||||
},
|
||||
@@ -126,7 +322,7 @@ func TestWriteReportSkipsMetricsWithoutAgentData(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
s := useReportTestStore(t, nil)
|
||||
timestamp := time.Now().UTC()
|
||||
if _, err := WriteReport(ctx, v1.Report{
|
||||
if _, err := WriteReport(ctx, v2.Report{
|
||||
UUID: "node-without-gpu", UpdatedAt: timestamp,
|
||||
}); err != nil {
|
||||
t.Fatalf("write report: %v", err)
|
||||
@@ -154,11 +350,11 @@ func TestReportBatcherFlushesQueuedReports(t *testing.T) {
|
||||
})
|
||||
|
||||
base := time.Now().UTC().Truncate(time.Minute).Add(10 * time.Second)
|
||||
first := v1.Report{
|
||||
first := v2.Report{
|
||||
UUID: "batched-node",
|
||||
UpdatedAt: base,
|
||||
CPU: v1.CPUReport{Usage: 10},
|
||||
Network: v1.NetworkReport{TotalUp: 100, TotalDown: 200},
|
||||
CPU: v2.CPUReport{Usage: 10},
|
||||
Network: v2.NetworkReport{TotalUp: 100, TotalDown: 200},
|
||||
}
|
||||
second := first
|
||||
second.UpdatedAt = base.Add(3 * time.Second)
|
||||
@@ -271,11 +467,11 @@ func TestReportBatchKeepsEverySample(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
s := useReportTestStore(t, nil)
|
||||
base := time.Now().UTC().Truncate(time.Second)
|
||||
pending := []v1.Report{
|
||||
{UUID: "node-a", UpdatedAt: base, CPU: v1.CPUReport{Usage: 10}, Network: v1.NetworkReport{TotalUp: 100}},
|
||||
{UUID: "node-a", UpdatedAt: base, CPU: v1.CPUReport{Usage: 20}, Network: v1.NetworkReport{TotalUp: 150}},
|
||||
{UUID: "node-b", UpdatedAt: base, CPU: v1.CPUReport{Usage: 30}, Network: v1.NetworkReport{TotalUp: 200}},
|
||||
{UUID: "node-b", UpdatedAt: base.Add(time.Second), CPU: v1.CPUReport{Usage: 40}, Network: v1.NetworkReport{TotalUp: 260}},
|
||||
pending := []v2.Report{
|
||||
{UUID: "node-a", UpdatedAt: base, CPU: v2.CPUReport{Usage: 10}, Network: v2.NetworkReport{TotalUp: 100}},
|
||||
{UUID: "node-a", UpdatedAt: base, CPU: v2.CPUReport{Usage: 20}, Network: v2.NetworkReport{TotalUp: 150}},
|
||||
{UUID: "node-b", UpdatedAt: base, CPU: v2.CPUReport{Usage: 30}, Network: v2.NetworkReport{TotalUp: 200}},
|
||||
{UUID: "node-b", UpdatedAt: base.Add(time.Second), CPU: v2.CPUReport{Usage: 40}, Network: v2.NetworkReport{TotalUp: 260}},
|
||||
}
|
||||
|
||||
if err := writePendingReports(ctx, &pending); err != nil {
|
||||
@@ -292,11 +488,11 @@ func TestReportQueueFullReturnsError(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
useReportTestStore(t, nil)
|
||||
worker := &reportBatchWorker{
|
||||
queue: make(chan v1.Report, 1),
|
||||
queue: make(chan v2.Report, 1),
|
||||
requests: make(chan reportBatchRequest, 1),
|
||||
done: make(chan struct{}),
|
||||
}
|
||||
worker.queue <- v1.Report{UUID: "already-queued"}
|
||||
worker.queue <- v2.Report{UUID: "already-queued"}
|
||||
reportBatcherMu.Lock()
|
||||
reportBatcher = worker
|
||||
reportBatcherMu.Unlock()
|
||||
@@ -308,7 +504,7 @@ func TestReportQueueFullReturnsError(t *testing.T) {
|
||||
reportBatcherMu.Unlock()
|
||||
})
|
||||
|
||||
report := v1.Report{
|
||||
report := v2.Report{
|
||||
UUID: "realtime-node",
|
||||
UpdatedAt: time.Now().UTC(),
|
||||
}
|
||||
@@ -348,6 +544,10 @@ func TestRecordReconstructionUsesMetricSpecificAggregation(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestTrafficCounterDelta(t *testing.T) {
|
||||
const (
|
||||
oneGB = int64(1_000_000_000)
|
||||
oneTB = int64(1_000_000_000_000)
|
||||
)
|
||||
tests := []struct {
|
||||
name string
|
||||
current int64
|
||||
@@ -360,6 +560,11 @@ func TestTrafficCounterDelta(t *testing.T) {
|
||||
{name: "counter reset", current: 15, previous: 250, want: 15},
|
||||
{name: "negative current", current: -1, previous: 100, want: 0},
|
||||
{name: "negative previous", current: 15, previous: -1, want: 0},
|
||||
{name: "tb-scale monotonic", current: 2*oneTB + 5*oneGB, previous: 2 * oneTB, want: 5 * oneGB},
|
||||
{name: "32-bit wrap leftover", current: 800_000_000, previous: 3*oneGB + 900_000_000, want: 800_000_000},
|
||||
{name: "reboot leftover below cap", current: 8 * oneGB, previous: 2 * oneTB, want: 8 * oneGB},
|
||||
{name: "tb-scale tiny dip", current: 2*oneTB - 100, previous: 2 * oneTB, want: 0},
|
||||
{name: "tb-scale false reset", current: 2*oneTB + 5*oneGB - 1000, previous: 2*oneTB + 5*oneGB, want: 0},
|
||||
}
|
||||
for _, test := range tests {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
@@ -370,16 +575,211 @@ func TestTrafficCounterDelta(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestWriteReportRebasesTrafficAfterAgentRestart(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
s := useReportTestStore(t, nil)
|
||||
base := time.Now().UTC().Truncate(time.Minute).Add(5 * time.Second)
|
||||
report := v2.Report{
|
||||
UUID: "restarted-node",
|
||||
UpdatedAt: base,
|
||||
Uptime: 1000,
|
||||
Network: v2.NetworkReport{TotalUp: 100, TotalDown: 200},
|
||||
}
|
||||
if _, err := WriteReport(ctx, report); err != nil {
|
||||
t.Fatalf("write first report: %v", err)
|
||||
}
|
||||
|
||||
report.UpdatedAt = base.Add(3 * time.Second)
|
||||
report.Uptime = 1003
|
||||
report.Network.TotalUp = 150
|
||||
report.Network.TotalDown = 260
|
||||
if _, err := WriteReport(ctx, report); err != nil {
|
||||
t.Fatalf("write continuous report: %v", err)
|
||||
}
|
||||
|
||||
report.UpdatedAt = base.Add(6 * time.Second)
|
||||
report.Uptime = 1
|
||||
report.Network.TotalUp = 155
|
||||
report.Network.TotalDown = 265
|
||||
if _, err := WriteReport(ctx, report); err != nil {
|
||||
t.Fatalf("write report after agent restart: %v", err)
|
||||
}
|
||||
|
||||
report.UpdatedAt = base.Add(9 * time.Second)
|
||||
report.Uptime = 4
|
||||
report.Network.TotalUp = 180
|
||||
report.Network.TotalDown = 300
|
||||
if _, err := WriteReport(ctx, report); err != nil {
|
||||
t.Fatalf("write report after new baseline: %v", err)
|
||||
}
|
||||
|
||||
assertMetricValues(t, s, MetricTrafficUp, report.UUID, base.Add(-time.Second), base.Add(time.Minute), []float64{0, 50, 5, 25})
|
||||
assertMetricValues(t, s, MetricTrafficDown, report.UUID, base.Add(-time.Second), base.Add(time.Minute), []float64{0, 60, 5, 35})
|
||||
}
|
||||
|
||||
func TestWriteReportCountsTBScaleTrafficDeltas(t *testing.T) {
|
||||
const (
|
||||
oneGB = int64(1_000_000_000)
|
||||
oneTB = int64(1_000_000_000_000)
|
||||
)
|
||||
ctx := context.Background()
|
||||
s := useReportTestStore(t, nil)
|
||||
base := time.Now().UTC().Truncate(time.Minute).Add(5 * time.Second)
|
||||
report := v2.Report{
|
||||
UUID: "tb-node",
|
||||
UpdatedAt: base,
|
||||
Uptime: 10_000,
|
||||
Network: v2.NetworkReport{TotalUp: 2 * oneTB, TotalDown: 3 * oneTB},
|
||||
}
|
||||
if _, err := WriteReport(ctx, report); err != nil {
|
||||
t.Fatalf("write first report: %v", err)
|
||||
}
|
||||
|
||||
report.UpdatedAt = base.Add(3 * time.Second)
|
||||
report.Uptime = 10_003
|
||||
report.Network.TotalUp = 2*oneTB + 5*oneGB
|
||||
report.Network.TotalDown = 3*oneTB + 7*oneGB
|
||||
if _, err := WriteReport(ctx, report); err != nil {
|
||||
t.Fatalf("write growing report: %v", err)
|
||||
}
|
||||
|
||||
assertMetricValues(t, s, MetricTrafficUp, report.UUID, base.Add(-time.Second), base.Add(time.Minute), []float64{0, float64(5 * oneGB)})
|
||||
assertMetricValues(t, s, MetricTrafficDown, report.UUID, base.Add(-time.Second), base.Add(time.Minute), []float64{0, float64(7 * oneGB)})
|
||||
}
|
||||
|
||||
func TestWriteReportRestoresTBScaleCountersFromStore(t *testing.T) {
|
||||
const (
|
||||
oneGB = int64(1_000_000_000)
|
||||
oneTB = int64(1_000_000_000_000)
|
||||
)
|
||||
ctx := context.Background()
|
||||
policy := defaultRollupPolicy()
|
||||
s := useReportTestStore(t, &policy)
|
||||
base := time.Now().UTC().Truncate(time.Minute).Add(5 * time.Second)
|
||||
now := base.Add(45 * time.Second)
|
||||
report := v2.Report{
|
||||
UUID: "tb-restore-node",
|
||||
UpdatedAt: base,
|
||||
Uptime: 10_000,
|
||||
Network: v2.NetworkReport{TotalUp: 2*oneTB + 123, TotalDown: 4*oneTB + 456},
|
||||
}
|
||||
if _, err := WriteReport(ctx, report); err != nil {
|
||||
t.Fatalf("write first report: %v", err)
|
||||
}
|
||||
|
||||
restoredUp, hasUp, err := latestReportCounter(ctx, s, MetricNetTotalUp, report.UUID, now)
|
||||
if err != nil {
|
||||
t.Fatalf("restore upload counter: %v", err)
|
||||
}
|
||||
if !hasUp || restoredUp != report.Network.TotalUp {
|
||||
t.Fatalf("restored upload counter = %d (has=%v), want %d", restoredUp, hasUp, report.Network.TotalUp)
|
||||
}
|
||||
restoredDown, hasDown, err := latestReportCounter(ctx, s, MetricNetTotalDown, report.UUID, now)
|
||||
if err != nil {
|
||||
t.Fatalf("restore download counter: %v", err)
|
||||
}
|
||||
if !hasDown || restoredDown != report.Network.TotalDown {
|
||||
t.Fatalf("restored download counter = %d (has=%v), want %d", restoredDown, hasDown, report.Network.TotalDown)
|
||||
}
|
||||
|
||||
if _, err := s.Compact(ctx, now); err != nil {
|
||||
t.Fatalf("compact reports: %v", err)
|
||||
}
|
||||
deleteReportTrafficState(report.UUID)
|
||||
report.UpdatedAt = now
|
||||
report.Uptime = 10_045
|
||||
report.Network.TotalUp = 2*oneTB + 123 + 5*oneGB
|
||||
report.Network.TotalDown = 4*oneTB + 456 + 7*oneGB
|
||||
if _, err := WriteReport(ctx, report); err != nil {
|
||||
t.Fatalf("write after restoring tb-scale baseline: %v", err)
|
||||
}
|
||||
assertMetricValues(t, s, MetricTrafficUp, report.UUID, now.Add(-time.Second), now.Add(time.Second), []float64{float64(5 * oneGB)})
|
||||
assertMetricValues(t, s, MetricTrafficDown, report.UUID, now.Add(-time.Second), now.Add(time.Second), []float64{float64(7 * oneGB)})
|
||||
}
|
||||
|
||||
func TestWriteReportCountsTrafficAfterHighRateCounterWrap(t *testing.T) {
|
||||
const oneGB = int64(1_000_000_000)
|
||||
ctx := context.Background()
|
||||
s := useReportTestStore(t, nil)
|
||||
base := time.Now().UTC().Truncate(time.Minute).Add(5 * time.Second)
|
||||
report := v2.Report{
|
||||
UUID: "wrap-node",
|
||||
UpdatedAt: base,
|
||||
Uptime: 10_000,
|
||||
Network: v2.NetworkReport{TotalUp: 3*oneGB + 900_000_000, TotalDown: 3*oneGB + 800_000_000},
|
||||
}
|
||||
if _, err := WriteReport(ctx, report); err != nil {
|
||||
t.Fatalf("write first report: %v", err)
|
||||
}
|
||||
|
||||
report.UpdatedAt = base.Add(3 * time.Second)
|
||||
report.Uptime = 10_003
|
||||
report.Network.TotalUp = 800_000_000
|
||||
report.Network.TotalDown = 900_000_000
|
||||
if _, err := WriteReport(ctx, report); err != nil {
|
||||
t.Fatalf("write wrapped report: %v", err)
|
||||
}
|
||||
|
||||
assertMetricValues(t, s, MetricTrafficUp, report.UUID, base.Add(-time.Second), base.Add(time.Minute), []float64{0, 800_000_000})
|
||||
assertMetricValues(t, s, MetricTrafficDown, report.UUID, base.Add(-time.Second), base.Add(time.Minute), []float64{0, 900_000_000})
|
||||
}
|
||||
|
||||
func TestWriteReportIgnoresTinyDipOfTBScaleCounter(t *testing.T) {
|
||||
const (
|
||||
oneGB = int64(1_000_000_000)
|
||||
oneTB = int64(1_000_000_000_000)
|
||||
)
|
||||
ctx := context.Background()
|
||||
s := useReportTestStore(t, nil)
|
||||
base := time.Now().UTC().Truncate(time.Minute).Add(5 * time.Second)
|
||||
report := v2.Report{
|
||||
UUID: "tb-jitter-node",
|
||||
UpdatedAt: base,
|
||||
Uptime: 10_000,
|
||||
Network: v2.NetworkReport{TotalUp: 2 * oneTB, TotalDown: 3 * oneTB},
|
||||
}
|
||||
if _, err := WriteReport(ctx, report); err != nil {
|
||||
t.Fatalf("write first report: %v", err)
|
||||
}
|
||||
|
||||
report.UpdatedAt = base.Add(3 * time.Second)
|
||||
report.Uptime = 10_003
|
||||
report.Network.TotalUp = 2*oneTB + 5*oneGB
|
||||
report.Network.TotalDown = 3*oneTB + 7*oneGB
|
||||
if _, err := WriteReport(ctx, report); err != nil {
|
||||
t.Fatalf("write growing report: %v", err)
|
||||
}
|
||||
|
||||
report.UpdatedAt = base.Add(6 * time.Second)
|
||||
report.Uptime = 10_006
|
||||
report.Network.TotalUp = 2*oneTB + 5*oneGB - 1000
|
||||
report.Network.TotalDown = 3*oneTB + 7*oneGB - 2000
|
||||
if _, err := WriteReport(ctx, report); err != nil {
|
||||
t.Fatalf("write jitter report: %v", err)
|
||||
}
|
||||
|
||||
report.UpdatedAt = base.Add(9 * time.Second)
|
||||
report.Uptime = 10_009
|
||||
report.Network.TotalUp = 2*oneTB + 6*oneGB
|
||||
report.Network.TotalDown = 3*oneTB + 8*oneGB
|
||||
if _, err := WriteReport(ctx, report); err != nil {
|
||||
t.Fatalf("write recovered report: %v", err)
|
||||
}
|
||||
|
||||
assertMetricValues(t, s, MetricTrafficUp, report.UUID, base.Add(-time.Second), base.Add(time.Minute), []float64{0, float64(5 * oneGB), 0, float64(oneGB + 1000)})
|
||||
assertMetricValues(t, s, MetricTrafficDown, report.UUID, base.Add(-time.Second), base.Add(time.Minute), []float64{0, float64(7 * oneGB), 0, float64(oneGB + 2000)})
|
||||
}
|
||||
|
||||
func TestWriteReportNormalizesReceiveTimeToUTC(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
s := useReportTestStore(t, nil)
|
||||
local := time.FixedZone("UTC+8", 8*60*60)
|
||||
receiveTime := time.Now().In(local).Add(-10 * time.Second)
|
||||
report := v1.Report{
|
||||
report := v2.Report{
|
||||
UUID: "utc-report",
|
||||
UpdatedAt: receiveTime,
|
||||
CPU: v1.CPUReport{Usage: 10},
|
||||
Network: v1.NetworkReport{TotalUp: 1, TotalDown: 2},
|
||||
CPU: v2.CPUReport{Usage: 10},
|
||||
Network: v2.NetworkReport{TotalUp: 1, TotalDown: 2},
|
||||
}
|
||||
|
||||
saved, err := WriteReport(ctx, report)
|
||||
|
||||
@@ -112,9 +112,6 @@ func Run(ctx Context) error {
|
||||
if err := migrateLegacyClientInfo(db); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := migrateLegacyLoadNotification(db); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := migrateLegacyPingAllClientsExpansion(db); err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -175,14 +172,6 @@ func hasTableColumn(db *gorm.DB, tableName, columnName string) bool {
|
||||
return false
|
||||
}
|
||||
|
||||
func migrateLegacyLoadNotification(db *gorm.DB) error {
|
||||
if db.Migrator().HasColumn(&models.LoadNotification{}, "client") {
|
||||
logger.InfoArgs("migration", "[>0.1.4] Rebuilding LoadNotification table....")
|
||||
return db.Migrator().DropTable(&models.LoadNotification{})
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func migrateLegacyOidcConfig(db *gorm.DB) error {
|
||||
if db.Migrator().HasTable(&models.OidcProvider{}) {
|
||||
return nil
|
||||
|
||||
@@ -35,10 +35,7 @@ var legacyTimestampColumns = [...]timestampColumn{
|
||||
{table: "sessions", column: "expires"},
|
||||
{table: "sessions", column: "created_at"},
|
||||
{table: "logs", column: "time"},
|
||||
{table: "clipboards", column: "created_at"},
|
||||
{table: "clipboards", column: "updated_at"},
|
||||
{table: "offline_notifications", column: "last_notified"},
|
||||
{table: "load_notifications", column: "last_notified"},
|
||||
{table: "task_results", column: "finished_at"},
|
||||
{table: "task_results", column: "created_at"},
|
||||
{table: "records", column: "time"},
|
||||
|
||||
@@ -0,0 +1,47 @@
|
||||
// Package notifications wires the built-in notification channels into the
|
||||
// shared messageSender registry.
|
||||
package notifications
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
|
||||
"github.com/komari-monitor/komari/database"
|
||||
"github.com/komari-monitor/komari/database/models"
|
||||
"github.com/komari-monitor/komari/internal/managedconfig"
|
||||
"github.com/komari-monitor/komari/utils/messageSender"
|
||||
"github.com/komari-monitor/komari/utils/messageSender/webhook"
|
||||
)
|
||||
|
||||
// Initialize registers the built-in webhook channel and seeds its defaults.
|
||||
func Initialize() error {
|
||||
if err := webhook.Register(); err != nil {
|
||||
return err
|
||||
}
|
||||
for _, item := range messageSender.ListNotificationChannels() {
|
||||
if _, err := database.GetMessageSenderConfigByName(item.ID); err == nil {
|
||||
continue
|
||||
}
|
||||
values := map[string]any{}
|
||||
for _, field := range managedconfig.Items(item.Configuration) {
|
||||
if field.Key != "" {
|
||||
values[field.Key] = managedconfig.DefaultValue(field)
|
||||
}
|
||||
}
|
||||
raw, err := json.Marshal(values)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := database.SaveMessageSenderConfig(&models.MessageSenderProvider{
|
||||
Name: item.ID,
|
||||
Addition: string(raw),
|
||||
}); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Shutdown unregisters the built-in webhook channel.
|
||||
func Shutdown() error {
|
||||
return messageSender.UnregisterNotificationChannel("webhook")
|
||||
}
|
||||
@@ -5,10 +5,10 @@ import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
|
||||
"github.com/komari-monitor/komari/database/dbcore"
|
||||
"github.com/komari-monitor/komari/database/models"
|
||||
"github.com/komari-monitor/komari/internal/managedconfig"
|
||||
"gorm.io/gorm"
|
||||
)
|
||||
|
||||
@@ -26,7 +26,11 @@ func GetConfiguration(short string) (map[string]any, error) {
|
||||
if err != nil {
|
||||
return values, nil // not installed (or unreadable): keep saved values
|
||||
}
|
||||
mergeConfigurationDefaults(values, info.Configuration)
|
||||
items := managedconfig.Items(info.Configuration)
|
||||
mergeConfigurationDefaults(values, items)
|
||||
if err := managedconfig.ResolveForOutput(values, items); err != nil {
|
||||
return nil, fmt.Errorf("resolve plugin configuration: %w", err)
|
||||
}
|
||||
return values, nil
|
||||
}
|
||||
|
||||
@@ -48,18 +52,7 @@ func savedConfiguration(short string) (map[string]any, error) {
|
||||
|
||||
// mergeConfigurationDefaults fills manifest-declared defaults for keys that
|
||||
// are missing from the saved values, following the theme merge rules.
|
||||
func mergeConfigurationDefaults(values map[string]any, configuration models.Configuration) {
|
||||
if configuration.Type != models.ThemeConfigurationManaged {
|
||||
return
|
||||
}
|
||||
raw, err := json.Marshal(configuration.Data)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
var items []models.ManagedThemeConfigurationItem
|
||||
if err := json.Unmarshal(raw, &items); err != nil {
|
||||
return
|
||||
}
|
||||
func mergeConfigurationDefaults(values map[string]any, items []models.ManagedThemeConfigurationItem) {
|
||||
for _, item := range items {
|
||||
if item.Key == "" {
|
||||
continue
|
||||
@@ -67,38 +60,10 @@ func mergeConfigurationDefaults(values map[string]any, configuration models.Conf
|
||||
if _, exists := values[item.Key]; exists {
|
||||
continue
|
||||
}
|
||||
values[item.Key] = configurationDefault(item)
|
||||
values[item.Key] = managedconfig.DefaultValue(item)
|
||||
}
|
||||
}
|
||||
|
||||
// configurationDefault resolves one item's effective default, mirroring the
|
||||
// theme rules: select falls back to its first option; nil defaults become
|
||||
// 0 for number, false for switch and "" otherwise.
|
||||
func configurationDefault(item models.ManagedThemeConfigurationItem) any {
|
||||
def := item.Default
|
||||
if item.Type == "select" {
|
||||
if def == nil || def == "" {
|
||||
if item.Options != "" {
|
||||
opts := strings.Split(item.Options, ",")
|
||||
if len(opts) > 0 {
|
||||
return strings.TrimSpace(opts[0])
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if def == nil {
|
||||
switch item.Type {
|
||||
case "number":
|
||||
return float64(0) // JSON numbers decode as float64
|
||||
case "switch":
|
||||
return false
|
||||
default:
|
||||
return ""
|
||||
}
|
||||
}
|
||||
return def
|
||||
}
|
||||
|
||||
// SaveConfiguration upserts the configuration values of one plugin.
|
||||
func SaveConfiguration(short string, data map[string]any) error {
|
||||
if data == nil {
|
||||
|
||||
@@ -14,6 +14,8 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/gin-gonic/gin"
|
||||
"github.com/komari-monitor/komari/database/dbcore"
|
||||
"github.com/komari-monitor/komari/database/models"
|
||||
"github.com/komari-monitor/komari/pkg/rpc"
|
||||
)
|
||||
|
||||
@@ -398,6 +400,58 @@ func TestGetConfigurationMergesManifestDefaults(t *testing.T) {
|
||||
t.Errorf("mode still default = %v", values["mode"])
|
||||
}
|
||||
}
|
||||
|
||||
func TestGetConfigurationResolvesSelectorValues(t *testing.T) {
|
||||
withTempDataDir(t)
|
||||
if err := dbcore.GetDBInstance().Create(&models.Client{UUID: "node-a"}).Error; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := dbcore.GetDBInstance().Create(&models.PingTask{Id: 8}).Error; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
zipPath := writePluginZip(t, map[string]string{
|
||||
"komari-plugin.json": `{"name":"Cfg","short":"cfg","version":"1.0.0","configuration":{"type":"managed","data":[
|
||||
{"key":"nodes","name":"Nodes","type":"nodes"},
|
||||
{"key":"tasks","name":"Tasks","type":"pingtasks"}
|
||||
]}}`,
|
||||
"script.js": `function load() {}`,
|
||||
})
|
||||
if _, err := InstallZip(zipPath); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := SaveConfiguration("cfg", map[string]any{
|
||||
"nodes": `["node-a","deleted-node"]`,
|
||||
"tasks": `[8,7]`,
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
values, err := GetConfiguration("cfg")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
nodes, ok := values["nodes"].([]string)
|
||||
if !ok || len(nodes) != 1 || nodes[0] != "node-a" {
|
||||
t.Fatalf("nodes = %#v", values["nodes"])
|
||||
}
|
||||
tasks, ok := values["tasks"].([]uint)
|
||||
if !ok || len(tasks) != 1 || tasks[0] != 8 {
|
||||
t.Fatalf("tasks = %#v", values["tasks"])
|
||||
}
|
||||
|
||||
var stored models.PluginConfiguration
|
||||
if err := dbcore.GetDBInstance().Where("short = ?", "cfg").First(&stored).Error; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var raw map[string]any
|
||||
if err := json.Unmarshal([]byte(stored.Data), &raw); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if raw["nodes"] != `["node-a","deleted-node"]` || raw["tasks"] != `[8,7]` {
|
||||
t.Fatalf("stored = %#v", raw)
|
||||
}
|
||||
}
|
||||
|
||||
func TestGetConfigFromScript(t *testing.T) {
|
||||
withTempDataDir(t)
|
||||
gin.SetMode(gin.TestMode)
|
||||
|
||||
+11
-2
@@ -392,9 +392,18 @@ func headerToMap(header http.Header) map[string]any {
|
||||
return out
|
||||
}
|
||||
|
||||
// isUpgradeRequest reports whether r is a genuine protocol-upgrade request:
|
||||
// both Connection: upgrade and an Upgrade header must be present (RFC 9110 §7.8).
|
||||
func isUpgradeRequest(r *http.Request) bool {
|
||||
return strings.EqualFold(strings.TrimSpace(r.Header.Get("Connection")), "upgrade") ||
|
||||
strings.EqualFold(r.Header.Get("Upgrade"), "websocket")
|
||||
if r.Header.Get("Upgrade") == "" {
|
||||
return false
|
||||
}
|
||||
for _, token := range strings.Split(r.Header.Get("Connection"), ",") {
|
||||
if strings.EqualFold(strings.TrimSpace(token), "upgrade") {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// bufferedResponseWriter captures status/headers/body so response hooks can
|
||||
|
||||
@@ -41,7 +41,7 @@ func validateManifest(info *models.Plugin) error {
|
||||
if !validShort(info.Short) {
|
||||
return fmt.Errorf("plugin short %q is invalid: only letters, digits, '_' and '-' are allowed", info.Short)
|
||||
}
|
||||
if !localizedText(info.Name) {
|
||||
if !models.IsLocalizedText(info.Name) {
|
||||
return fmt.Errorf("plugin name is required")
|
||||
}
|
||||
if info.Entry == "" {
|
||||
@@ -90,7 +90,7 @@ func validatePage(page *models.PluginPage) error {
|
||||
default:
|
||||
return fmt.Errorf("plugin page type %q is invalid: use \"iframe\" or \"redirect\"", page.Type)
|
||||
}
|
||||
if !localizedText(page.Title) {
|
||||
if !models.IsLocalizedText(page.Title) {
|
||||
return fmt.Errorf("plugin page %q requires a title", page.Title)
|
||||
}
|
||||
return nil
|
||||
@@ -113,23 +113,3 @@ func isSafeInternalPath(target string) bool {
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// localizedText reports whether v is a usable name/description/author value:
|
||||
// a non-empty string, or an i18n object with at least one non-empty value.
|
||||
func localizedText(v any) bool {
|
||||
switch t := v.(type) {
|
||||
case nil:
|
||||
return false
|
||||
case string:
|
||||
return strings.TrimSpace(t) != ""
|
||||
case map[string]any:
|
||||
for _, value := range t {
|
||||
if s, ok := value.(string); ok && strings.TrimSpace(s) != "" {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,160 @@
|
||||
package plugin
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/dop251/goja"
|
||||
"github.com/komari-monitor/komari/database/models"
|
||||
"github.com/komari-monitor/komari/pkg/jsruntime"
|
||||
"github.com/komari-monitor/komari/utils/messageSender"
|
||||
)
|
||||
|
||||
// registerNotificationChannel registers one channel owned by a plugin. The
|
||||
// channel is tied to the current runtime, so unloading the plugin removes it.
|
||||
func (m *Manager) registerNotificationChannel(
|
||||
short string,
|
||||
id string,
|
||||
configuration models.Configuration,
|
||||
handler goja.Callable,
|
||||
host *jsruntime.Host,
|
||||
) error {
|
||||
inst := m.instanceFor(short)
|
||||
if inst == nil {
|
||||
return fmt.Errorf("plugin %q is not loaded", short)
|
||||
}
|
||||
if host == nil {
|
||||
return fmt.Errorf("plugin %q notification runtime is unavailable", short)
|
||||
}
|
||||
notificationChannel := &pluginNotificationChannel{
|
||||
short: short,
|
||||
host: host,
|
||||
handler: handler,
|
||||
}
|
||||
inst.mu.Lock()
|
||||
defer inst.mu.Unlock()
|
||||
if inst.runtime == nil {
|
||||
return fmt.Errorf("plugin %q notification runtime is closed", short)
|
||||
}
|
||||
if err := messageSender.RegisterNotificationChannel(id, configuration, notificationChannel); err != nil {
|
||||
return err
|
||||
}
|
||||
if inst.notificationChannels == nil {
|
||||
inst.notificationChannels = make(map[string]struct{})
|
||||
}
|
||||
inst.notificationChannels[id] = struct{}{}
|
||||
return nil
|
||||
}
|
||||
|
||||
type pluginNotificationChannel struct {
|
||||
short string
|
||||
host *jsruntime.Host
|
||||
handler goja.Callable
|
||||
}
|
||||
|
||||
func (c *pluginNotificationChannel) Send(ctx context.Context, notification messageSender.Notification, config map[string]any) error {
|
||||
if c.host == nil || c.handler == nil {
|
||||
return fmt.Errorf("plugin %q notification channel is closed", c.short)
|
||||
}
|
||||
result := make(chan error, 1)
|
||||
var once sync.Once
|
||||
finish := func(err error) {
|
||||
once.Do(func() { result <- err })
|
||||
}
|
||||
|
||||
queued := c.host.RunOnLoop(func(vm *goja.Runtime) {
|
||||
var value goja.Value
|
||||
runErr := c.host.RunJob(vm, "plugin notification "+c.short, func() error {
|
||||
raw, err := json.Marshal(notification)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
var notificationValue map[string]any
|
||||
if err := json.Unmarshal(raw, ¬ificationValue); err != nil {
|
||||
return err
|
||||
}
|
||||
var callErr error
|
||||
value, callErr = c.handler(
|
||||
goja.Undefined(),
|
||||
vm.ToValue(notificationValue),
|
||||
vm.ToValue(config),
|
||||
)
|
||||
return callErr
|
||||
})
|
||||
if runErr != nil {
|
||||
finish(runErr)
|
||||
return
|
||||
}
|
||||
_, ok := value.Export().(*goja.Promise)
|
||||
if !ok {
|
||||
finish(nil)
|
||||
return
|
||||
}
|
||||
then, ok := goja.AssertFunction(value.ToObject(vm).Get("then"))
|
||||
if !ok {
|
||||
finish(fmt.Errorf("plugin %q notification channel returned a Promise without then", c.short))
|
||||
return
|
||||
}
|
||||
onFulfilled := vm.ToValue(func(goja.FunctionCall) goja.Value {
|
||||
finish(nil)
|
||||
return goja.Undefined()
|
||||
})
|
||||
onRejected := vm.ToValue(func(call goja.FunctionCall) goja.Value {
|
||||
finish(fmt.Errorf("plugin %q notification channel failed: %v", c.short, call.Argument(0)))
|
||||
return goja.Undefined()
|
||||
})
|
||||
if _, err := then(value, onFulfilled, onRejected); err != nil {
|
||||
finish(fmt.Errorf("plugin %q notification channel Promise failed: %w", c.short, err))
|
||||
}
|
||||
})
|
||||
if !queued {
|
||||
return fmt.Errorf("plugin %q notification runtime is closed", c.short)
|
||||
}
|
||||
|
||||
timeout := c.host.Timeout()
|
||||
if deadline, ok := ctx.Deadline(); ok {
|
||||
if remaining := time.Until(deadline); remaining < timeout {
|
||||
timeout = remaining
|
||||
}
|
||||
}
|
||||
if timeout <= 0 {
|
||||
return context.DeadlineExceeded
|
||||
}
|
||||
timer := time.NewTimer(timeout)
|
||||
defer timer.Stop()
|
||||
select {
|
||||
case err := <-result:
|
||||
return err
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
case <-timer.C:
|
||||
return fmt.Errorf("plugin %q notification channel timed out", c.short)
|
||||
}
|
||||
}
|
||||
|
||||
func (c *pluginNotificationChannel) Unload() error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (m *Manager) unregisterNotificationChannels(inst *Instance) error {
|
||||
if inst == nil {
|
||||
return nil
|
||||
}
|
||||
inst.mu.Lock()
|
||||
ids := make([]string, 0, len(inst.notificationChannels))
|
||||
for id := range inst.notificationChannels {
|
||||
ids = append(ids, id)
|
||||
}
|
||||
inst.notificationChannels = nil
|
||||
inst.mu.Unlock()
|
||||
var firstErr error
|
||||
for _, id := range ids {
|
||||
if err := messageSender.UnregisterNotificationChannel(id); err != nil && firstErr == nil {
|
||||
firstErr = err
|
||||
}
|
||||
}
|
||||
return firstErr
|
||||
}
|
||||
+36
-11
@@ -113,15 +113,16 @@ type Manager struct {
|
||||
|
||||
// Instance is one loaded plugin runtime plus its host bindings.
|
||||
type Instance struct {
|
||||
mu sync.RWMutex
|
||||
info models.Plugin
|
||||
dir string
|
||||
runtime *jsruntime.Runtime
|
||||
host *jsruntime.Host
|
||||
handlers map[string]goja.Callable // "METHOD path" -> current route handler
|
||||
statics map[string]*staticConfig // mount path -> static folder config
|
||||
rpcMethods map[string]goja.Callable // registered RPC method -> JS handler
|
||||
cronJobs []string // scheduler job names, removed on unload
|
||||
mu sync.RWMutex
|
||||
info models.Plugin
|
||||
dir string
|
||||
runtime *jsruntime.Runtime
|
||||
host *jsruntime.Host
|
||||
handlers map[string]goja.Callable // "METHOD path" -> current route handler
|
||||
statics map[string]*staticConfig // mount path -> static folder config
|
||||
rpcMethods map[string]goja.Callable // registered RPC method -> JS handler
|
||||
cronJobs []string // scheduler job names, removed on unload
|
||||
notificationChannels map[string]struct{}
|
||||
}
|
||||
|
||||
// global is the process-wide plugin manager.
|
||||
@@ -153,6 +154,13 @@ func SetEnabled(short string, enabled, approved bool) error {
|
||||
return global.setEnabled(short, enabled, approved)
|
||||
}
|
||||
|
||||
// Reload unloads and loads an enabled plugin. The load hook completes before
|
||||
// this function returns; a failed load is persisted as the plugin's last
|
||||
// error and disables the plugin, matching the normal enable/startup path.
|
||||
func Reload(short string) error {
|
||||
return global.reload(short)
|
||||
}
|
||||
|
||||
// CloseAll unloads every loaded plugin. Used during server shutdown.
|
||||
func CloseAll() error {
|
||||
return global.closeAll()
|
||||
@@ -298,8 +306,8 @@ func (m *Manager) load(short string) error {
|
||||
|
||||
if rt.HasFunction("load") {
|
||||
if err := rt.CallVoid("load"); err != nil {
|
||||
rt.Close()
|
||||
m.dropInstance(short, inst)
|
||||
rt.Close()
|
||||
return fmt.Errorf("plugin %q load() failed: %w", short, err)
|
||||
}
|
||||
}
|
||||
@@ -321,11 +329,15 @@ func (m *Manager) load(short string) error {
|
||||
// unload behavior. The caller must not hold any Manager lock.
|
||||
func (m *Manager) dropInstance(short string, inst *Instance) {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
if m.instances[short] != inst {
|
||||
m.mu.Unlock()
|
||||
return // replaced by a newer load or already unloaded
|
||||
}
|
||||
delete(m.instances, short)
|
||||
m.mu.Unlock()
|
||||
_ = m.unregisterNotificationChannels(inst)
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
inst.mu.Lock()
|
||||
for method := range inst.rpcMethods {
|
||||
rpc.Unregister(method)
|
||||
@@ -378,6 +390,9 @@ func (m *Manager) unload(short string) error {
|
||||
}
|
||||
}
|
||||
}
|
||||
if err := m.unregisterNotificationChannels(inst); err != nil && unloadErr == nil {
|
||||
unloadErr = err
|
||||
}
|
||||
m.mu.Lock()
|
||||
inst.mu.Lock()
|
||||
for method := range inst.rpcMethods {
|
||||
@@ -521,6 +536,16 @@ func (m *Manager) restartPlugin(short string) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (m *Manager) reload(short string) error {
|
||||
if !m.stateStore().get(short).Enabled {
|
||||
return nil
|
||||
}
|
||||
if err := m.unload(short); err != nil && !errors.Is(err, errNotLoaded) {
|
||||
return err
|
||||
}
|
||||
return m.restartPlugin(short)
|
||||
}
|
||||
|
||||
// disableWithError persists the auto-disable state for one plugin and wraps
|
||||
// the load failure for the startup error summary.
|
||||
func (m *Manager) disableWithError(short string, st PluginState, err error) error {
|
||||
|
||||
@@ -3,6 +3,7 @@ package plugin
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/http"
|
||||
@@ -17,6 +18,7 @@ import (
|
||||
"github.com/dop251/goja_nodejs/buffer"
|
||||
"github.com/dop251/goja_nodejs/require"
|
||||
"github.com/gin-gonic/gin"
|
||||
"github.com/komari-monitor/komari/database/models"
|
||||
"github.com/komari-monitor/komari/pkg/jsruntime"
|
||||
"github.com/komari-monitor/komari/pkg/jsruntime/httpbody"
|
||||
"github.com/komari-monitor/komari/pkg/rpc"
|
||||
@@ -52,6 +54,8 @@ import (
|
||||
// result or rejects with an Error
|
||||
// carrying code/message/data
|
||||
// server.registerRPC(method, handler) register a plugin-owned RPC method
|
||||
// server.registerNotificationChannel(id, configuration, handler)
|
||||
// register a notification channel
|
||||
// server.getConfig() resolve the saved plugin configuration
|
||||
// server.cron(expr, fn) run fn on the plugin event loop each
|
||||
// time the cron expression fires
|
||||
@@ -143,7 +147,12 @@ func (m *Manager) registerServerModule(host *jsruntime.Host, registry *require.R
|
||||
_ = reject(jsRPCError(vm, resp.Error))
|
||||
return
|
||||
}
|
||||
_ = resolve(resp.Result)
|
||||
result, err := normalizeRPCValue(resp.Result)
|
||||
if err != nil {
|
||||
_ = reject(vm.NewGoError(err))
|
||||
return
|
||||
}
|
||||
_ = resolve(result)
|
||||
})
|
||||
}()
|
||||
return vm.ToValue(promise)
|
||||
@@ -216,6 +225,32 @@ func (m *Manager) registerServerModule(host *jsruntime.Host, registry *require.R
|
||||
}
|
||||
return goja.Undefined()
|
||||
})
|
||||
_ = exports.Set("registerNotificationChannel", func(call goja.FunctionCall) goja.Value {
|
||||
id := strings.TrimSpace(call.Argument(0).String())
|
||||
handler, ok := goja.AssertFunction(call.Argument(2))
|
||||
if id == "" {
|
||||
panic(vm.NewTypeError("server.registerNotificationChannel requires a channel id"))
|
||||
}
|
||||
if !ok {
|
||||
panic(vm.NewTypeError("server.registerNotificationChannel requires a function handler"))
|
||||
}
|
||||
rawConfiguration, err := exportJSValue(call.Argument(1))
|
||||
if err != nil {
|
||||
panic(vm.NewTypeError("server.registerNotificationChannel requires a managed configuration object"))
|
||||
}
|
||||
configurationJSON, err := json.Marshal(rawConfiguration)
|
||||
if err != nil {
|
||||
panic(vm.NewTypeError("server.registerNotificationChannel requires a managed configuration object"))
|
||||
}
|
||||
var configuration models.Configuration
|
||||
if err := json.Unmarshal(configurationJSON, &configuration); err != nil {
|
||||
panic(vm.NewTypeError("server.registerNotificationChannel requires a managed configuration object"))
|
||||
}
|
||||
if err := m.registerNotificationChannel(inst.info.Short, id, configuration, handler, host); err != nil {
|
||||
panic(vm.NewGoError(err))
|
||||
}
|
||||
return goja.Undefined()
|
||||
})
|
||||
_ = exports.Set("getConfig", func(call goja.FunctionCall) goja.Value {
|
||||
promise, resolve, reject := vm.NewPromise()
|
||||
go func() {
|
||||
@@ -266,6 +301,19 @@ func jsRPCError(vm *goja.Runtime, e *rpc.JsonRpcError) *goja.Object {
|
||||
return obj
|
||||
}
|
||||
|
||||
func normalizeRPCValue(value any) (any, error) {
|
||||
data, err := json.Marshal(value)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("serialize RPC result: %w", err)
|
||||
}
|
||||
|
||||
var normalized any
|
||||
if err := json.Unmarshal(data, &normalized); err != nil {
|
||||
return nil, fmt.Errorf("decode RPC result: %w", err)
|
||||
}
|
||||
return normalized, nil
|
||||
}
|
||||
|
||||
// registerRoute registers (or reuses) a gin route slot for one plugin. It is
|
||||
// called from the plugin's own event loop during script evaluation and takes
|
||||
// the manager lock itself. The critical section is short and never waits on
|
||||
|
||||
@@ -0,0 +1,99 @@
|
||||
package plugin
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/gin-gonic/gin"
|
||||
)
|
||||
|
||||
// TestIsUpgradeRequest 覆盖升级请求判定:真正的协议升级必须同时携带
|
||||
// Connection: upgrade 记号与 Upgrade 目标协议头(RFC 9110 §7.8);仅带
|
||||
// 其中一个头的普通请求(如反代模板给每个请求无条件附加
|
||||
// Connection: upgrade)不得被视为升级。
|
||||
func TestIsUpgradeRequest(t *testing.T) {
|
||||
cases := []struct {
|
||||
name string
|
||||
connection string
|
||||
upgrade string
|
||||
want bool
|
||||
}{
|
||||
{"websocket upgrade with both headers", "upgrade", "websocket", true},
|
||||
{"case insensitive", "Upgrade", "WebSocket", true},
|
||||
{"connection list keeps other tokens", "keep-alive, upgrade", "websocket", true},
|
||||
{"bare connection upgrade is not an upgrade", "upgrade", "", false},
|
||||
{"upgrade header alone is not an upgrade", "", "websocket", false},
|
||||
{"plain request", "", "", false},
|
||||
{"keep-alive only", "keep-alive", "", false},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
req := httptest.NewRequest(http.MethodGet, "/", nil)
|
||||
if tc.connection != "" {
|
||||
req.Header.Set("Connection", tc.connection)
|
||||
}
|
||||
if tc.upgrade != "" {
|
||||
req.Header.Set("Upgrade", tc.upgrade)
|
||||
}
|
||||
if got := isUpgradeRequest(req); got != tc.want {
|
||||
t.Fatalf("isUpgradeRequest(Connection=%q, Upgrade=%q) = %v, want %v",
|
||||
tc.connection, tc.upgrade, got, tc.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestInjectSurvivesBareConnectionUpgradeHeader 回归:部分反代模板(如 1Panel
|
||||
// 的 OpenResty 站点模板)会给每个代理请求无条件附加 Connection: upgrade。
|
||||
// 这类普通页面请求必须照常获得插件 HTML 注入;只有同时携带两个头的真实升级
|
||||
// 才走放行路径。
|
||||
func TestInjectSurvivesBareConnectionUpgradeHeader(t *testing.T) {
|
||||
withTempDataDir(t)
|
||||
gin.SetMode(gin.TestMode)
|
||||
engine := gin.New()
|
||||
engine.GET("/admin/dashboard", func(c *gin.Context) {
|
||||
c.Data(http.StatusOK, "text/html; charset=utf-8",
|
||||
[]byte("<!DOCTYPE html><html><head></head><body></body></html>"))
|
||||
})
|
||||
Init(engine)
|
||||
|
||||
zipPath := writePluginZip(t, map[string]string{
|
||||
"komari-plugin.json": `{"name":"Guard","short":"guard","version":"1.0.0","permissions":{"allowHTMLInject":true,"timeout":5}}`,
|
||||
"script.js": `
|
||||
const server = require("server");
|
||||
function load() {
|
||||
server.injectHTML("", "<script>window.__guard = 1;</script>");
|
||||
}
|
||||
`,
|
||||
})
|
||||
if _, err := InstallZip(zipPath); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := SetEnabled("guard", true, true); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer func() { _ = SetEnabled("guard", false, false) }()
|
||||
|
||||
handler := HTMLInjectHandler(engine)
|
||||
|
||||
// 仅 Connection: upgrade(反代常见行为)→ 注入照常生效
|
||||
req := httptest.NewRequest(http.MethodGet, "/admin/dashboard", nil)
|
||||
req.Header.Set("Connection", "upgrade")
|
||||
rec := httptest.NewRecorder()
|
||||
handler.ServeHTTP(rec, req)
|
||||
if !strings.Contains(rec.Body.String(), "__guard") {
|
||||
t.Fatalf("bare Connection: upgrade must not skip injection, body = %q", rec.Body.String())
|
||||
}
|
||||
|
||||
// 两个头齐全的真实升级 → 放行,不注入
|
||||
req2 := httptest.NewRequest(http.MethodGet, "/admin/dashboard", nil)
|
||||
req2.Header.Set("Connection", "upgrade")
|
||||
req2.Header.Set("Upgrade", "websocket")
|
||||
rec2 := httptest.NewRecorder()
|
||||
handler.ServeHTTP(rec2, req2)
|
||||
if strings.Contains(rec2.Body.String(), "__guard") {
|
||||
t.Fatalf("real upgrade must skip injection, body = %q", rec2.Body.String())
|
||||
}
|
||||
}
|
||||
@@ -22,7 +22,7 @@ const (
|
||||
)
|
||||
|
||||
// wsFrameHookTimeout bounds one frame-level callback. The agent read pumps
|
||||
// run on deadlines (v1: 11s, v2: readWait); a frame hook must never stall
|
||||
// run on the agent read deadline; a frame hook must never stall
|
||||
// them, so the wait is cut short and the frame passes through untouched.
|
||||
const wsFrameHookTimeout = time.Second
|
||||
|
||||
|
||||
@@ -5,9 +5,9 @@ import (
|
||||
"fmt"
|
||||
|
||||
"github.com/komari-monitor/komari/database/auditlog"
|
||||
"github.com/komari-monitor/komari/internal/notifications"
|
||||
"github.com/komari-monitor/komari/utils/geoip"
|
||||
logger "github.com/komari-monitor/komari/utils/log"
|
||||
"github.com/komari-monitor/komari/utils/messageSender"
|
||||
"github.com/komari-monitor/komari/web/oauth"
|
||||
)
|
||||
|
||||
@@ -18,8 +18,12 @@ func (a *App) InitProviders() error {
|
||||
go geoip.InitGeoIp()
|
||||
a.addCleanup("geoip", func(context.Context) error { return geoip.Shutdown() })
|
||||
|
||||
messageSender.Initialize()
|
||||
a.addCleanup("message-sender", func(context.Context) error { return messageSender.Shutdown() })
|
||||
if err := notifications.Initialize(); err != nil {
|
||||
return fmt.Errorf("initialize notification channels: %w", err)
|
||||
}
|
||||
a.addCleanup("notification-channels", func(context.Context) error {
|
||||
return notifications.Shutdown()
|
||||
})
|
||||
return nil
|
||||
}
|
||||
|
||||
|
||||
@@ -14,7 +14,6 @@ import (
|
||||
"github.com/komari-monitor/komari/database"
|
||||
"github.com/komari-monitor/komari/database/accounts"
|
||||
"github.com/komari-monitor/komari/database/auditlog"
|
||||
d_notification "github.com/komari-monitor/komari/database/notification"
|
||||
"github.com/komari-monitor/komari/database/tasks"
|
||||
"github.com/komari-monitor/komari/internal/config"
|
||||
"github.com/komari-monitor/komari/internal/lifecycle"
|
||||
@@ -23,7 +22,6 @@ import (
|
||||
"github.com/komari-monitor/komari/internal/scheduler"
|
||||
"github.com/komari-monitor/komari/utils/geoip"
|
||||
logger "github.com/komari-monitor/komari/utils/log"
|
||||
"github.com/komari-monitor/komari/utils/messageSender"
|
||||
"github.com/komari-monitor/komari/utils/notifier"
|
||||
"github.com/komari-monitor/komari/web/api"
|
||||
"github.com/komari-monitor/komari/web/oauth"
|
||||
@@ -76,11 +74,6 @@ func (a *App) registerReloadHandlers(cors *security.CorsController) {
|
||||
go geoip.InitGeoIp()
|
||||
}
|
||||
})
|
||||
a.reload.Register("message-sender", func(event config.ConfigEvent) {
|
||||
if event.IsChanged(config.NotificationMethodKey) {
|
||||
go messageSender.Initialize()
|
||||
}
|
||||
})
|
||||
a.reload.Register("cors", func(event config.ConfigEvent) { cors.Update(event) })
|
||||
}
|
||||
|
||||
@@ -188,9 +181,6 @@ func registerScheduledWork() {
|
||||
if err := tasks.ReloadPingSchedule(); err != nil {
|
||||
logger.ErrorArgs("server", "Failed to reload ping schedule:", err)
|
||||
}
|
||||
if err := d_notification.ReloadLoadNotificationSchedule(); err != nil {
|
||||
logger.ErrorArgs("server", "Failed to reload load notification schedule:", err)
|
||||
}
|
||||
if err := scheduler.AddFunc("records:cleanup", "@every 30m", cleanupScheduledData); err != nil {
|
||||
logger.ErrorArgs("server", "Failed to add cleanup scheduled task:", err)
|
||||
}
|
||||
@@ -206,7 +196,6 @@ func registerScheduledWork() {
|
||||
if err := scheduler.AddFunc("notifier:expire", "0 0 9 * * *", notifier.CheckExpireScheduledWork); err != nil {
|
||||
logger.ErrorArgs("server", "Failed to add expire notification task:", err)
|
||||
}
|
||||
notifier.InitTrafficReportSchedule()
|
||||
}
|
||||
|
||||
const taskResultRetentionDays = 30
|
||||
|
||||
@@ -365,7 +365,7 @@ const value = Buffer.from("komari").toString("hex");
|
||||
| `domainToASCII/domainToUnicode` | `可用` |
|
||||
| 旧式 `parse/format/resolve/resolveObject/urlToHttpOptions` | `未实现` |
|
||||
|
||||
`URL` 和 `URLSearchParams` 不会自动成为全局变量,应从模块取得:
|
||||
`URL` 和 `URLSearchParams` 会作为全局变量注入,也可以从模块取得:
|
||||
|
||||
```js
|
||||
const { URL, URLSearchParams } = require("node:url");
|
||||
@@ -705,7 +705,8 @@ function runtimeInfo() {
|
||||
| ChildProcess `kill/ref/unref/disconnect/send` | `部分实现` 或调用抛错,见下文 |
|
||||
|
||||
支持的 options:`cwd`、`env`、`shell`、`timeout`、`encoding`、`maxBuffer`。
|
||||
`cwd` 受 `BaseDir` 限制;请求的 timeout 和 maxBuffer 只能收紧 Go 侧全局上限,不能放大。
|
||||
`cwd` 受 `BaseDir` 限制;子进程默认不继承 runtime 的 `Timeout`,只有显式传入 `timeout` 时才会自动结束。
|
||||
请求的 `maxBuffer` 只能收紧 Go 侧全局上限,不能放大。
|
||||
`stdio/input/detached/uid/gid/windowsHide/windowsVerbatimArguments/serialization` 等未实现。
|
||||
|
||||
- `spawn()` 的 `stdin` 是 `stream.Writable`(写入经后台写队列串行执行,`end()` 后关闭管道),
|
||||
|
||||
@@ -163,7 +163,7 @@ func (m *Module) childCommandArguments(vm *goja.Runtime, call goja.FunctionCall,
|
||||
}
|
||||
|
||||
func (m *Module) childOptions(vm *goja.Runtime, value goja.Value) childCommandOptions {
|
||||
options := childCommandOptions{cwd: m.filesystem.Cwd(), env: os.Environ(), timeout: m.runtime.Timeout(), maxBuffer: m.maxOutput}
|
||||
options := childCommandOptions{cwd: m.filesystem.Cwd(), env: os.Environ(), maxBuffer: m.maxOutput}
|
||||
if value == nil || goja.IsUndefined(value) || goja.IsNull(value) {
|
||||
return options
|
||||
}
|
||||
@@ -190,7 +190,7 @@ func (m *Module) childOptions(vm *goja.Runtime, value goja.Value) childCommandOp
|
||||
}
|
||||
if timeoutValue := object.Get("timeout"); timeoutValue != nil && timeoutValue.ToInteger() > 0 {
|
||||
timeout := timeoutValue.ToInteger()
|
||||
options.timeout = min(options.timeout, time.Duration(timeout)*time.Millisecond)
|
||||
options.timeout = time.Duration(timeout) * time.Millisecond
|
||||
}
|
||||
if encoding := object.Get("encoding"); encoding != nil && !goja.IsUndefined(encoding) && !goja.IsNull(encoding) {
|
||||
options.encoding = encoding.String()
|
||||
|
||||
@@ -5,6 +5,7 @@ package console
|
||||
// substitutions. assert(false, ...) and trace() include a stack trace.
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"strings"
|
||||
@@ -121,11 +122,27 @@ func formatConsoleValues(values []goja.Value) string {
|
||||
}
|
||||
parts := make([]string, 0, len(args))
|
||||
for _, arg := range args {
|
||||
parts = append(parts, fmt.Sprint(arg))
|
||||
parts = append(parts, formatConsoleValue(arg))
|
||||
}
|
||||
return strings.Join(parts, " ")
|
||||
}
|
||||
|
||||
func formatConsoleValue(value any) string {
|
||||
switch value := value.(type) {
|
||||
case nil:
|
||||
return "null"
|
||||
case string:
|
||||
return value
|
||||
case error:
|
||||
return value.Error()
|
||||
}
|
||||
|
||||
if data, err := json.Marshal(value); err == nil {
|
||||
return string(data)
|
||||
}
|
||||
return fmt.Sprint(value)
|
||||
}
|
||||
|
||||
func formatConsoleString(format string, args []any) string {
|
||||
var builder strings.Builder
|
||||
argIndex := 0
|
||||
@@ -154,7 +171,7 @@ func formatConsoleString(format string, args []any) string {
|
||||
argIndex++
|
||||
switch format[i] {
|
||||
case 's', 'o', 'O':
|
||||
builder.WriteString(fmt.Sprint(arg))
|
||||
builder.WriteString(formatConsoleValue(arg))
|
||||
case 'd', 'i':
|
||||
builder.WriteString(fmt.Sprintf("%d", toConsoleInteger(arg)))
|
||||
case 'f':
|
||||
@@ -168,7 +185,7 @@ func formatConsoleString(format string, args []any) string {
|
||||
if argIndex < len(args) {
|
||||
remaining := make([]string, 0, len(args)-argIndex)
|
||||
for _, arg := range args[argIndex:] {
|
||||
remaining = append(remaining, fmt.Sprint(arg))
|
||||
remaining = append(remaining, formatConsoleValue(arg))
|
||||
}
|
||||
if builder.Len() > 0 {
|
||||
builder.WriteByte(' ')
|
||||
|
||||
@@ -4,15 +4,17 @@ import (
|
||||
_ "embed"
|
||||
"fmt"
|
||||
|
||||
nodeurl "github.com/dop251/goja_nodejs/url"
|
||||
"github.com/komari-monitor/komari/pkg/jsruntime/xhr"
|
||||
)
|
||||
|
||||
// JavaScript compatibility: injects bounded timer APIs, console, Fetch API,
|
||||
// XMLHttpRequest, and queueMicrotask.
|
||||
// JavaScript compatibility: injects bounded timer APIs, console, URL APIs,
|
||||
// Fetch API, XMLHttpRequest, and queueMicrotask.
|
||||
func (r *Runtime) injectGlobals() error {
|
||||
if err := r.consoleMod.Inject(r.vm); err != nil {
|
||||
return fmt.Errorf("inject console: %w", err)
|
||||
}
|
||||
nodeurl.Enable(r.vm)
|
||||
if err := r.timersMod.Inject(r.vm); err != nil {
|
||||
return fmt.Errorf("inject timers: %w", err)
|
||||
}
|
||||
|
||||
@@ -22,6 +22,15 @@ import (
|
||||
type Module struct {
|
||||
runtime *bridge.Runtime
|
||||
allowListen bool
|
||||
sockets sync.Map
|
||||
}
|
||||
|
||||
type socketState struct {
|
||||
connection net.Conn
|
||||
resourceID uint64
|
||||
readerDone chan struct{}
|
||||
detaching atomic.Bool
|
||||
closed atomic.Bool
|
||||
}
|
||||
|
||||
func New(runtime *bridge.Runtime, allowListen bool) *Module {
|
||||
@@ -335,6 +344,41 @@ func (m *Module) newNetSocket(vm *goja.Runtime, connection net.Conn, onClose fun
|
||||
return socket
|
||||
}
|
||||
|
||||
// AttachSocket configures a JavaScript socket around an established
|
||||
// connection. It is shared with the TLS module after a handshake completes.
|
||||
func (m *Module) AttachSocket(vm *goja.Runtime, socket *goja.Object, connection net.Conn, onClose func()) bool {
|
||||
return m.configureNetSocket(vm, socket, connection, onClose)
|
||||
}
|
||||
|
||||
// DetachSocket stops a socket's reader and transfers ownership of its
|
||||
// connection to the caller. It is used to upgrade an SMTP connection with
|
||||
// STARTTLS without closing the underlying TCP stream.
|
||||
func (m *Module) DetachSocket(socket *goja.Object) (net.Conn, error) {
|
||||
value, ok := m.sockets.Load(socket)
|
||||
if !ok {
|
||||
return nil, errors.New("socket is not connected")
|
||||
}
|
||||
state := value.(*socketState)
|
||||
if !state.detaching.CompareAndSwap(false, true) {
|
||||
return nil, errors.New("socket is already being upgraded")
|
||||
}
|
||||
if err := state.connection.SetReadDeadline(time.Now()); err != nil {
|
||||
state.detaching.Store(false)
|
||||
return nil, err
|
||||
}
|
||||
<-state.readerDone
|
||||
if state.closed.Load() {
|
||||
return nil, errors.New("socket closed during upgrade")
|
||||
}
|
||||
m.sockets.Delete(socket)
|
||||
m.runtime.RemoveResource(state.resourceID)
|
||||
if err := state.connection.SetDeadline(time.Time{}); err != nil {
|
||||
_ = state.connection.Close()
|
||||
return nil, err
|
||||
}
|
||||
return state.connection, nil
|
||||
}
|
||||
|
||||
func (m *Module) configureNetSocket(vm *goja.Runtime, socket *goja.Object, connection net.Conn, onClose func()) bool {
|
||||
localHost, localPort := splitNetAddress(connection.LocalAddr())
|
||||
remoteHost, remotePort := splitNetAddress(connection.RemoteAddr())
|
||||
@@ -356,12 +400,18 @@ func (m *Module) configureNetSocket(vm *goja.Runtime, socket *goja.Object, conne
|
||||
}
|
||||
return false
|
||||
}
|
||||
state := &socketState{
|
||||
connection: connection,
|
||||
resourceID: resourceID,
|
||||
readerDone: make(chan struct{}),
|
||||
}
|
||||
m.sockets.Store(socket, state)
|
||||
var encoding atomic.Value
|
||||
encoding.Store("")
|
||||
var closed atomic.Bool
|
||||
var writes writequeue.Queue
|
||||
closeSocket := func() {
|
||||
if closed.CompareAndSwap(false, true) {
|
||||
if state.closed.CompareAndSwap(false, true) {
|
||||
m.sockets.Delete(socket)
|
||||
m.runtime.RemoveResource(resourceID)
|
||||
_ = connection.Close()
|
||||
if onClose != nil {
|
||||
@@ -468,6 +518,7 @@ func (m *Module) configureNetSocket(vm *goja.Runtime, socket *goja.Object, conne
|
||||
panic(vm.NewGoError(fmt.Errorf("net.Socket.unref is not supported by jsruntime; the event loop is host-driven")))
|
||||
})
|
||||
go func() {
|
||||
defer close(state.readerDone)
|
||||
data := make([]byte, 32*1024)
|
||||
for {
|
||||
count, err := connection.Read(data)
|
||||
@@ -490,6 +541,9 @@ func (m *Module) configureNetSocket(vm *goja.Runtime, socket *goja.Object, conne
|
||||
<-delivered
|
||||
}
|
||||
if err != nil {
|
||||
if state.detaching.Load() {
|
||||
return
|
||||
}
|
||||
closeSocket()
|
||||
m.runtime.RunOnLoop(func(vm *goja.Runtime) {
|
||||
_ = m.runtime.RunJob(vm, "net.Socket close", func() error {
|
||||
|
||||
@@ -25,6 +25,7 @@ func (r *Runtime) registerNodeModules(registry *require.Registry) {
|
||||
r.registerNodeModule(registry, "fs", r.fsModule.Load)
|
||||
r.registerNodeModule(registry, "child_process", r.childProcessModule.Load)
|
||||
r.registerNodeModule(registry, "net", r.netModule.Load)
|
||||
r.registerNodeModule(registry, "tls", r.tlsModule.Load)
|
||||
r.registerNodeModule(registry, "http", r.httpModule.Load)
|
||||
r.registerNodeModule(registry, "stream", streammodule.Load)
|
||||
r.registerNodeModule(registry, "stream/promises", streammodule.LoadPromises)
|
||||
|
||||
@@ -22,7 +22,6 @@ import (
|
||||
_ "github.com/dop251/goja_nodejs/buffer"
|
||||
"github.com/dop251/goja_nodejs/eventloop"
|
||||
"github.com/dop251/goja_nodejs/require"
|
||||
_ "github.com/dop251/goja_nodejs/url"
|
||||
_ "github.com/dop251/goja_nodejs/util"
|
||||
childprocess "github.com/komari-monitor/komari/pkg/jsruntime/child_process"
|
||||
"github.com/komari-monitor/komari/pkg/jsruntime/console"
|
||||
@@ -35,6 +34,7 @@ import (
|
||||
pathmodule "github.com/komari-monitor/komari/pkg/jsruntime/path"
|
||||
processmodule "github.com/komari-monitor/komari/pkg/jsruntime/process"
|
||||
"github.com/komari-monitor/komari/pkg/jsruntime/timers"
|
||||
tlsmodule "github.com/komari-monitor/komari/pkg/jsruntime/tls"
|
||||
)
|
||||
|
||||
const defaultTimeout = 30 * time.Second
|
||||
@@ -123,6 +123,7 @@ type Runtime struct {
|
||||
processModule *processmodule.Module
|
||||
childProcessModule *childprocess.Module
|
||||
netModule *netmodule.Module
|
||||
tlsModule *tlsmodule.Module
|
||||
httpModule *httpmodule.Module
|
||||
cryptoModule *cryptomodule.Module
|
||||
resourceMu sync.Mutex
|
||||
@@ -237,6 +238,7 @@ func New(script string, options Options) (*Runtime, error) {
|
||||
})
|
||||
runtime.childProcessModule = childprocess.New(host, filesystem, options.AllowExec, maxChildOutputBytes)
|
||||
runtime.netModule = netmodule.New(host, options.AllowListen)
|
||||
runtime.tlsModule = tlsmodule.New(host, runtime.netModule)
|
||||
runtime.httpModule = httpmodule.New(host, options.AllowListen, maxHTTPBodyBytes)
|
||||
runtime.cryptoModule = cryptomodule.New(host)
|
||||
runtime.registerNodeModules(registry)
|
||||
|
||||
@@ -29,6 +29,8 @@ func TestNewReturnsRuntimeWithInjectedGlobals(t *testing.T) {
|
||||
typeof console.warn === "function" &&
|
||||
typeof fetch === "function" &&
|
||||
typeof XMLHttpRequest === "function" &&
|
||||
typeof URL === "function" &&
|
||||
typeof URLSearchParams === "function" &&
|
||||
typeof Promise === "function" &&
|
||||
typeof require === "function" &&
|
||||
typeof queueMicrotask === "function" &&
|
||||
@@ -49,6 +51,25 @@ func TestNewReturnsRuntimeWithInjectedGlobals(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestURLSearchParamsGlobal(t *testing.T) {
|
||||
runtime, err := New(`
|
||||
function sendMessage() {
|
||||
const params = new URLSearchParams();
|
||||
params.set("chat_id", "-100 123");
|
||||
params.set("text", "通知 & test");
|
||||
return params.toString() === "chat_id=-100+123&text=%E9%80%9A%E7%9F%A5+%26+test";
|
||||
}
|
||||
`, Options{Console: io.Discard, Timeout: time.Second})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer runtime.Close()
|
||||
|
||||
if err := runtime.Call("sendMessage"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestConsoleMethodsWriteMessagesAndStacks(t *testing.T) {
|
||||
var output bytes.Buffer
|
||||
runtime, err := New(`
|
||||
|
||||
@@ -0,0 +1,163 @@
|
||||
package tlsmodule
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/tls"
|
||||
"crypto/x509"
|
||||
"fmt"
|
||||
"net"
|
||||
"strconv"
|
||||
|
||||
"github.com/dop251/goja"
|
||||
"github.com/komari-monitor/komari/pkg/jsruntime/events"
|
||||
"github.com/komari-monitor/komari/pkg/jsruntime/internal/bridge"
|
||||
netmodule "github.com/komari-monitor/komari/pkg/jsruntime/net"
|
||||
)
|
||||
|
||||
// Module provides the subset of node:tls needed by notification plugins.
|
||||
type Module struct {
|
||||
runtime *bridge.Runtime
|
||||
netModule *netmodule.Module
|
||||
}
|
||||
|
||||
func New(runtime *bridge.Runtime, netModule *netmodule.Module) *Module {
|
||||
return &Module{runtime: runtime, netModule: netModule}
|
||||
}
|
||||
|
||||
func (m *Module) Load(vm *goja.Runtime, module *goja.Object) {
|
||||
exports := vm.NewObject()
|
||||
connect := func(call goja.FunctionCall) goja.Value { return m.connect(vm, call) }
|
||||
_ = exports.Set("connect", connect)
|
||||
_ = exports.Set("createConnection", connect)
|
||||
_ = module.Set("exports", exports)
|
||||
}
|
||||
|
||||
func (m *Module) connect(vm *goja.Runtime, call goja.FunctionCall) *goja.Object {
|
||||
options, ok := call.Argument(0).(*goja.Object)
|
||||
if !ok {
|
||||
panic(vm.NewTypeError("tls.connect requires an options object"))
|
||||
}
|
||||
|
||||
host := stringOption(options, "host")
|
||||
port := intOption(options, "port")
|
||||
serverName := stringOption(options, "servername")
|
||||
if serverName == "" {
|
||||
serverName = host
|
||||
}
|
||||
var sourceSocket *goja.Object
|
||||
if value := options.Get("socket"); value != nil && !goja.IsUndefined(value) && !goja.IsNull(value) {
|
||||
sourceSocket, ok = value.(*goja.Object)
|
||||
if !ok {
|
||||
panic(vm.NewTypeError("tls.connect socket must be a net.Socket"))
|
||||
}
|
||||
}
|
||||
if sourceSocket == nil && (host == "" || port < 1) {
|
||||
panic(vm.NewTypeError("tls.connect requires host and port"))
|
||||
}
|
||||
if serverName == "" {
|
||||
panic(vm.NewTypeError("tls.connect requires servername when upgrading a socket"))
|
||||
}
|
||||
|
||||
config := &tls.Config{ServerName: serverName, MinVersion: tls.VersionTLS12}
|
||||
if ca := stringOption(options, "ca"); ca != "" {
|
||||
roots := x509.NewCertPool()
|
||||
if !roots.AppendCertsFromPEM([]byte(ca)) {
|
||||
panic(vm.NewTypeError("tls.connect ca does not contain a certificate"))
|
||||
}
|
||||
config.RootCAs = roots
|
||||
}
|
||||
|
||||
socket := events.NewEmitter(vm)
|
||||
_ = socket.Set("connecting", true)
|
||||
_ = socket.Set("encrypted", true)
|
||||
_ = socket.Set("authorized", false)
|
||||
_ = socket.Set("servername", serverName)
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), m.runtime.Timeout())
|
||||
_ = socket.Set("destroy", func() *goja.Object {
|
||||
cancel()
|
||||
return socket
|
||||
})
|
||||
if callback, ok := goja.AssertFunction(call.Argument(1)); ok {
|
||||
once, _ := goja.AssertFunction(socket.Get("once"))
|
||||
_, _ = once(socket, vm.ToValue("secureConnect"), vm.ToValue(callback))
|
||||
}
|
||||
|
||||
go m.establish(ctx, cancel, socket, sourceSocket, host, port, config)
|
||||
return socket
|
||||
}
|
||||
|
||||
func (m *Module) establish(
|
||||
ctx context.Context,
|
||||
cancel context.CancelFunc,
|
||||
socket *goja.Object,
|
||||
sourceSocket *goja.Object,
|
||||
host string,
|
||||
port int,
|
||||
config *tls.Config,
|
||||
) {
|
||||
defer cancel()
|
||||
var connection net.Conn
|
||||
var err error
|
||||
if sourceSocket != nil {
|
||||
connection, err = m.netModule.DetachSocket(sourceSocket)
|
||||
} else {
|
||||
connection, err = (&net.Dialer{}).DialContext(ctx, "tcp", net.JoinHostPort(host, strconv.Itoa(port)))
|
||||
}
|
||||
if err != nil {
|
||||
m.emitError(socket, err)
|
||||
return
|
||||
}
|
||||
|
||||
resourceID := m.runtime.AddResource(func() { _ = connection.Close() })
|
||||
if resourceID == 0 {
|
||||
return
|
||||
}
|
||||
secureConnection := tls.Client(connection, config)
|
||||
if err := secureConnection.HandshakeContext(ctx); err != nil {
|
||||
m.runtime.RemoveResource(resourceID)
|
||||
_ = connection.Close()
|
||||
m.emitError(socket, fmt.Errorf("TLS handshake failed: %w", err))
|
||||
return
|
||||
}
|
||||
|
||||
queued := m.runtime.RunOnLoop(func(vm *goja.Runtime) {
|
||||
_ = m.runtime.RunJob(vm, "tls secureConnect", func() error {
|
||||
if !m.netModule.AttachSocket(vm, socket, secureConnection, nil) {
|
||||
return fmt.Errorf("TLS socket runtime is closed")
|
||||
}
|
||||
m.runtime.RemoveResource(resourceID)
|
||||
_ = socket.Set("authorized", true)
|
||||
return events.Emit(vm, socket, "secureConnect")
|
||||
})
|
||||
})
|
||||
if !queued {
|
||||
m.runtime.RemoveResource(resourceID)
|
||||
_ = secureConnection.Close()
|
||||
}
|
||||
}
|
||||
|
||||
func (m *Module) emitError(socket *goja.Object, err error) {
|
||||
m.runtime.RunOnLoop(func(vm *goja.Runtime) {
|
||||
_ = m.runtime.RunJob(vm, "tls error", func() error {
|
||||
_ = socket.Set("connecting", false)
|
||||
return events.Emit(vm, socket, "error", vm.NewGoError(err))
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
func stringOption(options *goja.Object, name string) string {
|
||||
value := options.Get(name)
|
||||
if value == nil || goja.IsUndefined(value) || goja.IsNull(value) {
|
||||
return ""
|
||||
}
|
||||
return value.String()
|
||||
}
|
||||
|
||||
func intOption(options *goja.Object, name string) int {
|
||||
value := options.Get(name)
|
||||
if value == nil || goja.IsUndefined(value) || goja.IsNull(value) {
|
||||
return 0
|
||||
}
|
||||
return int(value.ToInteger())
|
||||
}
|
||||
@@ -0,0 +1,609 @@
|
||||
package metric
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
type batchSeriesGroupKey struct {
|
||||
resolution time.Duration
|
||||
needDigest bool
|
||||
}
|
||||
|
||||
type batchSeriesGroup struct {
|
||||
key batchSeriesGroupKey
|
||||
metricNames map[string]struct{}
|
||||
fields rollupReadFields
|
||||
}
|
||||
|
||||
type seriesIdentity struct {
|
||||
metricName string
|
||||
entityID string
|
||||
tagsHash string
|
||||
}
|
||||
|
||||
type seriesReadMeta struct {
|
||||
id int64
|
||||
metricName string
|
||||
entityID string
|
||||
tagsHash string
|
||||
tagsJSON string
|
||||
tags map[string]string
|
||||
}
|
||||
|
||||
type rollupRateSample struct {
|
||||
timestamp int64
|
||||
value float64
|
||||
}
|
||||
|
||||
type rollupAggregateState struct {
|
||||
summary *rollupBucket
|
||||
rateSamples []rollupRateSample
|
||||
tags map[string]string
|
||||
}
|
||||
|
||||
type metricSeriesAccumulator struct {
|
||||
spec BatchSeriesSpec
|
||||
compression float64
|
||||
needDigest bool
|
||||
needSummary bool
|
||||
needRate bool
|
||||
groups map[rollupKey]*rollupAggregateState
|
||||
rateGroups map[rollupKey]*rollupAggregateState
|
||||
}
|
||||
|
||||
// SeriesBatch is the single rollup query implementation. It resolves metric
|
||||
// metadata once, scans each selected backing tier once, and streams rows into
|
||||
// the final output buckets.
|
||||
func (s *Store) SeriesBatch(ctx context.Context, query BatchSeriesQuery, now time.Time) (BatchSeriesResult, error) {
|
||||
if err := s.ensureOpen(); err != nil {
|
||||
return BatchSeriesResult{}, err
|
||||
}
|
||||
if err := query.Validate(); err != nil {
|
||||
return BatchSeriesResult{}, err
|
||||
}
|
||||
return s.seriesBatchAt(ctx, query.normalized(), now.UTC(), nil)
|
||||
}
|
||||
|
||||
// GetMetrics loads the requested metric definitions with one database query.
|
||||
func (s *Store) GetMetrics(ctx context.Context, metricNames []string) (map[string]Definition, error) {
|
||||
if err := s.ensureOpen(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if len(metricNames) == 0 {
|
||||
return map[string]Definition{}, nil
|
||||
}
|
||||
return s.loadBatchDefinitions(ctx, metricNames)
|
||||
}
|
||||
|
||||
func (s *Store) seriesBatchAt(ctx context.Context, query BatchSeriesQuery, now time.Time, forcedResolutions map[string]time.Duration) (BatchSeriesResult, error) {
|
||||
metricNames := make([]string, 0, len(query.Specs))
|
||||
for _, spec := range query.Specs {
|
||||
metricNames = append(metricNames, spec.MetricName)
|
||||
}
|
||||
definitions, err := s.loadBatchDefinitions(ctx, metricNames)
|
||||
if err != nil {
|
||||
return BatchSeriesResult{}, err
|
||||
}
|
||||
|
||||
result := BatchSeriesResult{
|
||||
Definitions: definitions,
|
||||
Values: make(map[string]map[Aggregation][]AggregatePoint, len(query.Specs)),
|
||||
}
|
||||
accumulators := make(map[string]*metricSeriesAccumulator, len(query.Specs))
|
||||
groups := make(map[batchSeriesGroupKey]*batchSeriesGroup)
|
||||
for _, spec := range query.Specs {
|
||||
values := make(map[Aggregation][]AggregatePoint, len(spec.Aggregations))
|
||||
accumulator := &metricSeriesAccumulator{
|
||||
spec: spec, compression: s.cfg.RollupPolicy.compression(),
|
||||
groups: make(map[rollupKey]*rollupAggregateState), rateGroups: make(map[rollupKey]*rollupAggregateState),
|
||||
}
|
||||
for _, aggregation := range spec.Aggregations {
|
||||
values[aggregation] = []AggregatePoint{}
|
||||
if aggregation == AggRate {
|
||||
accumulator.needRate = true
|
||||
} else {
|
||||
accumulator.needSummary = true
|
||||
}
|
||||
if isPercentile(aggregation) {
|
||||
accumulator.needDigest = true
|
||||
}
|
||||
}
|
||||
result.Values[spec.MetricName] = values
|
||||
accumulators[spec.MetricName] = accumulator
|
||||
|
||||
policy := s.cfg.RollupPolicy
|
||||
if definition, ok := definitions[spec.MetricName]; ok {
|
||||
if definition.RetentionDays <= 0 {
|
||||
continue
|
||||
}
|
||||
policy = policy.withMetricRetention(time.Duration(definition.RetentionDays) * 24 * time.Hour)
|
||||
}
|
||||
if len(policy.Tiers) == 0 {
|
||||
continue
|
||||
}
|
||||
resolution := seriesResolutionForPolicy(query.Start, spec.Interval, now, policy)
|
||||
if forced, ok := forcedResolutions[spec.MetricName]; ok {
|
||||
resolution = forced
|
||||
}
|
||||
key := batchSeriesGroupKey{resolution: resolution, needDigest: accumulator.needDigest}
|
||||
group := groups[key]
|
||||
if group == nil {
|
||||
group = &batchSeriesGroup{key: key, metricNames: make(map[string]struct{})}
|
||||
groups[key] = group
|
||||
}
|
||||
group.metricNames[spec.MetricName] = struct{}{}
|
||||
group.fields |= rollupFieldsForAggregations(spec.Aggregations)
|
||||
}
|
||||
|
||||
seriesByID, seriesByIdentity, err := s.loadSeriesDictionary(ctx, seriesDictionaryPlan{
|
||||
MetricNames: metricNames,
|
||||
EntityIDs: query.EntityIDs,
|
||||
Tags: query.Tags,
|
||||
})
|
||||
if err != nil {
|
||||
return BatchSeriesResult{}, err
|
||||
}
|
||||
|
||||
orderedGroups := make([]*batchSeriesGroup, 0, len(groups))
|
||||
for _, group := range groups {
|
||||
orderedGroups = append(orderedGroups, group)
|
||||
}
|
||||
sort.Slice(orderedGroups, func(i, j int) bool {
|
||||
if orderedGroups[i].key.resolution != orderedGroups[j].key.resolution {
|
||||
return orderedGroups[i].key.resolution < orderedGroups[j].key.resolution
|
||||
}
|
||||
return !orderedGroups[i].key.needDigest && orderedGroups[j].key.needDigest
|
||||
})
|
||||
|
||||
s.rollupViewMu.RLock()
|
||||
defer s.rollupViewMu.RUnlock()
|
||||
for _, group := range orderedGroups {
|
||||
if err := s.scanPersistedRollupGroup(ctx, query, group, seriesByID, accumulators); err != nil {
|
||||
return BatchSeriesResult{}, err
|
||||
}
|
||||
if err := s.scanMemoryRollupGroup(query, group, seriesByIdentity, accumulators); err != nil {
|
||||
return BatchSeriesResult{}, err
|
||||
}
|
||||
}
|
||||
|
||||
for metricName, accumulator := range accumulators {
|
||||
values, err := accumulator.points(query.Order)
|
||||
if err != nil {
|
||||
return BatchSeriesResult{}, err
|
||||
}
|
||||
result.Values[metricName] = values
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func (s *Store) loadBatchDefinitions(ctx context.Context, metricNames []string) (map[string]Definition, error) {
|
||||
args := make([]any, 0, len(metricNames))
|
||||
placeholders := make([]string, 0, len(metricNames))
|
||||
for _, metricName := range metricNames {
|
||||
args = append(args, metricName)
|
||||
placeholders = append(placeholders, s.dialect.placeholder(len(args)))
|
||||
}
|
||||
rows, err := s.reader().QueryContext(ctx, fmt.Sprintf(
|
||||
"SELECT name, type, unit, description, retention_days, metadata, created_at_milli, updated_at_milli FROM %s WHERE name IN (%s) ORDER BY name ASC",
|
||||
s.tables.definitions, strings.Join(placeholders, ", "),
|
||||
), args...)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
definitions := make(map[string]Definition, len(metricNames))
|
||||
for rows.Next() {
|
||||
definition, err := scanDefinition(rows)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
definitions[definition.Name] = definition
|
||||
}
|
||||
return definitions, rows.Err()
|
||||
}
|
||||
|
||||
func (s *Store) loadSeriesDictionary(ctx context.Context, plan seriesDictionaryPlan) (map[int64]*seriesReadMeta, map[seriesIdentity]*seriesReadMeta, error) {
|
||||
rendered := s.dialect.renderSeriesDictionary(s.tables, s.cfg.TablePrefix+"series_metric_entity_idx", plan)
|
||||
rows, err := s.reader().QueryContext(ctx, rendered.Query, rendered.Args...)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
byID := make(map[int64]*seriesReadMeta)
|
||||
byIdentity := make(map[seriesIdentity]*seriesReadMeta)
|
||||
for rows.Next() {
|
||||
var meta seriesReadMeta
|
||||
var rawTags any
|
||||
if err := rows.Scan(&meta.id, &meta.metricName, &meta.entityID, &meta.tagsHash, &rawTags); err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
meta.tagsJSON, err = rawJSONToString(rawTags)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
meta.tags, err = rollupTagsFromJSON(meta.tagsJSON)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
byID[meta.id] = &meta
|
||||
byIdentity[seriesIdentity{metricName: meta.metricName, entityID: meta.entityID, tagsHash: meta.tagsHash}] = &meta
|
||||
}
|
||||
return byID, byIdentity, rows.Err()
|
||||
}
|
||||
|
||||
func (s *Store) scanPersistedRollupGroup(ctx context.Context, query BatchSeriesQuery, group *batchSeriesGroup, seriesByID map[int64]*seriesReadMeta, accumulators map[string]*metricSeriesAccumulator) error {
|
||||
seriesIDs := make([]int64, 0)
|
||||
for seriesID, meta := range seriesByID {
|
||||
if _, ok := group.metricNames[meta.metricName]; ok {
|
||||
seriesIDs = append(seriesIDs, seriesID)
|
||||
}
|
||||
}
|
||||
if len(seriesIDs) == 0 {
|
||||
return nil
|
||||
}
|
||||
sort.Slice(seriesIDs, func(i, j int) bool { return seriesIDs[i] < seriesIDs[j] })
|
||||
resolutionID, found, err := s.resolveResolutionID(ctx, group.key.resolution)
|
||||
if err != nil || !found {
|
||||
return err
|
||||
}
|
||||
rendered := s.dialect.renderRollupRead(s.tables, s.cfg.TablePrefix+"rollups_resolution_bucket_idx", rollupReadPlan{
|
||||
SeriesIDs: seriesIDs,
|
||||
ResolutionID: resolutionID,
|
||||
StartMilli: bucketStartMillis(query.Start.UnixMilli(), group.key.resolution.Milliseconds()),
|
||||
EndMilli: query.End.UnixMilli(),
|
||||
Fields: group.fields,
|
||||
})
|
||||
rows, err := s.reader().QueryContext(ctx, rendered.Query, rendered.Args...)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer rows.Close()
|
||||
var seriesID, bucket, count, firstTS, lastTS int64
|
||||
var sum, sumSq, min, max, firstVal, lastVal float64
|
||||
var digestBlob []byte
|
||||
destinations := []any{&seriesID, &bucket, &count}
|
||||
if group.fields&rollupReadSum != 0 {
|
||||
destinations = append(destinations, &sum)
|
||||
}
|
||||
if group.fields&rollupReadSumSq != 0 {
|
||||
destinations = append(destinations, &sumSq)
|
||||
}
|
||||
if group.fields&rollupReadMin != 0 {
|
||||
destinations = append(destinations, &min)
|
||||
}
|
||||
if group.fields&rollupReadMax != 0 {
|
||||
destinations = append(destinations, &max)
|
||||
}
|
||||
if group.fields&rollupReadFirst != 0 {
|
||||
destinations = append(destinations, &firstVal, &firstTS)
|
||||
}
|
||||
if group.fields&rollupReadLast != 0 {
|
||||
destinations = append(destinations, &lastVal, &lastTS)
|
||||
}
|
||||
if group.fields&rollupReadDigest != 0 {
|
||||
destinations = append(destinations, &digestBlob)
|
||||
}
|
||||
for rows.Next() {
|
||||
err = rows.Scan(destinations...)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
meta := seriesByID[seriesID]
|
||||
accumulator := accumulators[meta.metricName]
|
||||
state := accumulator.consume(meta, bucket, count, sum, sumSq, min, max, firstVal, firstTS, lastVal, lastTS, nil)
|
||||
if accumulator.needDigest {
|
||||
if err := mergeEncodedRollupDigest(state.summary.digest, count, min, max, digestBlob); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
return rows.Err()
|
||||
}
|
||||
|
||||
func rollupFieldsForAggregations(aggregations []Aggregation) rollupReadFields {
|
||||
var fields rollupReadFields
|
||||
for _, aggregation := range aggregations {
|
||||
switch aggregation {
|
||||
case AggAvg, AggSum:
|
||||
fields |= rollupReadSum
|
||||
case AggMin:
|
||||
fields |= rollupReadMin
|
||||
case AggMax:
|
||||
fields |= rollupReadMax
|
||||
case AggFirst:
|
||||
fields |= rollupReadFirst
|
||||
case AggLast, AggRate:
|
||||
fields |= rollupReadLast
|
||||
case AggStdDev:
|
||||
fields |= rollupReadSum | rollupReadSumSq
|
||||
default:
|
||||
if isPercentile(aggregation) {
|
||||
fields |= rollupReadMin | rollupReadMax | rollupReadDigest
|
||||
}
|
||||
}
|
||||
}
|
||||
return fields
|
||||
}
|
||||
|
||||
func (s *Store) resolveResolutionID(ctx context.Context, resolution time.Duration) (int64, bool, error) {
|
||||
var resolutionID int64
|
||||
err := s.reader().QueryRowContext(ctx,
|
||||
"SELECT id FROM "+s.tables.resolutions+" WHERE resolution_milli = "+s.dialect.placeholder(1),
|
||||
resolution.Milliseconds(),
|
||||
).Scan(&resolutionID)
|
||||
if err == sql.ErrNoRows {
|
||||
return 0, false, nil
|
||||
}
|
||||
return resolutionID, err == nil, err
|
||||
}
|
||||
|
||||
func (s *Store) scanMemoryRollupGroup(query BatchSeriesQuery, group *batchSeriesGroup, seriesByIdentity map[seriesIdentity]*seriesReadMeta, accumulators map[string]*metricSeriesAccumulator) error {
|
||||
if group.key.resolution == time.Minute {
|
||||
return s.scanHotRollupGroup(query, group, seriesByIdentity, accumulators)
|
||||
}
|
||||
return s.scanCoarseRollupGroup(query, group, seriesByIdentity, accumulators)
|
||||
}
|
||||
|
||||
func (s *Store) scanHotRollupGroup(query BatchSeriesQuery, group *batchSeriesGroup, seriesByIdentity map[seriesIdentity]*seriesReadMeta, accumulators map[string]*metricSeriesAccumulator) error {
|
||||
start := bucketStartMillis(query.Start.UnixMilli(), time.Minute.Milliseconds())
|
||||
end := query.End.UnixMilli()
|
||||
entityIDs := stringSet(query.EntityIDs)
|
||||
s.hotMu.RLock()
|
||||
defer s.hotMu.RUnlock()
|
||||
for key, bucket := range s.hot {
|
||||
if _, ok := group.metricNames[key.metricName]; !ok || key.bucket < start || key.bucket > end || bucket.count == 0 {
|
||||
continue
|
||||
}
|
||||
if len(entityIDs) > 0 {
|
||||
if _, ok := entityIDs[key.entityID]; !ok {
|
||||
continue
|
||||
}
|
||||
}
|
||||
meta, matched, err := memorySeriesMeta(seriesByIdentity, key.metricName, key.entityID, key.tagsHash, bucket.tagsJSON, query.Tags)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !matched {
|
||||
continue
|
||||
}
|
||||
accumulators[key.metricName].consume(meta, key.bucket, bucket.count, bucket.sum, bucket.sumSq, bucket.min, bucket.max, bucket.firstVal, bucket.firstTS, bucket.lastVal, bucket.lastTS, bucket.digest)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *Store) scanCoarseRollupGroup(query BatchSeriesQuery, group *batchSeriesGroup, seriesByIdentity map[seriesIdentity]*seriesReadMeta, accumulators map[string]*metricSeriesAccumulator) error {
|
||||
start := bucketStartMillis(query.Start.UnixMilli(), group.key.resolution.Milliseconds())
|
||||
end := query.End.UnixMilli()
|
||||
entityIDs := stringSet(query.EntityIDs)
|
||||
s.coarseMu.RLock()
|
||||
defer s.coarseMu.RUnlock()
|
||||
for key, parent := range s.coarse {
|
||||
if key.interval != group.key.resolution || key.bucket < start || key.bucket > end {
|
||||
continue
|
||||
}
|
||||
if _, ok := group.metricNames[key.metricName]; !ok {
|
||||
continue
|
||||
}
|
||||
if len(entityIDs) > 0 {
|
||||
if _, ok := entityIDs[key.entityID]; !ok {
|
||||
continue
|
||||
}
|
||||
}
|
||||
childKeys := make([]rollupKey, 0, len(parent.children))
|
||||
for childKey := range parent.children {
|
||||
childKeys = append(childKeys, childKey)
|
||||
}
|
||||
if len(childKeys) == 0 {
|
||||
continue
|
||||
}
|
||||
sortRollupKeys(childKeys)
|
||||
firstChild := parent.children[childKeys[0]]
|
||||
meta, matched, err := memorySeriesMeta(seriesByIdentity, key.metricName, key.entityID, key.tagsHash, firstChild.tagsJSON, query.Tags)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !matched {
|
||||
continue
|
||||
}
|
||||
summary := newRollupBucketWithDigest(s.cfg.RollupPolicy.compression(), group.key.needDigest)
|
||||
for _, childKey := range childKeys {
|
||||
child := parent.children[childKey]
|
||||
mergeRollupValues(summary, child.count, child.sum, child.sumSq, child.min, child.max, child.firstVal, child.firstTS, child.lastVal, child.lastTS, child.digest, group.key.needDigest)
|
||||
}
|
||||
accumulators[key.metricName].consume(meta, key.bucket, summary.count, summary.sum, summary.sumSq, summary.min, summary.max, summary.firstVal, summary.firstTS, summary.lastVal, summary.lastTS, summary.digest)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func memorySeriesMeta(seriesByIdentity map[seriesIdentity]*seriesReadMeta, metricName, entityID, tagsHash, tagsJSON string, filter map[string]string) (*seriesReadMeta, bool, error) {
|
||||
identity := seriesIdentity{metricName: metricName, entityID: entityID, tagsHash: tagsHash}
|
||||
if meta := seriesByIdentity[identity]; meta != nil {
|
||||
return meta, true, nil
|
||||
}
|
||||
tags, err := rollupTagsFromJSON(tagsJSON)
|
||||
if err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
for key, value := range filter {
|
||||
if tags[key] != value {
|
||||
return nil, false, nil
|
||||
}
|
||||
}
|
||||
meta := &seriesReadMeta{metricName: metricName, entityID: entityID, tagsHash: tagsHash, tagsJSON: tagsJSON, tags: tags}
|
||||
seriesByIdentity[identity] = meta
|
||||
return meta, true, nil
|
||||
}
|
||||
|
||||
func stringSet(values []string) map[string]struct{} {
|
||||
set := make(map[string]struct{}, len(values))
|
||||
for _, value := range values {
|
||||
set[value] = struct{}{}
|
||||
}
|
||||
return set
|
||||
}
|
||||
|
||||
func (a *metricSeriesAccumulator) consume(meta *seriesReadMeta, sourceBucket, count int64, sum, sumSq, min, max, firstVal float64, firstTS int64, lastVal float64, lastTS int64, digest *TDigest) *rollupAggregateState {
|
||||
key := rollupKey{bucket: bucketStartMillis(sourceBucket, a.spec.Interval.Milliseconds())}
|
||||
if a.spec.PreserveSeries {
|
||||
key.entityID = meta.entityID
|
||||
key.tagsHash = meta.tagsHash
|
||||
}
|
||||
if a.needSummary {
|
||||
state := a.groups[key]
|
||||
if state == nil {
|
||||
state = &rollupAggregateState{summary: newRollupBucketWithDigest(a.compression, a.needDigest)}
|
||||
if a.spec.PreserveSeries {
|
||||
state.tags = meta.tags
|
||||
}
|
||||
a.groups[key] = state
|
||||
}
|
||||
mergeRollupValues(state.summary, count, sum, sumSq, min, max, firstVal, firstTS, lastVal, lastTS, digest, a.needDigest)
|
||||
if !a.needRate {
|
||||
return state
|
||||
}
|
||||
}
|
||||
if a.needRate {
|
||||
rateKey := rollupKey{
|
||||
bucket: bucketStartMillis(sourceBucket, a.spec.Interval.Milliseconds()),
|
||||
entityID: meta.entityID,
|
||||
tagsHash: meta.tagsHash,
|
||||
}
|
||||
state := a.rateGroups[rateKey]
|
||||
if state == nil {
|
||||
state = &rollupAggregateState{tags: meta.tags}
|
||||
a.rateGroups[rateKey] = state
|
||||
}
|
||||
state.rateSamples = append(state.rateSamples, rollupRateSample{timestamp: lastTS, value: lastVal})
|
||||
}
|
||||
return a.groups[key]
|
||||
}
|
||||
|
||||
func mergeEncodedRollupDigest(target *TDigest, count int64, min, max float64, blob []byte) error {
|
||||
if count == 0 {
|
||||
return nil
|
||||
}
|
||||
if len(blob) == 0 {
|
||||
if min != max {
|
||||
return fmt.Errorf("metric: missing t-digest for rollup with %d samples", count)
|
||||
}
|
||||
target.Add(min, float64(count))
|
||||
return nil
|
||||
}
|
||||
digest, err := DecodeTDigest(blob)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
target.Merge(digest)
|
||||
return nil
|
||||
}
|
||||
|
||||
func mergeRollupValues(target *rollupBucket, count int64, sum, sumSq, min, max, firstVal float64, firstTS int64, lastVal float64, lastTS int64, digest *TDigest, includeDigest bool) {
|
||||
if count == 0 {
|
||||
return
|
||||
}
|
||||
if target.count == 0 {
|
||||
target.min, target.max = min, max
|
||||
target.firstVal, target.firstTS = firstVal, firstTS
|
||||
target.lastVal, target.lastTS = lastVal, lastTS
|
||||
} else {
|
||||
if min < target.min {
|
||||
target.min = min
|
||||
}
|
||||
if max > target.max {
|
||||
target.max = max
|
||||
}
|
||||
if firstTS < target.firstTS {
|
||||
target.firstVal, target.firstTS = firstVal, firstTS
|
||||
}
|
||||
if lastTS > target.lastTS {
|
||||
target.lastVal, target.lastTS = lastVal, lastTS
|
||||
}
|
||||
}
|
||||
target.count += count
|
||||
target.sum += sum
|
||||
target.sumSq += sumSq
|
||||
if includeDigest && digest != nil {
|
||||
target.digest.Merge(digest)
|
||||
}
|
||||
}
|
||||
|
||||
func (a *metricSeriesAccumulator) points(order Order) (map[Aggregation][]AggregatePoint, error) {
|
||||
values := make(map[Aggregation][]AggregatePoint, len(a.spec.Aggregations))
|
||||
var summaryKeys []rollupKey
|
||||
if a.needSummary {
|
||||
summaryKeys = make([]rollupKey, 0, len(a.groups))
|
||||
for key := range a.groups {
|
||||
summaryKeys = append(summaryKeys, key)
|
||||
}
|
||||
sortRollupKeys(summaryKeys)
|
||||
}
|
||||
var rateKeys []rollupKey
|
||||
if a.needRate {
|
||||
rateKeys = make([]rollupKey, 0, len(a.rateGroups))
|
||||
for key, state := range a.rateGroups {
|
||||
rateKeys = append(rateKeys, key)
|
||||
sort.SliceStable(state.rateSamples, func(i, j int) bool { return state.rateSamples[i].timestamp < state.rateSamples[j].timestamp })
|
||||
}
|
||||
sortRollupKeys(rateKeys)
|
||||
}
|
||||
for _, aggregation := range a.spec.Aggregations {
|
||||
keys := summaryKeys
|
||||
if aggregation == AggRate {
|
||||
keys = rateKeys
|
||||
}
|
||||
start, end, step := 0, len(keys), 1
|
||||
if order == OrderDesc {
|
||||
start, end, step = len(keys)-1, -1, -1
|
||||
}
|
||||
points := make([]AggregatePoint, 0, len(keys))
|
||||
for keyIndex := start; keyIndex != end; keyIndex += step {
|
||||
key := keys[keyIndex]
|
||||
state := a.groups[key]
|
||||
count, value := 0, 0.0
|
||||
if aggregation == AggRate {
|
||||
state = a.rateGroups[key]
|
||||
value = rollupCounterRate(state.rateSamples)
|
||||
count = len(state.rateSamples)
|
||||
} else {
|
||||
var ok bool
|
||||
value, ok = state.summary.value(aggregation)
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("%w: aggregation %q requires raw samples", ErrInvalidArgument, aggregation)
|
||||
}
|
||||
count = int(state.summary.count)
|
||||
}
|
||||
points = append(points, AggregatePoint{
|
||||
MetricName: a.spec.MetricName,
|
||||
EntityID: key.entityID,
|
||||
Bucket: fromMillis(key.bucket),
|
||||
Value: value,
|
||||
Count: count,
|
||||
Tags: state.tags,
|
||||
})
|
||||
}
|
||||
values[aggregation] = points
|
||||
}
|
||||
return values, nil
|
||||
}
|
||||
|
||||
func rollupCounterRate(samples []rollupRateSample) float64 {
|
||||
if len(samples) < 2 {
|
||||
return 0
|
||||
}
|
||||
seconds := float64(samples[len(samples)-1].timestamp-samples[0].timestamp) / 1000
|
||||
if seconds <= 0 {
|
||||
return 0
|
||||
}
|
||||
increase := 0.0
|
||||
for i := 1; i < len(samples); i++ {
|
||||
if delta := samples[i].value - samples[i-1].value; delta > 0 {
|
||||
increase += delta
|
||||
}
|
||||
}
|
||||
return increase / seconds
|
||||
}
|
||||
@@ -104,6 +104,12 @@ func (s *Store) addCoarseChildren(metricName string, interval time.Duration, hig
|
||||
// late-arrival window. It flushes one tier at a time so a sealed child can feed
|
||||
// the next parent without any periodic rewrite of that parent.
|
||||
func (s *Store) flushClosedCoarseRollups(ctx context.Context, now time.Time) (int, error) {
|
||||
s.rollupViewMu.Lock()
|
||||
defer s.rollupViewMu.Unlock()
|
||||
return s.flushClosedCoarseRollupsUnderView(ctx, now)
|
||||
}
|
||||
|
||||
func (s *Store) flushClosedCoarseRollupsUnderView(ctx context.Context, now time.Time) (int, error) {
|
||||
now = now.UTC()
|
||||
written := 0
|
||||
for {
|
||||
|
||||
@@ -158,11 +158,11 @@ func DefaultConfig(driver Driver, dsn string) Config {
|
||||
SQLite: SQLiteOptions{
|
||||
PerformanceProfile: SQLiteProfileBalanced,
|
||||
BusyTimeout: 5 * time.Second,
|
||||
CacheSizeKB: 16 * 1024,
|
||||
CacheSizeKB: 4 * 1024,
|
||||
TempStoreMemory: false,
|
||||
MMapSizeBytes: 32 * 1024 * 1024,
|
||||
MMapSizeBytes: 0,
|
||||
WALAutoCheckpoint: 4000,
|
||||
JournalSizeLimitBytes: 16 * 1024 * 1024,
|
||||
JournalSizeLimitBytes: 4 * 1024 * 1024,
|
||||
},
|
||||
RollupPolicy: RollupPolicy{
|
||||
RawRetention: 10 * time.Minute,
|
||||
|
||||
+2
-62
@@ -35,6 +35,8 @@ type dialect interface {
|
||||
//
|
||||
// blobType 是 rollup t-digest sketch 的列类型。
|
||||
blobType() string
|
||||
renderSeriesDictionary(tables tables, indexName string, plan seriesDictionaryPlan) renderedSQL
|
||||
renderRollupRead(tables tables, indexName string, plan rollupReadPlan) renderedSQL
|
||||
}
|
||||
|
||||
// tables stores the physical table names used by a Store.
|
||||
@@ -287,65 +289,3 @@ func (postgresDialect) jsonExtractEquals(column, key, placeholder string) string
|
||||
// escaping.
|
||||
return fmt.Sprintf("%s->>'%s' = %s", column, sqlSingleQuote(key), placeholder)
|
||||
}
|
||||
|
||||
// sqlAggValueExpr returns the SQL value expression that computes agg over the
|
||||
// "value" column for the given driver, and whether that aggregation can be
|
||||
// pushed down to this backend. Aggregations that need the ordered raw series
|
||||
// (first/last/rate) are never pushed down here. Percentiles and population
|
||||
// standard deviation are pushed down only where the backend has a portable,
|
||||
// semantics-matching function:
|
||||
//
|
||||
// - STDDEV_POP: MySQL and PostgreSQL (matches the in-memory population stddev,
|
||||
// which divides by N). SQLite has no built-in, so it falls back to memory.
|
||||
// - percentile_cont(p) WITHIN GROUP (ORDER BY value): PostgreSQL only. This
|
||||
// matches the in-memory linear-interpolation percentile for ANY p (p50,
|
||||
// p95, p99, p99.9, ...). MySQL lacks a portable continuous-percentile
|
||||
// function and SQLite has none, so both fall back to memory.
|
||||
//
|
||||
// The simple reductions (avg/min/max/sum/count) are portable everywhere.
|
||||
//
|
||||
// sqlAggValueExpr 返回给定驱动上对 "value" 列计算 agg 的 SQL 值表达式,
|
||||
// 以及该聚合是否能下推到该后端。需要有序原始序列的聚合(first/last/rate)
|
||||
// 永远不会在这里下推。百分位和总体标准差只会在后端提供可移植且语义匹配的函数时
|
||||
// 下推:
|
||||
//
|
||||
// - STDDEV_POP:MySQL 和 PostgreSQL 支持(匹配内存中的总体标准差,
|
||||
// 即除以 N)。SQLite 没有内置函数,因此回退到内存计算。
|
||||
// - percentile_cont(p) WITHIN GROUP (ORDER BY value):仅 PostgreSQL 支持。
|
||||
// 它匹配内存中任意 p(p50、p95、p99、p99.9 等)的线性插值百分位。
|
||||
// MySQL 没有可移植的连续百分位函数,SQLite 也没有,因此两者都回退到内存计算。
|
||||
//
|
||||
// 简单归约(avg/min/max/sum/count)在所有后端都可移植。
|
||||
func sqlAggValueExpr(driver Driver, agg Aggregation) (string, bool) {
|
||||
switch agg {
|
||||
case AggAvg:
|
||||
return "AVG(value)", true
|
||||
case AggMin:
|
||||
return "MIN(value)", true
|
||||
case AggMax:
|
||||
return "MAX(value)", true
|
||||
case AggSum:
|
||||
return "SUM(value)", true
|
||||
case AggCount:
|
||||
return "COUNT(*)", true
|
||||
case AggStdDev:
|
||||
switch driver {
|
||||
case DriverMySQL, DriverPostgreSQL:
|
||||
return "STDDEV_POP(value)", true
|
||||
default:
|
||||
return "", false
|
||||
}
|
||||
default:
|
||||
// Any percentile (fixed p50/p95/p99 and arbitrary pXX / pXX.X all land
|
||||
// here). PostgreSQL has a portable continuous percentile that matches the
|
||||
// in-memory linear-interpolation percentile; other backends fall back to
|
||||
// the in-memory path.
|
||||
if frac, ok := percentileFractionString(agg); ok {
|
||||
if driver == DriverPostgreSQL {
|
||||
return fmt.Sprintf("percentile_cont(%s) WITHIN GROUP (ORDER BY value)", frac), true
|
||||
}
|
||||
return "", false
|
||||
}
|
||||
return "", false
|
||||
}
|
||||
}
|
||||
@@ -30,7 +30,11 @@ var (
|
||||
|
||||
func getDigestEncoder() (*zstd.Encoder, error) {
|
||||
digestEncoderOnce.Do(func() {
|
||||
digestEncoder, digestEncoderErr = zstd.NewWriter(nil, zstd.WithEncoderLevel(zstd.EncoderLevelFromZstd(1)))
|
||||
digestEncoder, digestEncoderErr = zstd.NewWriter(nil,
|
||||
zstd.WithEncoderLevel(zstd.EncoderLevelFromZstd(1)),
|
||||
zstd.WithEncoderConcurrency(1),
|
||||
zstd.WithLowerEncoderMem(true),
|
||||
)
|
||||
})
|
||||
return digestEncoder, digestEncoderErr
|
||||
}
|
||||
|
||||
@@ -60,7 +60,11 @@ func (s *Store) writePreparedHotRollups(ctx context.Context, prepared []prepared
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
_, err = s.flushClosedCoarseRollups(ctx, now)
|
||||
if len(rebuild) > 0 {
|
||||
_, err = s.flushClosedCoarseRollupsUnderView(ctx, now)
|
||||
} else {
|
||||
_, err = s.flushClosedCoarseRollups(ctx, now)
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -78,8 +82,25 @@ func (s *Store) rebuildHotRollupsLocked(keys map[hotRollupKey]struct{}, compress
|
||||
if labelID, ok := series.labelIDs[key.labelsHash]; ok {
|
||||
bucket.labelsJSON = series.labelsJSON[labelID]
|
||||
}
|
||||
for _, sample := range series.allSamples() {
|
||||
if bucketStartMillis(sample.timestamp, minuteMillis) != key.bucket || series.labelHashes[sample.labelID] != key.labelsHash {
|
||||
bucketEnd := key.bucket + minuteMillis
|
||||
decoder := newRawSampleDecoder(series.compressed)
|
||||
for sample, more := decoder.next(); more; sample, more = decoder.next() {
|
||||
if sample.timestamp < key.bucket {
|
||||
continue
|
||||
}
|
||||
if sample.timestamp >= bucketEnd {
|
||||
break
|
||||
}
|
||||
if series.labelHashes[sample.labelID] != key.labelsHash {
|
||||
continue
|
||||
}
|
||||
bucket.labelsJSON = series.labelsJSON[sample.labelID]
|
||||
bucket.addPoint(sample.value, sample.timestamp)
|
||||
}
|
||||
directStart := sort.Search(len(series.samples), func(i int) bool { return series.samples[i].timestamp >= key.bucket })
|
||||
directEnd := sort.Search(len(series.samples), func(i int) bool { return series.samples[i].timestamp >= bucketEnd })
|
||||
for _, sample := range series.samples[directStart:directEnd] {
|
||||
if series.labelHashes[sample.labelID] != key.labelsHash {
|
||||
continue
|
||||
}
|
||||
bucket.labelsJSON = series.labelsJSON[sample.labelID]
|
||||
|
||||
@@ -634,7 +634,7 @@ func TestSeriesMergesLabelsWithoutCollapsingRawShape(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestCoveringSeriesResolutionUsesFinestTierCoveringStart(t *testing.T) {
|
||||
func TestSeriesResolutionDoesNotInspectStoredCoverage(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
policy := RollupPolicy{
|
||||
Tiers: []RollupTier{
|
||||
@@ -686,16 +686,13 @@ func TestCoveringSeriesResolutionUsesFinestTierCoveringStart(t *testing.T) {
|
||||
t.Fatalf("commit seed tiers: %v", err)
|
||||
}
|
||||
|
||||
resolution, err := s.coveringSeriesResolution(ctx, Query{MetricName: "coverage-tier", EntityID: "n1", Start: now.Add(-6 * time.Hour), End: now}, time.Minute)
|
||||
if err != nil {
|
||||
t.Fatalf("select covering tier: %v", err)
|
||||
}
|
||||
if resolution != 5*time.Minute {
|
||||
t.Fatalf("covering tier = %s, want 5m", resolution)
|
||||
resolution := seriesResolutionForPolicy(now.Add(-6*time.Hour), time.Minute, now, policy)
|
||||
if resolution != time.Minute {
|
||||
t.Fatalf("selected tier = %s, want 1m", resolution)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSeriesFallsBackWhenPreferredPolicyTierHasNoRows(t *testing.T) {
|
||||
func TestSeriesReturnsEmptyWhenSelectedTierHasNoRows(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
policy := RollupPolicy{
|
||||
Tiers: []RollupTier{
|
||||
@@ -743,8 +740,8 @@ func TestSeriesFallsBackWhenPreferredPolicyTierHasNoRows(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatalf("query series: %v", err)
|
||||
}
|
||||
if len(got) != 1 || got[0].Value != 42 || got[0].Count != 1 {
|
||||
t.Fatalf("series did not fall back to existing hourly tier: %#v", got)
|
||||
if len(got) != 0 {
|
||||
t.Fatalf("series read a non-selected hourly tier: %#v", got)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -931,15 +928,33 @@ func TestSeriesHasNoTrailingEmptyBucketAcrossDashboardRanges(t *testing.T) {
|
||||
t.Fatalf("create metric: %v", err)
|
||||
}
|
||||
now := time.Date(2026, 7, 24, 12, 0, 0, 0, time.UTC)
|
||||
points := make([]Point, 0, 60*24*60)
|
||||
for at := now.Add(-60 * 24 * time.Hour); at.Before(now); at = at.Add(time.Minute) {
|
||||
points = append(points, Point{MetricName: "coverage", EntityID: "n1", Timestamp: at, Value: 1})
|
||||
tagsHash, tagsJSON, err := tagsFingerprint(nil)
|
||||
if err != nil {
|
||||
t.Fatalf("fingerprint tags: %v", err)
|
||||
}
|
||||
if err := s.WriteBatch(ctx, points); err != nil {
|
||||
t.Fatalf("write history: %v", err)
|
||||
tx, err := s.db.BeginTx(ctx, nil)
|
||||
if err != nil {
|
||||
t.Fatalf("begin history transaction: %v", err)
|
||||
}
|
||||
if _, err := s.Compact(ctx, now); err != nil {
|
||||
t.Fatalf("compact history: %v", err)
|
||||
metricPolicy := policy.withMetricRetention(90 * 24 * time.Hour)
|
||||
for _, tierIndex := range []int{0, 2, 3} {
|
||||
tier := metricPolicy.Tiers[tierIndex]
|
||||
buckets := make(map[rollupKey]*rollupBucket)
|
||||
start := bucketStartMillis(now.Add(-tier.Retention).UnixMilli(), tier.Interval.Milliseconds())
|
||||
for bucketStart := start; bucketStart < now.UnixMilli(); bucketStart += tier.Interval.Milliseconds() {
|
||||
bucket := newRollupBucket(policy.compression())
|
||||
bucket.tagsHash, bucket.tagsJSON = tagsHash, tagsJSON
|
||||
bucket.labelsHash, bucket.labelsJSON = emptyLabelsHash, "{}"
|
||||
bucket.addPoint(1, bucketStart)
|
||||
buckets[rollupKey{entityID: "n1", tagsHash: tagsHash, labelsHash: emptyLabelsHash, bucket: bucketStart}] = bucket
|
||||
}
|
||||
if _, err := s.writeRollupBucketsTx(ctx, "coverage", tier.Interval, buckets, tx); err != nil {
|
||||
_ = tx.Rollback()
|
||||
t.Fatalf("seed %s history: %v", tier.Interval, err)
|
||||
}
|
||||
}
|
||||
if err := tx.Commit(); err != nil {
|
||||
t.Fatalf("commit history: %v", err)
|
||||
}
|
||||
for _, duration := range []time.Duration{time.Hour, 6 * time.Hour, 7 * 24 * time.Hour, 30 * 24 * time.Hour, 60 * 24 * time.Hour} {
|
||||
interval := s.CompatibleSeriesInterval(now.Add(-duration), now, time.Minute)
|
||||
|
||||
@@ -104,10 +104,11 @@ func TestTagFilterPushdownWithPaging(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// TestSQLAggregateMatchesInMemory compares SQL and in-memory aggregation.
|
||||
// TestAggregateMatchesAggregatePoints compares Store aggregation with the
|
||||
// standalone aggregation helper.
|
||||
//
|
||||
// TestSQLAggregateMatchesInMemory 对比 SQL 下推聚合和内存聚合结果。
|
||||
func TestSQLAggregateMatchesInMemory(t *testing.T) {
|
||||
// TestAggregateMatchesAggregatePoints 对比 Store 聚合和独立聚合 helper 的结果。
|
||||
func TestAggregateMatchesAggregatePoints(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
s := newMemStore(t)
|
||||
if err := s.CreateMetric(ctx, Definition{Name: "g", Type: TypeGauge, RetentionDays: 30}); err != nil {
|
||||
@@ -131,10 +132,9 @@ func TestSQLAggregateMatchesInMemory(t *testing.T) {
|
||||
Aggregation: AggAvg,
|
||||
Interval: 15 * time.Second,
|
||||
}
|
||||
// SQL pushdown path.
|
||||
sqlRes, err := s.Aggregate(ctx, q)
|
||||
storeRes, err := s.Aggregate(ctx, q)
|
||||
if err != nil {
|
||||
t.Fatalf("sql aggregate: %v", err)
|
||||
t.Fatalf("store aggregate: %v", err)
|
||||
}
|
||||
// In-memory reference over the same raw points.
|
||||
pts, err := s.Query(ctx, q.Query)
|
||||
@@ -145,12 +145,12 @@ func TestSQLAggregateMatchesInMemory(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatalf("mem aggregate: %v", err)
|
||||
}
|
||||
if len(sqlRes) != len(memRes) {
|
||||
t.Fatalf("bucket count mismatch: sql=%d mem=%d", len(sqlRes), len(memRes))
|
||||
if len(storeRes) != len(memRes) {
|
||||
t.Fatalf("bucket count mismatch: store=%d helper=%d", len(storeRes), len(memRes))
|
||||
}
|
||||
for i := range sqlRes {
|
||||
if !sqlRes[i].Bucket.Equal(memRes[i].Bucket) || sqlRes[i].Value != memRes[i].Value || sqlRes[i].Count != memRes[i].Count {
|
||||
t.Fatalf("bucket %d mismatch: sql=%#v mem=%#v", i, sqlRes[i], memRes[i])
|
||||
for i := range storeRes {
|
||||
if !storeRes[i].Bucket.Equal(memRes[i].Bucket) || storeRes[i].Value != memRes[i].Value || storeRes[i].Count != memRes[i].Count {
|
||||
t.Fatalf("bucket %d mismatch: store=%#v helper=%#v", i, storeRes[i], memRes[i])
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -260,10 +260,10 @@ func TestStdDevPopMatchesCalculateStats(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// TestAggregateStdDevSQLiteUsesMemoryPath verifies SQLite stddev fallback.
|
||||
// TestAggregateStdDev verifies raw population standard deviation aggregation.
|
||||
//
|
||||
// TestAggregateStdDevSQLiteUsesMemoryPath 验证 SQLite 标准差聚合会回退到内存路径。
|
||||
func TestAggregateStdDevSQLiteUsesMemoryPath(t *testing.T) {
|
||||
// TestAggregateStdDev 验证原始样本总体标准差聚合。
|
||||
func TestAggregateStdDev(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
s := newMemStore(t)
|
||||
if err := s.CreateMetric(ctx, Definition{Name: "sd", Type: TypeGauge, RetentionDays: 30}); err != nil {
|
||||
@@ -294,36 +294,6 @@ func TestAggregateStdDevSQLiteUsesMemoryPath(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// TestSQLAggValueExprPushdownMatrix verifies aggregation pushdown support.
|
||||
//
|
||||
// TestSQLAggValueExprPushdownMatrix 验证各后端支持的聚合下推矩阵。
|
||||
func TestSQLAggValueExprPushdownMatrix(t *testing.T) {
|
||||
cases := []struct {
|
||||
driver Driver
|
||||
agg Aggregation
|
||||
ok bool
|
||||
}{
|
||||
{DriverSQLite, AggAvg, true},
|
||||
{DriverSQLite, AggStdDev, false}, // no STDDEV_POP in sqlite -> memory
|
||||
{DriverSQLite, AggP95, false}, // no percentile in sqlite -> memory
|
||||
{DriverMySQL, AggStdDev, true}, // STDDEV_POP
|
||||
{DriverMySQL, AggP95, false}, // no portable continuous percentile
|
||||
{DriverPostgreSQL, AggStdDev, true},
|
||||
{DriverPostgreSQL, AggP50, true}, // percentile_cont
|
||||
{DriverPostgreSQL, AggP99, true},
|
||||
{DriverPostgreSQL, AggFirst, false}, // needs ordered raw series
|
||||
}
|
||||
for _, c := range cases {
|
||||
expr, ok := sqlAggValueExpr(c.driver, c.agg)
|
||||
if ok != c.ok {
|
||||
t.Fatalf("%s/%s: pushdown=%v want %v (expr=%q)", c.driver, c.agg, ok, c.ok, expr)
|
||||
}
|
||||
if ok && expr == "" {
|
||||
t.Fatalf("%s/%s: pushdown ok but empty expr", c.driver, c.agg)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestSQLiteReadPoolOpens verifies SQLite read-pool creation.
|
||||
//
|
||||
// TestSQLiteReadPoolOpens 验证 SQLite 只读连接池会按配置打开。
|
||||
@@ -403,10 +373,10 @@ func TestWriteBatchAtomicAcrossChunks(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// TestAggregateBucketPagingSQLPath verifies bucket paging in SQL aggregation.
|
||||
// TestAggregateBucketPaging verifies bucket paging.
|
||||
//
|
||||
// TestAggregateBucketPagingSQLPath 验证 SQL 聚合路径按桶分页。
|
||||
func TestAggregateBucketPagingSQLPath(t *testing.T) {
|
||||
// TestAggregateBucketPaging 验证聚合桶分页。
|
||||
func TestAggregateBucketPaging(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
s := newMemStore(t)
|
||||
if err := s.CreateMetric(ctx, Definition{Name: "bp", Type: TypeGauge, RetentionDays: 30}); err != nil {
|
||||
@@ -441,10 +411,11 @@ func TestAggregateBucketPagingSQLPath(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// TestAggregateBucketPagingMemoryPathMatchesSQL compares bucket paging paths.
|
||||
// TestAggregateBucketPagingMatchesAcrossAggregations compares bucket paging
|
||||
// across aggregation types.
|
||||
//
|
||||
// TestAggregateBucketPagingMemoryPathMatchesSQL 对比内存聚合和 SQL 聚合的桶分页语义。
|
||||
func TestAggregateBucketPagingMemoryPathMatchesSQL(t *testing.T) {
|
||||
// TestAggregateBucketPagingMatchesAcrossAggregations 对比不同聚合类型的桶分页语义。
|
||||
func TestAggregateBucketPagingMatchesAcrossAggregations(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
s := newMemStore(t)
|
||||
if err := s.CreateMetric(ctx, Definition{Name: "bp2", Type: TypeGauge, RetentionDays: 30}); err != nil {
|
||||
@@ -459,28 +430,26 @@ func TestAggregateBucketPagingMemoryPathMatchesSQL(t *testing.T) {
|
||||
t.Fatalf("write: %v", err)
|
||||
}
|
||||
q := Query{MetricName: "bp2", EntityID: "n1", Start: base, End: base.Add(time.Minute)}
|
||||
// AggMin (SQL pushdown) and AggP50 (memory fallback on sqlite) with identical
|
||||
// bucket paging must select the same bucket window (the same timestamps).
|
||||
sqlPaged, err := s.Aggregate(ctx, AggregateQuery{Query: q, Aggregation: AggMin, Interval: 10 * time.Second, BucketLimit: 3, BucketOffset: 2})
|
||||
minPaged, err := s.Aggregate(ctx, AggregateQuery{Query: q, Aggregation: AggMin, Interval: 10 * time.Second, BucketLimit: 3, BucketOffset: 2})
|
||||
if err != nil {
|
||||
t.Fatalf("sql aggregate: %v", err)
|
||||
t.Fatalf("min aggregate: %v", err)
|
||||
}
|
||||
memPaged, err := s.Aggregate(ctx, AggregateQuery{Query: q, Aggregation: AggP50, Interval: 10 * time.Second, BucketLimit: 3, BucketOffset: 2})
|
||||
p50Paged, err := s.Aggregate(ctx, AggregateQuery{Query: q, Aggregation: AggP50, Interval: 10 * time.Second, BucketLimit: 3, BucketOffset: 2})
|
||||
if err != nil {
|
||||
t.Fatalf("mem aggregate: %v", err)
|
||||
t.Fatalf("p50 aggregate: %v", err)
|
||||
}
|
||||
if len(sqlPaged) != 3 || len(memPaged) != 3 {
|
||||
t.Fatalf("expected 3 buckets each, got sql=%d mem=%d", len(sqlPaged), len(memPaged))
|
||||
if len(minPaged) != 3 || len(p50Paged) != 3 {
|
||||
t.Fatalf("expected 3 buckets each, got min=%d p50=%d", len(minPaged), len(p50Paged))
|
||||
}
|
||||
for i := range sqlPaged {
|
||||
if !sqlPaged[i].Bucket.Equal(memPaged[i].Bucket) {
|
||||
t.Fatalf("bucket window mismatch at %d: sql=%v mem=%v", i, sqlPaged[i].Bucket, memPaged[i].Bucket)
|
||||
for i := range minPaged {
|
||||
if !minPaged[i].Bucket.Equal(p50Paged[i].Bucket) {
|
||||
t.Fatalf("bucket window mismatch at %d: min=%v p50=%v", i, minPaged[i].Bucket, p50Paged[i].Bucket)
|
||||
}
|
||||
}
|
||||
// Per-bucket single point, so min == p50 == the value.
|
||||
for i := range sqlPaged {
|
||||
if sqlPaged[i].Value != memPaged[i].Value {
|
||||
t.Fatalf("value mismatch at %d: sql=%v mem=%v", i, sqlPaged[i].Value, memPaged[i].Value)
|
||||
for i := range minPaged {
|
||||
if minPaged[i].Value != p50Paged[i].Value {
|
||||
t.Fatalf("value mismatch at %d: min=%v p50=%v", i, minPaged[i].Value, p50Paged[i].Value)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -125,6 +125,9 @@ func (s *Store) ReclaimSpace(ctx context.Context) error {
|
||||
if _, err := s.db.ExecContext(ctx, sqliteVacuumSQL); err != nil {
|
||||
return fmt.Errorf("metric: vacuum sqlite database: %w", err)
|
||||
}
|
||||
if _, err := s.db.ExecContext(ctx, "PRAGMA optimize"); err != nil {
|
||||
return fmt.Errorf("metric: optimize sqlite database: %w", err)
|
||||
}
|
||||
// VACUUM itself can populate the WAL; truncate it again so the reported
|
||||
// physical size reflects the completed reclamation.
|
||||
return sqliteCheckpoint(ctx, s.db)
|
||||
|
||||
@@ -21,7 +21,14 @@ func (s *Store) Migrate(ctx context.Context) error {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return s.createNormalizedIndexes(ctx)
|
||||
if err := s.createNormalizedIndexes(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
if s.cfg.Driver == DriverSQLite {
|
||||
_, err := s.db.ExecContext(ctx, "PRAGMA optimize")
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *Store) normalizedSchemaStatements() []string {
|
||||
|
||||
@@ -18,7 +18,7 @@ func TestPostgreSQLIntegration(t *testing.T) {
|
||||
t.Skip("METRIC_POSTGRES_DSN is not set")
|
||||
}
|
||||
|
||||
runSQLIntegration(t, "postgres", PostgreSQL(dsn), true)
|
||||
runSQLIntegration(t, "postgres", PostgreSQL(dsn))
|
||||
runSQLRestructureIntegration(t, "postgres", PostgreSQL(dsn))
|
||||
}
|
||||
|
||||
@@ -31,14 +31,24 @@ func TestMySQLIntegration(t *testing.T) {
|
||||
t.Skip("METRIC_MYSQL_DSN is not set")
|
||||
}
|
||||
|
||||
runSQLIntegration(t, "mysql", MySQL(dsn), false)
|
||||
runSQLIntegration(t, "mysql", MySQL(dsn))
|
||||
runSQLRestructureIntegration(t, "mysql", MySQL(dsn))
|
||||
}
|
||||
|
||||
func TestMariaDBIntegration(t *testing.T) {
|
||||
dsn := os.Getenv("METRIC_MARIADB_DSN")
|
||||
if dsn == "" {
|
||||
t.Skip("METRIC_MARIADB_DSN is not set")
|
||||
}
|
||||
|
||||
runSQLIntegration(t, "mariadb", MySQL(dsn))
|
||||
runSQLRestructureIntegration(t, "mariadb", MySQL(dsn))
|
||||
}
|
||||
|
||||
// runSQLIntegration exercises the SQL store against an external database.
|
||||
//
|
||||
// runSQLIntegration 在外部数据库上执行通用 SQL 集成测试流程。
|
||||
func runSQLIntegration(t *testing.T, name string, cfg Config, expectSQLPercentile bool) {
|
||||
func runSQLIntegration(t *testing.T, name string, cfg Config) {
|
||||
t.Helper()
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 6*time.Minute)
|
||||
@@ -160,10 +170,6 @@ func runSQLIntegration(t *testing.T, name string, cfg Config, expectSQLPercentil
|
||||
if len(p95Buckets) != 1 || p95Buckets[0].Count != 3 || p95Buckets[0].Value <= 28 {
|
||||
t.Fatalf("unexpected p95 aggregate: %#v", p95Buckets)
|
||||
}
|
||||
if _, ok := sqlAggValueExpr(cfg.Driver, AggP95); ok != expectSQLPercentile {
|
||||
t.Fatalf("%s p95 pushdown expectation mismatch: got %v want %v", name, ok, expectSQLPercentile)
|
||||
}
|
||||
|
||||
latest, err := store.Latest(ctx, "http.latency", "api-1", 1)
|
||||
if err != nil {
|
||||
t.Fatalf("latest: %v", err)
|
||||
|
||||
+308
-173
@@ -38,6 +38,61 @@ type compressedRawSamples struct {
|
||||
lastStamp int64
|
||||
}
|
||||
|
||||
type rawSampleDecoder struct {
|
||||
data []byte
|
||||
offset int
|
||||
index int
|
||||
count int
|
||||
previous int64
|
||||
}
|
||||
|
||||
func newRawSampleDecoder(samples compressedRawSamples) rawSampleDecoder {
|
||||
return rawSampleDecoder{data: samples.data, count: samples.count}
|
||||
}
|
||||
|
||||
func (d *rawSampleDecoder) next() (rawSample, bool) {
|
||||
if d.index == d.count {
|
||||
return rawSample{}, false
|
||||
}
|
||||
var timestamp int64
|
||||
if d.index == 0 {
|
||||
value, size := binary.Varint(d.data[d.offset:])
|
||||
d.offset += size
|
||||
timestamp = value
|
||||
} else {
|
||||
delta, size := binary.Uvarint(d.data[d.offset:])
|
||||
d.offset += size
|
||||
timestamp = d.previous + int64(delta)
|
||||
}
|
||||
value := math.Float64frombits(binary.LittleEndian.Uint64(d.data[d.offset:]))
|
||||
d.offset += 8
|
||||
labelID, size := binary.Uvarint(d.data[d.offset:])
|
||||
d.offset += size
|
||||
d.index++
|
||||
d.previous = timestamp
|
||||
return rawSample{timestamp: timestamp, value: value, labelID: uint32(labelID)}, true
|
||||
}
|
||||
|
||||
func newCompressedRawSamples(capacity int) compressedRawSamples {
|
||||
return compressedRawSamples{data: make([]byte, 0, capacity)}
|
||||
}
|
||||
|
||||
func (c *compressedRawSamples) appendSample(sample rawSample) {
|
||||
if c.count == 0 {
|
||||
c.data = appendVarint(c.data, sample.timestamp)
|
||||
} else {
|
||||
c.data = appendUvarint(c.data, uint64(sample.timestamp-c.lastStamp))
|
||||
}
|
||||
c.data = appendRawSample(c.data, sample)
|
||||
c.count++
|
||||
c.lastStamp = sample.timestamp
|
||||
}
|
||||
|
||||
func (c compressedRawSamples) firstStamp() int64 {
|
||||
value, _ := binary.Varint(c.data)
|
||||
return value
|
||||
}
|
||||
|
||||
// rawSeries stores exact samples compactly for one logical series. Tags are
|
||||
// shared by the series and labels are interned into small integer ids.
|
||||
type rawSeries struct {
|
||||
@@ -170,31 +225,98 @@ func markRawRebuild(rebuild map[hotRollupKey]struct{}, point preparedMetricPoint
|
||||
rebuild[hotRollupKey{metricName: point.metricName, entityID: point.entityID, tagsHash: point.tagsHash, labelsHash: labelsHash, bucket: bucketStartMillis(point.timestamp, time.Minute.Milliseconds())}] = struct{}{}
|
||||
}
|
||||
|
||||
type rawBatchPoints struct {
|
||||
points []Point
|
||||
tagsHashes []string
|
||||
order Order
|
||||
}
|
||||
|
||||
func (p *rawBatchPoints) Len() int { return len(p.points) }
|
||||
|
||||
func (p *rawBatchPoints) Less(i, j int) bool {
|
||||
if p.order == OrderDesc {
|
||||
return rawPointLess(p.points[j], p.tagsHashes[j], p.points[i], p.tagsHashes[i])
|
||||
}
|
||||
return rawPointLess(p.points[i], p.tagsHashes[i], p.points[j], p.tagsHashes[j])
|
||||
}
|
||||
|
||||
func (p *rawBatchPoints) Swap(i, j int) {
|
||||
p.points[i], p.points[j] = p.points[j], p.points[i]
|
||||
p.tagsHashes[i], p.tagsHashes[j] = p.tagsHashes[j], p.tagsHashes[i]
|
||||
}
|
||||
|
||||
func (p *rawBatchPoints) grow(additional int) {
|
||||
if additional <= cap(p.points)-len(p.points) {
|
||||
return
|
||||
}
|
||||
pointCount := len(p.points) + additional
|
||||
points := make([]Point, len(p.points), pointCount)
|
||||
copy(points, p.points)
|
||||
p.points = points
|
||||
tagsHashes := make([]string, len(p.tagsHashes), pointCount)
|
||||
copy(tagsHashes, p.tagsHashes)
|
||||
p.tagsHashes = tagsHashes
|
||||
}
|
||||
|
||||
func (s *Store) queryRawPoints(ctx context.Context, query Query) ([]Point, error) {
|
||||
entityIDs := []string(nil)
|
||||
if query.EntityID != "" {
|
||||
entityIDs = []string{query.EntityID}
|
||||
}
|
||||
batch, err := s.queryRawPointsBatch(ctx, BatchQuery{
|
||||
MetricNames: []string{query.MetricName},
|
||||
EntityIDs: entityIDs,
|
||||
Start: query.Start,
|
||||
End: query.End,
|
||||
Tags: query.Tags,
|
||||
Order: query.Order,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
points := batch[query.MetricName]
|
||||
startIndex := query.Offset
|
||||
if startIndex > len(points) {
|
||||
startIndex = len(points)
|
||||
}
|
||||
endIndex := len(points)
|
||||
if query.Limit > 0 && startIndex+query.Limit < endIndex {
|
||||
endIndex = startIndex + query.Limit
|
||||
}
|
||||
return points[startIndex:endIndex], nil
|
||||
}
|
||||
|
||||
func (s *Store) queryRawPointsBatch(ctx context.Context, query BatchQuery) (map[string][]Point, error) {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
start, end := query.Start.UnixMilli(), query.End.UnixMilli()
|
||||
s.rawMu.Lock()
|
||||
defer s.rawMu.Unlock()
|
||||
now := time.Now().UTC()
|
||||
visibleCutoff := now.Add(-rawMemoryRetention).UnixMilli()
|
||||
visibleCutoff := time.Now().UTC().Add(-rawMemoryRetention).UnixMilli()
|
||||
if start < visibleCutoff {
|
||||
start = visibleCutoff
|
||||
}
|
||||
s.compressRawBeforeLocked("", now.Add(-directRawRetention).UnixMilli())
|
||||
s.pruneRawBeforeLocked("", rawMemoryCutoff(now))
|
||||
|
||||
type resultPoint struct {
|
||||
point Point
|
||||
entityID string
|
||||
tagsHash string
|
||||
metricNames := make(map[string]struct{}, len(query.MetricNames))
|
||||
matched := make(map[string]*rawBatchPoints, len(query.MetricNames))
|
||||
for _, metricName := range query.MetricNames {
|
||||
metricNames[metricName] = struct{}{}
|
||||
matched[metricName] = &rawBatchPoints{order: query.Order}
|
||||
}
|
||||
matched := make([]resultPoint, 0)
|
||||
entityIDs := make(map[string]struct{}, len(query.EntityIDs))
|
||||
for _, entityID := range query.EntityIDs {
|
||||
entityIDs[entityID] = struct{}{}
|
||||
}
|
||||
|
||||
s.rawMu.RLock()
|
||||
defer s.rawMu.RUnlock()
|
||||
for key, series := range s.raw {
|
||||
if key.metricName != query.MetricName || (query.EntityID != "" && key.entityID != query.EntityID) {
|
||||
if _, ok := metricNames[key.metricName]; !ok {
|
||||
continue
|
||||
}
|
||||
if len(entityIDs) > 0 {
|
||||
if _, ok := entityIDs[key.entityID]; !ok {
|
||||
continue
|
||||
}
|
||||
}
|
||||
tags, ok, err := matchRawTags(series.tagsJSON, query.Tags)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
@@ -202,68 +324,88 @@ func (s *Store) queryRawPoints(ctx context.Context, query Query) ([]Point, error
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
all := series.allSamples()
|
||||
labels := make(map[uint32]map[string]string)
|
||||
first := sort.Search(len(all), func(i int) bool { return all[i].timestamp >= start })
|
||||
for _, sample := range all[first:] {
|
||||
if sample.timestamp > end {
|
||||
break
|
||||
}
|
||||
labelMap := labels[sample.labelID]
|
||||
if labelMap == nil {
|
||||
labelMap, err = decodeMapString(series.labelsJSON[sample.labelID])
|
||||
directStart := sort.Search(len(series.samples), func(i int) bool { return series.samples[i].timestamp >= start })
|
||||
directEnd := sort.Search(len(series.samples), func(i int) bool { return series.samples[i].timestamp > end })
|
||||
additional := directEnd - directStart
|
||||
if series.compressed.count > 0 && start <= series.compressed.firstStamp() && end >= series.compressed.lastStamp {
|
||||
additional += series.compressed.count
|
||||
}
|
||||
metricPoints := matched[key.metricName]
|
||||
metricPoints.grow(additional)
|
||||
if series.compressed.count > 0 && start <= series.compressed.lastStamp && end >= series.compressed.firstStamp() {
|
||||
decoder := newRawSampleDecoder(series.compressed)
|
||||
for sample, more := decoder.next(); more; sample, more = decoder.next() {
|
||||
if sample.timestamp < start {
|
||||
continue
|
||||
}
|
||||
if sample.timestamp > end {
|
||||
break
|
||||
}
|
||||
labelMap, err := rawSampleLabels(series, labels, sample.labelID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
labels[sample.labelID] = labelMap
|
||||
appendRawQueryPoint(metricPoints, key, tags, labelMap, sample)
|
||||
}
|
||||
matched = append(matched, resultPoint{
|
||||
point: Point{MetricName: key.metricName, EntityID: key.entityID, Timestamp: fromMillis(sample.timestamp), Value: sample.value, Tags: cloneStringMap(tags), Labels: cloneStringMap(labelMap)},
|
||||
entityID: key.entityID,
|
||||
tagsHash: key.tagsHash,
|
||||
})
|
||||
}
|
||||
for _, sample := range series.samples[directStart:directEnd] {
|
||||
labelMap, err := rawSampleLabels(series, labels, sample.labelID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
appendRawQueryPoint(metricPoints, key, tags, labelMap, sample)
|
||||
}
|
||||
}
|
||||
sort.Slice(matched, func(i, j int) bool {
|
||||
left, right := matched[i], matched[j]
|
||||
less := left.point.Timestamp.Before(right.point.Timestamp)
|
||||
if left.point.Timestamp.Equal(right.point.Timestamp) {
|
||||
less = left.entityID < right.entityID || (left.entityID == right.entityID && left.tagsHash < right.tagsHash)
|
||||
}
|
||||
if query.Order == OrderDesc {
|
||||
return !less && !(left.point.Timestamp.Equal(right.point.Timestamp) && left.entityID == right.entityID && left.tagsHash == right.tagsHash)
|
||||
}
|
||||
return less
|
||||
})
|
||||
startIndex := query.Offset
|
||||
if startIndex > len(matched) {
|
||||
startIndex = len(matched)
|
||||
}
|
||||
endIndex := len(matched)
|
||||
if query.Limit > 0 && startIndex+query.Limit < endIndex {
|
||||
endIndex = startIndex + query.Limit
|
||||
}
|
||||
result := make([]Point, 0, endIndex-startIndex)
|
||||
for _, item := range matched[startIndex:endIndex] {
|
||||
result = append(result, item.point)
|
||||
result := make(map[string][]Point, len(matched))
|
||||
for metricName, metricPoints := range matched {
|
||||
sort.Sort(metricPoints)
|
||||
result[metricName] = metricPoints.points
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func rawSampleLabels(series *rawSeries, labels map[uint32]map[string]string, labelID uint32) (map[string]string, error) {
|
||||
if labelMap := labels[labelID]; labelMap != nil {
|
||||
return labelMap, nil
|
||||
}
|
||||
labelMap, err := decodeMapString(series.labelsJSON[labelID])
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
labels[labelID] = labelMap
|
||||
return labelMap, nil
|
||||
}
|
||||
|
||||
func appendRawQueryPoint(points *rawBatchPoints, key rawSeriesKey, tags, labels map[string]string, sample rawSample) {
|
||||
points.points = append(points.points, Point{
|
||||
MetricName: key.metricName, EntityID: key.entityID,
|
||||
Timestamp: fromMillis(sample.timestamp), Value: sample.value, Tags: tags, Labels: labels,
|
||||
})
|
||||
points.tagsHashes = append(points.tagsHashes, key.tagsHash)
|
||||
}
|
||||
|
||||
func rawPointLess(left Point, leftTagsHash string, right Point, rightTagsHash string) bool {
|
||||
if !left.Timestamp.Equal(right.Timestamp) {
|
||||
return left.Timestamp.Before(right.Timestamp)
|
||||
}
|
||||
if left.EntityID != right.EntityID {
|
||||
return left.EntityID < right.EntityID
|
||||
}
|
||||
return leftTagsHash < rightTagsHash
|
||||
}
|
||||
|
||||
func (s *Store) rawEntityIDs(ctx context.Context, query Query) ([]string, error) {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
start, end := query.Start.UnixMilli(), query.End.UnixMilli()
|
||||
s.rawMu.Lock()
|
||||
defer s.rawMu.Unlock()
|
||||
now := time.Now().UTC()
|
||||
visibleCutoff := now.Add(-rawMemoryRetention).UnixMilli()
|
||||
s.rawMu.RLock()
|
||||
defer s.rawMu.RUnlock()
|
||||
visibleCutoff := time.Now().UTC().Add(-rawMemoryRetention).UnixMilli()
|
||||
if start < visibleCutoff {
|
||||
start = visibleCutoff
|
||||
}
|
||||
s.compressRawBeforeLocked("", now.Add(-directRawRetention).UnixMilli())
|
||||
s.pruneRawBeforeLocked("", rawMemoryCutoff(now))
|
||||
seen := make(map[string]struct{})
|
||||
for key, series := range s.raw {
|
||||
if key.metricName != query.MetricName || (query.EntityID != "" && key.entityID != query.EntityID) {
|
||||
@@ -276,9 +418,25 @@ func (s *Store) rawEntityIDs(ctx context.Context, query Query) ([]string, error)
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
all := series.allSamples()
|
||||
index := sort.Search(len(all), func(i int) bool { return all[i].timestamp >= start })
|
||||
if index < len(all) && all[index].timestamp <= end {
|
||||
found := false
|
||||
if series.compressed.count > 0 && start <= series.compressed.lastStamp && end >= series.compressed.firstStamp() {
|
||||
decoder := newRawSampleDecoder(series.compressed)
|
||||
for sample, more := decoder.next(); more; sample, more = decoder.next() {
|
||||
if sample.timestamp < start {
|
||||
continue
|
||||
}
|
||||
if sample.timestamp > end {
|
||||
break
|
||||
}
|
||||
found = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if !found {
|
||||
index := sort.Search(len(series.samples), func(i int) bool { return series.samples[i].timestamp >= start })
|
||||
found = index < len(series.samples) && series.samples[index].timestamp <= end
|
||||
}
|
||||
if found {
|
||||
seen[key.entityID] = struct{}{}
|
||||
}
|
||||
}
|
||||
@@ -355,32 +513,48 @@ func (s *Store) pruneRawBeforeLocked(metricName string, beforeMilli int64) int64
|
||||
if metricName != "" && key.metricName != metricName {
|
||||
continue
|
||||
}
|
||||
compressed := series.decodeCompressed()
|
||||
compressedIndex := sort.Search(len(compressed), func(i int) bool { return compressed[i].timestamp >= beforeMilli })
|
||||
if compressedIndex > 0 {
|
||||
compressed = compressed[compressedIndex:]
|
||||
series.compressed.set(compressed)
|
||||
deleted += int64(compressedIndex)
|
||||
}
|
||||
if len(compressed) == 0 {
|
||||
series.compressed = compressedRawSamples{}
|
||||
}
|
||||
compressedDeleted := pruneCompressedRawSamples(&series.compressed, beforeMilli)
|
||||
deleted += int64(compressedDeleted)
|
||||
directIndex := sort.Search(len(series.samples), func(i int) bool { return series.samples[i].timestamp >= beforeMilli })
|
||||
if directIndex > 0 {
|
||||
deleted += int64(directIndex)
|
||||
series.samples = append([]rawSample(nil), series.samples[directIndex:]...)
|
||||
copy(series.samples, series.samples[directIndex:])
|
||||
series.samples = series.samples[:len(series.samples)-directIndex]
|
||||
}
|
||||
if series.compressed.count == 0 && len(series.samples) == 0 {
|
||||
delete(s.raw, key)
|
||||
continue
|
||||
}
|
||||
if compressedIndex > 0 || directIndex > 0 {
|
||||
if compressedDeleted > 0 || directIndex > 0 {
|
||||
compactRawLabels(series)
|
||||
}
|
||||
}
|
||||
return deleted
|
||||
}
|
||||
|
||||
func pruneCompressedRawSamples(samples *compressedRawSamples, beforeMilli int64) int {
|
||||
if samples.count == 0 || samples.firstStamp() >= beforeMilli {
|
||||
return 0
|
||||
}
|
||||
if samples.lastStamp < beforeMilli {
|
||||
deleted := samples.count
|
||||
*samples = compressedRawSamples{}
|
||||
return deleted
|
||||
}
|
||||
decoder := newRawSampleDecoder(*samples)
|
||||
kept := newCompressedRawSamples(len(samples.data))
|
||||
deleted := 0
|
||||
for sample, more := decoder.next(); more; sample, more = decoder.next() {
|
||||
if sample.timestamp < beforeMilli {
|
||||
deleted++
|
||||
continue
|
||||
}
|
||||
kept.appendSample(sample)
|
||||
}
|
||||
*samples = kept
|
||||
return deleted
|
||||
}
|
||||
|
||||
func (s *Store) compressRawBeforeLocked(metricName string, beforeMilli int64) {
|
||||
for key, series := range s.raw {
|
||||
if metricName != "" && key.metricName != metricName {
|
||||
@@ -391,70 +565,38 @@ func (s *Store) compressRawBeforeLocked(metricName string, beforeMilli int64) {
|
||||
continue
|
||||
}
|
||||
series.compressed.append(series.samples[:index])
|
||||
series.samples = append([]rawSample(nil), series.samples[index:]...)
|
||||
}
|
||||
}
|
||||
|
||||
func (s *rawSeries) decodeCompressed() []rawSample {
|
||||
if s.compressed.count == 0 {
|
||||
return nil
|
||||
}
|
||||
result := make([]rawSample, 0, s.compressed.count)
|
||||
for offset, previous := 0, int64(0); offset < len(s.compressed.data); {
|
||||
var timestamp int64
|
||||
if len(result) == 0 {
|
||||
first, n := binary.Varint(s.compressed.data[offset:])
|
||||
if n <= 0 {
|
||||
return result
|
||||
}
|
||||
offset += n
|
||||
timestamp = first
|
||||
} else {
|
||||
delta, dn := binary.Uvarint(s.compressed.data[offset:])
|
||||
if dn <= 0 {
|
||||
return result
|
||||
}
|
||||
offset += dn
|
||||
timestamp = previous + int64(delta)
|
||||
copy(series.samples, series.samples[index:])
|
||||
series.samples = series.samples[:len(series.samples)-index]
|
||||
if len(series.samples) == 0 {
|
||||
series.samples = nil
|
||||
}
|
||||
if offset+8 > len(s.compressed.data) {
|
||||
return result
|
||||
}
|
||||
value := math.Float64frombits(binary.LittleEndian.Uint64(s.compressed.data[offset:]))
|
||||
offset += 8
|
||||
labelID, ln := binary.Uvarint(s.compressed.data[offset:])
|
||||
if ln <= 0 {
|
||||
return result
|
||||
}
|
||||
offset += ln
|
||||
result = append(result, rawSample{timestamp: timestamp, value: value, labelID: uint32(labelID)})
|
||||
previous = timestamp
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
func (s *rawSeries) allSamples() []rawSample {
|
||||
compressed := s.decodeCompressed()
|
||||
result := make([]rawSample, 0, len(compressed)+len(s.samples))
|
||||
result = append(result, compressed...)
|
||||
result = append(result, s.samples...)
|
||||
return result
|
||||
}
|
||||
|
||||
func (s *rawSeries) replaceCompressed(sample rawSample) (rawSample, bool) {
|
||||
samples := s.decodeCompressed()
|
||||
index := sort.Search(len(samples), func(i int) bool { return samples[i].timestamp >= sample.timestamp })
|
||||
var old rawSample
|
||||
replaced := index < len(samples) && samples[index].timestamp == sample.timestamp
|
||||
if replaced {
|
||||
old = samples[index]
|
||||
samples[index] = sample
|
||||
} else {
|
||||
samples = append(samples, rawSample{})
|
||||
copy(samples[index+1:], samples[index:])
|
||||
samples[index] = sample
|
||||
replaced := false
|
||||
inserted := false
|
||||
decoder := newRawSampleDecoder(s.compressed)
|
||||
encoded := newCompressedRawSamples(len(s.compressed.data) + binary.MaxVarintLen64 + 9)
|
||||
for current, more := decoder.next(); more; current, more = decoder.next() {
|
||||
if !inserted && sample.timestamp < current.timestamp {
|
||||
encoded.appendSample(sample)
|
||||
inserted = true
|
||||
}
|
||||
if !inserted && sample.timestamp == current.timestamp {
|
||||
old = current
|
||||
replaced = true
|
||||
encoded.appendSample(sample)
|
||||
inserted = true
|
||||
continue
|
||||
}
|
||||
encoded.appendSample(current)
|
||||
}
|
||||
s.compressed.set(samples)
|
||||
if !inserted {
|
||||
encoded.appendSample(sample)
|
||||
}
|
||||
s.compressed = encoded
|
||||
return old, replaced
|
||||
}
|
||||
|
||||
@@ -463,51 +605,39 @@ func (c *compressedRawSamples) append(samples []rawSample) {
|
||||
return
|
||||
}
|
||||
if c.count > 0 && samples[0].timestamp <= c.lastStamp {
|
||||
merged := make([]rawSample, 0, c.count+len(samples))
|
||||
merged = append(merged, c.decode()...)
|
||||
for _, sample := range samples {
|
||||
index := sort.Search(len(merged), func(i int) bool { return merged[i].timestamp >= sample.timestamp })
|
||||
if index < len(merged) && merged[index].timestamp == sample.timestamp {
|
||||
merged[index] = sample
|
||||
decoder := newRawSampleDecoder(*c)
|
||||
merged := newCompressedRawSamples(len(c.data) + len(samples)*10)
|
||||
current, more := decoder.next()
|
||||
for sampleIndex := 0; more || sampleIndex < len(samples); {
|
||||
if !more {
|
||||
merged.appendSample(samples[sampleIndex])
|
||||
sampleIndex++
|
||||
continue
|
||||
}
|
||||
merged = append(merged, rawSample{})
|
||||
copy(merged[index+1:], merged[index:])
|
||||
merged[index] = sample
|
||||
if sampleIndex == len(samples) {
|
||||
merged.appendSample(current)
|
||||
current, more = decoder.next()
|
||||
continue
|
||||
}
|
||||
next := samples[sampleIndex]
|
||||
switch {
|
||||
case next.timestamp < current.timestamp:
|
||||
merged.appendSample(next)
|
||||
sampleIndex++
|
||||
case next.timestamp == current.timestamp:
|
||||
merged.appendSample(next)
|
||||
sampleIndex++
|
||||
current, more = decoder.next()
|
||||
default:
|
||||
merged.appendSample(current)
|
||||
current, more = decoder.next()
|
||||
}
|
||||
}
|
||||
c.set(merged)
|
||||
*c = merged
|
||||
return
|
||||
}
|
||||
for _, sample := range samples {
|
||||
if c.count == 0 {
|
||||
c.data = appendVarint(c.data, sample.timestamp)
|
||||
} else {
|
||||
c.data = appendUvarint(c.data, uint64(sample.timestamp-c.lastStamp))
|
||||
}
|
||||
c.data = appendRawSample(c.data, sample)
|
||||
c.count++
|
||||
c.lastStamp = sample.timestamp
|
||||
}
|
||||
}
|
||||
|
||||
func (c *compressedRawSamples) decode() []rawSample {
|
||||
series := rawSeries{compressed: *c}
|
||||
return series.decodeCompressed()
|
||||
}
|
||||
|
||||
func (c *compressedRawSamples) set(samples []rawSample) {
|
||||
c.data = nil
|
||||
c.count = 0
|
||||
c.lastStamp = 0
|
||||
for _, sample := range samples {
|
||||
if c.count == 0 {
|
||||
c.data = appendVarint(c.data, sample.timestamp)
|
||||
} else {
|
||||
c.data = appendUvarint(c.data, uint64(sample.timestamp-c.lastStamp))
|
||||
}
|
||||
c.data = appendRawSample(c.data, sample)
|
||||
c.count++
|
||||
c.lastStamp = sample.timestamp
|
||||
c.appendSample(sample)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -534,9 +664,12 @@ func compactRawLabels(series *rawSeries) {
|
||||
if len(series.labelHashes) <= 1 {
|
||||
return
|
||||
}
|
||||
all := series.allSamples()
|
||||
used := make([]bool, len(series.labelHashes))
|
||||
for _, sample := range all {
|
||||
decoder := newRawSampleDecoder(series.compressed)
|
||||
for sample, more := decoder.next(); more; sample, more = decoder.next() {
|
||||
used[sample.labelID] = true
|
||||
}
|
||||
for _, sample := range series.samples {
|
||||
used[sample.labelID] = true
|
||||
}
|
||||
usedCount := 0
|
||||
@@ -563,11 +696,13 @@ func compactRawLabels(series *rawSeries) {
|
||||
labelsJSON = append(labelsJSON, series.labelsJSON[oldID])
|
||||
labelIDs[hash] = newID
|
||||
}
|
||||
compressed := series.decodeCompressed()
|
||||
for i := range compressed {
|
||||
compressed[i].labelID = remap[compressed[i].labelID]
|
||||
decoder = newRawSampleDecoder(series.compressed)
|
||||
compressed := newCompressedRawSamples(len(series.compressed.data))
|
||||
for sample, more := decoder.next(); more; sample, more = decoder.next() {
|
||||
sample.labelID = remap[sample.labelID]
|
||||
compressed.appendSample(sample)
|
||||
}
|
||||
series.compressed.set(compressed)
|
||||
series.compressed = compressed
|
||||
for i := range series.samples {
|
||||
series.samples[i].labelID = remap[series.samples[i].labelID]
|
||||
}
|
||||
|
||||
@@ -124,14 +124,121 @@ func TestCompressedRawSamplesRoundTripExactly(t *testing.T) {
|
||||
}
|
||||
var compressed compressedRawSamples
|
||||
compressed.append(samples)
|
||||
decoded := compressed.decode()
|
||||
if len(decoded) != len(samples) {
|
||||
t.Fatalf("decoded sample count = %d, want %d", len(decoded), len(samples))
|
||||
}
|
||||
decoder := newRawSampleDecoder(compressed)
|
||||
for i := range samples {
|
||||
if decoded[i].timestamp != samples[i].timestamp || decoded[i].labelID != samples[i].labelID || math.Float64bits(decoded[i].value) != math.Float64bits(samples[i].value) {
|
||||
t.Fatalf("decoded sample %d = %#v, want %#v", i, decoded[i], samples[i])
|
||||
decoded, more := decoder.next()
|
||||
if !more {
|
||||
t.Fatalf("decoded sample count = %d, want %d", i, len(samples))
|
||||
}
|
||||
if decoded.timestamp != samples[i].timestamp || decoded.labelID != samples[i].labelID || math.Float64bits(decoded.value) != math.Float64bits(samples[i].value) {
|
||||
t.Fatalf("decoded sample %d = %#v, want %#v", i, decoded, samples[i])
|
||||
}
|
||||
}
|
||||
if _, more := decoder.next(); more {
|
||||
t.Fatal("decoder returned extra sample")
|
||||
}
|
||||
}
|
||||
|
||||
func TestCompressedRawSamplesMergeInsertionAndReplacement(t *testing.T) {
|
||||
base := time.Now().UTC().Add(-5 * time.Minute).UnixMilli()
|
||||
var compressed compressedRawSamples
|
||||
compressed.append([]rawSample{
|
||||
{timestamp: base, value: 1, labelID: 0},
|
||||
{timestamp: base + 2000, value: 3, labelID: 2},
|
||||
})
|
||||
compressed.append([]rawSample{
|
||||
{timestamp: base + 1000, value: math.Copysign(0, -1), labelID: 1},
|
||||
{timestamp: base + 2000, value: 33, labelID: 3},
|
||||
})
|
||||
want := []rawSample{
|
||||
{timestamp: base, value: 1, labelID: 0},
|
||||
{timestamp: base + 1000, value: math.Copysign(0, -1), labelID: 1},
|
||||
{timestamp: base + 2000, value: 33, labelID: 3},
|
||||
}
|
||||
decoder := newRawSampleDecoder(compressed)
|
||||
for i := range want {
|
||||
got, more := decoder.next()
|
||||
if !more || got.timestamp != want[i].timestamp || got.labelID != want[i].labelID || math.Float64bits(got.value) != math.Float64bits(want[i].value) {
|
||||
t.Fatalf("sample %d = %#v, want %#v", i, got, want[i])
|
||||
}
|
||||
}
|
||||
if _, more := decoder.next(); more {
|
||||
t.Fatal("decoder returned extra merged sample")
|
||||
}
|
||||
}
|
||||
|
||||
func TestRawQueryCompressedDirectBoundaryClosedRange(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
s := newMemStore(t)
|
||||
if err := s.CreateMetric(ctx, Definition{Name: "raw-boundary", RetentionDays: 1}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
now := time.Now().UTC().Truncate(time.Millisecond)
|
||||
compressedAt := now.Add(-time.Minute)
|
||||
directAt := compressedAt.Add(time.Millisecond)
|
||||
series := &rawSeries{
|
||||
tagsJSON: "{}", labelIDs: map[string]uint32{emptyLabelsHash: 0},
|
||||
labelHashes: []string{emptyLabelsHash}, labelsJSON: []string{"{}"},
|
||||
samples: []rawSample{{timestamp: directAt.UnixMilli(), value: 2}},
|
||||
}
|
||||
series.compressed.append([]rawSample{{timestamp: compressedAt.UnixMilli(), value: 1}})
|
||||
s.raw[rawSeriesKey{metricName: "raw-boundary", entityID: "n1", tagsHash: emptyLabelsHash}] = series
|
||||
|
||||
ascending, err := s.Query(ctx, Query{MetricName: "raw-boundary", Start: compressedAt, End: directAt, Order: OrderAsc})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(ascending) != 2 || ascending[0].Value != 1 || ascending[1].Value != 2 {
|
||||
t.Fatalf("closed ascending range = %#v", ascending)
|
||||
}
|
||||
descending, err := s.Query(ctx, Query{MetricName: "raw-boundary", Start: compressedAt, End: directAt, Order: OrderDesc, Offset: 1, Limit: 1})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(descending) != 1 || descending[0].Value != 1 {
|
||||
t.Fatalf("paged descending range = %#v", descending)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRawPruneStreamsAndReclaimsLabels(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
s := newMemStore(t)
|
||||
if err := s.CreateMetric(ctx, Definition{Name: "raw-prune", RetentionDays: 1}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
now := time.Now().UTC().Truncate(time.Millisecond)
|
||||
cutoff := now.Add(-3 * time.Minute)
|
||||
series := &rawSeries{
|
||||
tagsJSON: "{}",
|
||||
labelIDs: map[string]uint32{"old": 0, "middle": 1, "keep": 2},
|
||||
labelHashes: []string{"old", "middle", "keep"},
|
||||
labelsJSON: []string{`{"label":"old"}`, `{"label":"middle"}`, `{"label":"keep"}`},
|
||||
samples: []rawSample{{timestamp: now.Add(-30 * time.Second).UnixMilli(), value: 4, labelID: 2}},
|
||||
}
|
||||
series.compressed.append([]rawSample{
|
||||
{timestamp: now.Add(-5 * time.Minute).UnixMilli(), value: 1, labelID: 0},
|
||||
{timestamp: now.Add(-4 * time.Minute).UnixMilli(), value: 2, labelID: 1},
|
||||
{timestamp: cutoff.UnixMilli(), value: 3, labelID: 2},
|
||||
})
|
||||
s.raw[rawSeriesKey{metricName: "raw-prune", entityID: "n1", tagsHash: emptyLabelsHash}] = series
|
||||
|
||||
if deleted := s.deleteRawBefore("raw-prune", cutoff.UnixMilli()); deleted != 2 {
|
||||
t.Fatalf("deleted %d samples, want 2", deleted)
|
||||
}
|
||||
if len(series.labelsJSON) != 1 || series.labelsJSON[0] != `{"label":"keep"}` || series.compressed.count != 1 || series.samples[0].labelID != 0 {
|
||||
t.Fatalf("compacted series = %#v", series)
|
||||
}
|
||||
decoder := newRawSampleDecoder(series.compressed)
|
||||
kept, more := decoder.next()
|
||||
if !more || kept.timestamp != cutoff.UnixMilli() || kept.labelID != 0 || kept.value != 3 {
|
||||
t.Fatalf("kept compressed sample = %#v", kept)
|
||||
}
|
||||
points, err := s.Query(ctx, Query{MetricName: "raw-prune", Start: cutoff, End: now})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(points) != 2 || points[0].Labels["label"] != "keep" || points[1].Labels["label"] != "keep" {
|
||||
t.Fatalf("pruned query = %#v", points)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -832,7 +832,7 @@ func TestValidateNormalizedRestructureRejectsMissingDigest(t *testing.T) {
|
||||
if err := s.CreateMetric(ctx, Definition{Name: "invalid", Type: TypeGauge, RetentionDays: 1}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
at := time.Now().UTC().Add(-2 * time.Minute)
|
||||
at := time.Now().UTC().Truncate(time.Minute).Add(-2 * time.Minute)
|
||||
if err := s.WriteBatch(ctx, []Point{
|
||||
{MetricName: "invalid", EntityID: "n1", Timestamp: at, Value: 1},
|
||||
{MetricName: "invalid", EntityID: "n1", Timestamp: at.Add(time.Second), Value: 2},
|
||||
@@ -857,7 +857,7 @@ func TestValidateNormalizedRestructureRejectsWrongDigestCompression(t *testing.T
|
||||
if err := s.CreateMetric(ctx, Definition{Name: "invalid-compression", Type: TypeGauge, RetentionDays: 1}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
at := time.Now().UTC().Add(-2 * time.Minute)
|
||||
at := time.Now().UTC().Truncate(time.Minute).Add(-2 * time.Minute)
|
||||
if err := s.WriteBatch(ctx, []Point{
|
||||
{MetricName: "invalid-compression", EntityID: "n1", Timestamp: at, Value: 1},
|
||||
{MetricName: "invalid-compression", EntityID: "n1", Timestamp: at.Add(time.Second), Value: 2},
|
||||
@@ -929,7 +929,6 @@ func TestRestructureDump(t *testing.T) {
|
||||
if err := s.ReclaimSpace(ctx); err != nil {
|
||||
t.Fatalf("reclaim rebuilt store: %v", err)
|
||||
}
|
||||
assertRestructuredDumpQueryContinuity(t, ctx, s)
|
||||
after, err := s.StorageSize(ctx)
|
||||
if err != nil {
|
||||
t.Fatalf("measure rebuilt storage: %v", err)
|
||||
@@ -944,85 +943,3 @@ func TestRestructureDump(t *testing.T) {
|
||||
}
|
||||
t.Logf("restructured rows=%d metrics=%d before=%d after=%d saved=%d percent=%.2f", result.RowsCopied, result.Metrics, before, after, saved, percent)
|
||||
}
|
||||
|
||||
func assertRestructuredDumpQueryContinuity(t *testing.T, ctx context.Context, s *Store) {
|
||||
t.Helper()
|
||||
var metricName, entityID string
|
||||
var latestMilli int64
|
||||
err := s.db.QueryRowContext(ctx, fmt.Sprintf(`SELECT series.metric_name, series.entity_id,
|
||||
(SELECT MAX(recent.last_ts_milli) FROM %s recent
|
||||
JOIN %s recent_series ON recent_series.id = recent.series_id
|
||||
WHERE recent_series.metric_name = series.metric_name AND recent_series.entity_id = series.entity_id)
|
||||
FROM %s daily
|
||||
JOIN %s series ON series.id = daily.series_id
|
||||
JOIN %s daily_resolution ON daily_resolution.id = daily.resolution_id
|
||||
WHERE daily_resolution.resolution_milli = ?
|
||||
GROUP BY series.metric_name, series.entity_id
|
||||
ORDER BY COUNT(DISTINCT daily.bucket_milli) DESC, series.metric_name, series.entity_id LIMIT 1`,
|
||||
s.tables.rollups, s.tables.series,
|
||||
s.tables.rollups, s.tables.series, s.tables.resolutions),
|
||||
(24*time.Hour).Milliseconds()).Scan(&metricName, &entityID, &latestMilli)
|
||||
if err != nil {
|
||||
t.Fatalf("select migrated series: %v", err)
|
||||
}
|
||||
now := fromMillis(latestMilli).Add(time.Millisecond)
|
||||
for _, duration := range []time.Duration{time.Hour, 6 * time.Hour, 7 * 24 * time.Hour, 30 * 24 * time.Hour, 60 * 24 * time.Hour} {
|
||||
start := now.Add(-duration)
|
||||
interval := s.CompatibleSeriesInterval(start, now, time.Minute)
|
||||
resolution, err := s.coveringSeriesResolution(ctx, Query{MetricName: metricName, EntityID: entityID, Start: start, End: now}, interval)
|
||||
if err != nil {
|
||||
t.Fatalf("select %s backing tier: %v", duration, err)
|
||||
}
|
||||
series, err := s.Series(ctx, AggregateQuery{
|
||||
Query: Query{MetricName: metricName, EntityID: entityID, Start: start, End: now},
|
||||
Aggregation: AggAvg,
|
||||
Interval: interval,
|
||||
}, now)
|
||||
if err != nil {
|
||||
t.Fatalf("query %s: %v", duration, err)
|
||||
}
|
||||
if len(series) == 0 || series[len(series)-1].Count == 0 {
|
||||
t.Fatalf("query %s has no final data bucket: %#v", duration, series)
|
||||
}
|
||||
expectedStep := max(interval, resolution)
|
||||
maxGap := time.Duration(0)
|
||||
for i := 1; i < len(series); i++ {
|
||||
gap := series[i].Bucket.Sub(series[i-1].Bucket)
|
||||
if gap > maxGap {
|
||||
maxGap = gap
|
||||
}
|
||||
}
|
||||
if maxGap > 2*expectedStep {
|
||||
t.Fatalf("query %s has a %s discontinuity larger than two %s buckets", duration, maxGap, expectedStep)
|
||||
}
|
||||
tailGap := now.Sub(series[len(series)-1].Bucket)
|
||||
if tailGap < 0 || tailGap > 2*expectedStep {
|
||||
t.Fatalf("query %s final data is %s from the query end, expected at most %s", duration, tailGap, 2*expectedStep)
|
||||
}
|
||||
t.Logf("range=%s metric=%s entity=%s requested=%s backing=%s points=%d first=%s last=%s max_gap=%s tail_gap=%s",
|
||||
duration, metricName, entityID, interval, resolution, len(series), series[0].Bucket, series[len(series)-1].Bucket, maxGap, tailGap)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRestructuredDumpQueryContinuity(t *testing.T) {
|
||||
path := os.Getenv("KOMARI_METRIC_RESTRUCTURE_RESULT")
|
||||
if path == "" {
|
||||
t.Skip("set KOMARI_METRIC_RESTRUCTURE_RESULT to validate a rebuilt dump")
|
||||
}
|
||||
ctx := context.Background()
|
||||
s, err := Open(ctx, SQLite(path, WithAutoMigrate(false), WithMaxOpenConns(1), WithRollupPolicy(RollupPolicy{
|
||||
RawRetention: 15 * time.Minute,
|
||||
Tiers: []RollupTier{
|
||||
{Interval: time.Minute, Retention: 600 * time.Minute},
|
||||
{Interval: 5 * time.Minute, Retention: 600 * 5 * time.Minute},
|
||||
{Interval: time.Hour, Retention: 600 * time.Hour},
|
||||
{Interval: 24 * time.Hour, Retention: 100 * 365 * 24 * time.Hour},
|
||||
},
|
||||
Compression: 30,
|
||||
})))
|
||||
if err != nil {
|
||||
t.Fatalf("open rebuilt dump: %v", err)
|
||||
}
|
||||
defer s.Close()
|
||||
assertRestructuredDumpQueryContinuity(t, ctx, s)
|
||||
}
|
||||
@@ -348,12 +348,10 @@ func TestSeriesHybridIncludesPartialCoarseBucketAtRawCutoff(t *testing.T) {
|
||||
now := time.Date(2026, 6, 18, 12, 50, 0, 0, time.UTC)
|
||||
old := now.Add(-40 * time.Minute)
|
||||
recent := now.Add(-5 * time.Minute)
|
||||
if err := s.WriteBatch(ctx, []Point{
|
||||
{MetricName: "coarse-cutoff", EntityID: "n1", Timestamp: old, Value: 10},
|
||||
{MetricName: "coarse-cutoff", EntityID: "n1", Timestamp: recent, Value: 100},
|
||||
}); err != nil {
|
||||
t.Fatalf("write: %v", err)
|
||||
}
|
||||
writeRollupPointsAt(t, ctx, s, now,
|
||||
Point{MetricName: "coarse-cutoff", EntityID: "n1", Timestamp: old, Value: 10},
|
||||
Point{MetricName: "coarse-cutoff", EntityID: "n1", Timestamp: recent, Value: 100},
|
||||
)
|
||||
if _, err := s.Compact(ctx, now); err != nil {
|
||||
t.Fatalf("compact: %v", err)
|
||||
}
|
||||
@@ -367,7 +365,7 @@ func TestSeriesHybridIncludesPartialCoarseBucketAtRawCutoff(t *testing.T) {
|
||||
t.Fatalf("series: %v", err)
|
||||
}
|
||||
if len(got) != 1 || got[0].Count != 2 || math.Abs(got[0].Value-55) > 1e-9 {
|
||||
t.Fatalf("expected coarse rollup and recent raw point, got %#v", got)
|
||||
t.Fatalf("expected persisted and in-memory hourly data, got %#v", got)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -411,7 +409,9 @@ func TestSeriesAndAggregateRollupBridgeSealedCoarseTail(t *testing.T) {
|
||||
read func() ([]AggregatePoint, error)
|
||||
}{
|
||||
{name: "series", read: func() ([]AggregatePoint, error) { return s.Series(ctx, query, base.Add(16*time.Minute)) }},
|
||||
{name: "aggregate", read: func() ([]AggregatePoint, error) { return s.aggregateRollupAt(ctx, query, 5*time.Minute, base.Add(16*time.Minute)) }},
|
||||
{name: "aggregate", read: func() ([]AggregatePoint, error) {
|
||||
return s.aggregateRollupAt(ctx, query, 5*time.Minute, base.Add(16*time.Minute))
|
||||
}},
|
||||
} {
|
||||
points, err := result.read()
|
||||
if err != nil {
|
||||
|
||||
+193
-353
@@ -7,292 +7,118 @@ import (
|
||||
"time"
|
||||
)
|
||||
|
||||
// AggregateRollup reduces one materialized tier into the requested output
|
||||
// interval. The hot minute is folded in separately so recent charts never wait
|
||||
// for another report before their final real bucket becomes visible.
|
||||
// AggregateRollup reduces one explicitly selected materialized tier into the
|
||||
// requested output interval.
|
||||
func (s *Store) AggregateRollup(ctx context.Context, query AggregateQuery, resolution time.Duration) ([]AggregatePoint, error) {
|
||||
return s.aggregateRollupAt(ctx, query, resolution, time.Now().UTC())
|
||||
}
|
||||
|
||||
// aggregateRollupAt reads a sealed rollup prefix at the requested backing
|
||||
// resolution, then recursively falls back through finer tiers for the
|
||||
// ten-minute late-arrival tail. Series passes its supplied now value here so
|
||||
// callers receive a stable view while AggregateRollup keeps its public API.
|
||||
func (s *Store) aggregateRollupAt(ctx context.Context, query AggregateQuery, resolution time.Duration, now time.Time) ([]AggregatePoint, error) {
|
||||
if err := s.ensureOpen(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := query.Validate(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if resolution <= 0 {
|
||||
return nil, fmt.Errorf("%w: rollup resolution must be positive", ErrInvalidArgument)
|
||||
}
|
||||
s.rollupViewMu.RLock()
|
||||
defer s.rollupViewMu.RUnlock()
|
||||
q := query.Query.normalized()
|
||||
needDigest := isPercentile(query.Aggregation)
|
||||
policy, err := s.metricRollupPolicy(ctx, q.MetricName)
|
||||
entityIDs := []string(nil)
|
||||
if query.EntityID != "" {
|
||||
entityIDs = []string{query.EntityID}
|
||||
}
|
||||
batch := BatchSeriesQuery{
|
||||
Specs: []BatchSeriesSpec{{
|
||||
MetricName: query.MetricName,
|
||||
Aggregations: []Aggregation{query.Aggregation},
|
||||
Interval: query.Interval,
|
||||
PreserveSeries: query.PreserveSeries,
|
||||
}},
|
||||
EntityIDs: entityIDs,
|
||||
Start: query.Start,
|
||||
End: query.End,
|
||||
Tags: query.Tags,
|
||||
Order: query.Order,
|
||||
}
|
||||
if err := batch.Validate(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
result, err := s.seriesBatchAt(ctx, batch.normalized(), now.UTC(), map[string]time.Duration{query.MetricName: resolution})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
rows, err := s.hybridRollupRows(ctx, q, policy, resolution, now.UTC(), needDigest)
|
||||
return pageBuckets(result.Values[query.MetricName][query.Aggregation], query.BucketLimit, query.BucketOffset), nil
|
||||
}
|
||||
|
||||
// Series selects one deterministic backing tier and delegates to SeriesBatch.
|
||||
func (s *Store) Series(ctx context.Context, query AggregateQuery, now time.Time) ([]AggregatePoint, error) {
|
||||
if err := s.ensureOpen(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := query.Validate(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
entityIDs := []string(nil)
|
||||
if query.EntityID != "" {
|
||||
entityIDs = []string{query.EntityID}
|
||||
}
|
||||
result, err := s.SeriesBatch(ctx, BatchSeriesQuery{
|
||||
Specs: []BatchSeriesSpec{{
|
||||
MetricName: query.MetricName,
|
||||
Aggregations: []Aggregation{query.Aggregation},
|
||||
Interval: query.Interval,
|
||||
PreserveSeries: query.PreserveSeries,
|
||||
}},
|
||||
EntityIDs: entityIDs,
|
||||
Start: query.Start,
|
||||
End: query.End,
|
||||
Tags: query.Tags,
|
||||
Order: query.Order,
|
||||
}, now)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if query.Aggregation == AggRate {
|
||||
points := representativePoints(q.MetricName, rows)
|
||||
return pageBuckets(mustAggregatePoints(points, query), query.BucketLimit, query.BucketOffset), nil
|
||||
return pageBuckets(result.Values[query.MetricName][query.Aggregation], query.BucketLimit, query.BucketOffset), nil
|
||||
}
|
||||
|
||||
// SeriesAggregates derives multiple aggregations from the same streamed scan.
|
||||
func (s *Store) SeriesAggregates(ctx context.Context, query Query, aggregations []Aggregation, interval time.Duration, preserveSeries bool, now time.Time) (map[Aggregation][]AggregatePoint, error) {
|
||||
entityIDs := []string(nil)
|
||||
if query.EntityID != "" {
|
||||
entityIDs = []string{query.EntityID}
|
||||
}
|
||||
groups := foldRollupRows(nil, rows, query.Interval, s.cfg.RollupPolicy.compression(), query.PreserveSeries, false, needDigest)
|
||||
result, err := rollupGroupsToPoints(groups, query)
|
||||
result, err := s.SeriesBatch(ctx, BatchSeriesQuery{
|
||||
Specs: []BatchSeriesSpec{{
|
||||
MetricName: query.MetricName,
|
||||
Aggregations: aggregations,
|
||||
Interval: interval,
|
||||
PreserveSeries: preserveSeries,
|
||||
}},
|
||||
EntityIDs: entityIDs,
|
||||
Start: query.Start,
|
||||
End: query.End,
|
||||
Tags: query.Tags,
|
||||
Order: query.Order,
|
||||
}, now)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return pageBuckets(result, query.BucketLimit, query.BucketOffset), nil
|
||||
return result.Values[query.MetricName], nil
|
||||
}
|
||||
|
||||
// hybridRollupRows reads each part of a range once. A parent bucket becomes
|
||||
// eligible only after its end is outside coarseRollupGrace; the remainder is
|
||||
// represented by the immediate finer tier until minute buckets, where hot
|
||||
// in-memory observations complete the view.
|
||||
func (s *Store) hybridRollupRows(ctx context.Context, query Query, policy RollupPolicy, resolution time.Duration, now time.Time, needDigest bool) ([]storedRollup, error) {
|
||||
if len(policy.Tiers) == 0 {
|
||||
return nil, nil
|
||||
func seriesResolutionForPolicy(start time.Time, interval time.Duration, now time.Time, policy RollupPolicy) time.Duration {
|
||||
preferred := policy.Tiers[0].Interval
|
||||
if tier := bestRollupTier(policy, interval, start.UTC(), now.UTC()); tier != nil {
|
||||
return tier.Interval
|
||||
}
|
||||
index := -1
|
||||
for i, tier := range policy.Tiers {
|
||||
if tier.Interval <= resolution && resolution%tier.Interval == 0 {
|
||||
index = i
|
||||
for i := len(policy.Tiers) - 1; i >= 0; i-- {
|
||||
tier := policy.Tiers[i]
|
||||
if interval >= tier.Interval && interval%tier.Interval == 0 {
|
||||
preferred = tier.Interval
|
||||
break
|
||||
}
|
||||
}
|
||||
if index < 0 {
|
||||
return s.scanRollupRowsBetween(ctx, query.MetricName, query.EntityID, query.Tags, resolution.Milliseconds(), bucketStartMillis(query.Start.UnixMilli(), resolution.Milliseconds()), query.End.UnixMilli(), needDigest)
|
||||
}
|
||||
return s.hybridRollupRowsAtTier(ctx, query, policy, index, query.Start.UTC().UnixMilli(), query.End.UTC().UnixMilli(), now, needDigest)
|
||||
}
|
||||
|
||||
func (s *Store) hybridRollupRowsAtTier(ctx context.Context, query Query, policy RollupPolicy, index int, startMilli, endMilli int64, now time.Time, needDigest bool) ([]storedRollup, error) {
|
||||
if startMilli > endMilli {
|
||||
return nil, nil
|
||||
}
|
||||
tier := policy.Tiers[index]
|
||||
intervalMilli := tier.Interval.Milliseconds()
|
||||
// The greatest bucket start whose end is at least coarseRollupGrace behind
|
||||
// now. Buckets after it are still mutable in the in-memory parent map.
|
||||
sealedThrough := bucketStartMillis(now.Add(-coarseRollupGrace-tier.Interval).UnixMilli(), intervalMilli)
|
||||
rows := make([]storedRollup, 0)
|
||||
if startMilli <= sealedThrough {
|
||||
upper := sealedThrough
|
||||
if upper > endMilli {
|
||||
upper = endMilli
|
||||
}
|
||||
sealed, err := s.scanRollupRowsBetween(ctx, query.MetricName, query.EntityID, query.Tags, intervalMilli, bucketStartMillis(startMilli, intervalMilli), upper, needDigest)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
rows = append(rows, sealed...)
|
||||
}
|
||||
|
||||
tailStart := startMilli
|
||||
if tailStart <= sealedThrough {
|
||||
tailStart = sealedThrough + intervalMilli
|
||||
}
|
||||
if tailStart > endMilli {
|
||||
return rows, nil
|
||||
}
|
||||
if index == 0 {
|
||||
minute, err := s.scanRollupRowsBetween(ctx, query.MetricName, query.EntityID, query.Tags, intervalMilli, bucketStartMillis(tailStart, intervalMilli), endMilli, needDigest)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
hot, err := s.hotRollupRows(query.MetricName, query.EntityID, query.Tags, fromMillis(tailStart), fromMillis(endMilli), needDigest)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return append(rows, append(minute, hot...)...), nil
|
||||
}
|
||||
tail, err := s.hybridRollupRowsAtTier(ctx, query, policy, index-1, tailStart, endMilli, now, needDigest)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return append(rows, tail...), nil
|
||||
}
|
||||
|
||||
func mustAggregatePoints(points []Point, query AggregateQuery) []AggregatePoint {
|
||||
result, err := AggregatePoints(points, query)
|
||||
if err != nil {
|
||||
return []AggregatePoint{}
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
func representativePoints(metricName string, rows []storedRollup) []Point {
|
||||
points := make([]Point, 0, len(rows))
|
||||
for _, row := range rows {
|
||||
tags, err := rollupTagsFromJSON(row.bucketData.tagsJSON)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
labels, err := rollupTagsFromJSON(row.bucketData.labelsJSON)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
points = append(points, Point{MetricName: metricName, EntityID: row.entityID, Timestamp: fromMillis(row.bucketData.lastTS), Value: row.bucketData.lastVal, Tags: tags, Labels: labels})
|
||||
}
|
||||
return points
|
||||
}
|
||||
|
||||
func foldRollupRows(groups map[rollupKey]*rollupBucket, rows []storedRollup, interval time.Duration, compression float64, preserveSeries, preserveLabels, needDigest bool) map[rollupKey]*rollupBucket {
|
||||
if groups == nil {
|
||||
groups = make(map[rollupKey]*rollupBucket)
|
||||
}
|
||||
for _, row := range rows {
|
||||
key := rollupKey{bucket: bucketStartMillis(row.bucket, interval.Milliseconds())}
|
||||
if preserveSeries {
|
||||
key.entityID, key.tagsHash = row.entityID, row.bucketData.tagsHash
|
||||
if preserveLabels {
|
||||
key.labelsHash = row.bucketData.labelsHash
|
||||
}
|
||||
}
|
||||
bucket := groups[key]
|
||||
if bucket == nil {
|
||||
bucket = newRollupBucketWithDigest(compression, needDigest)
|
||||
bucket.tagsHash, bucket.tagsJSON = row.bucketData.tagsHash, row.bucketData.tagsJSON
|
||||
bucket.labelsHash, bucket.labelsJSON = row.bucketData.labelsHash, row.bucketData.labelsJSON
|
||||
groups[key] = bucket
|
||||
}
|
||||
bucket.mergeStored(row.bucketData)
|
||||
}
|
||||
return groups
|
||||
}
|
||||
|
||||
func rollupGroupsToPoints(groups map[rollupKey]*rollupBucket, query AggregateQuery) ([]AggregatePoint, error) {
|
||||
keys := make([]rollupKey, 0, len(groups))
|
||||
for key := range groups {
|
||||
keys = append(keys, key)
|
||||
}
|
||||
sortRollupKeys(keys)
|
||||
result := make([]AggregatePoint, 0, len(keys))
|
||||
for _, key := range keys {
|
||||
bucket := groups[key]
|
||||
value, ok := bucket.value(query.Aggregation)
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("%w: aggregation %q requires raw samples", ErrInvalidArgument, query.Aggregation)
|
||||
}
|
||||
tags, err := rollupTagsFromJSON(bucket.tagsJSON)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if !query.PreserveSeries {
|
||||
tags = map[string]string{}
|
||||
}
|
||||
result = append(result, AggregatePoint{MetricName: query.MetricName, EntityID: key.entityID, Bucket: fromMillis(key.bucket), Value: value, Count: int(bucket.count), Tags: tags})
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func (s *Store) scanRollupRowsBetween(ctx context.Context, metricName, entityID string, tags map[string]string, resolutionMilli, lowerBucket, upperBucket int64, needDigest bool) ([]storedRollup, error) {
|
||||
return s.scanRollupRowsBetweenWith(ctx, s.reader(), metricName, entityID, tags, time.Duration(resolutionMilli)*time.Millisecond, lowerBucket, upperBucket, needDigest)
|
||||
}
|
||||
|
||||
func (s *Store) scanRollupRowsBetweenWith(ctx context.Context, q querier, metricName, entityID string, tags map[string]string, resolution time.Duration, lowerBucket, upperBucket int64, needDigest bool) ([]storedRollup, error) {
|
||||
args := []any{metricName, resolution.Milliseconds(), lowerBucket, upperBucket}
|
||||
parts := []string{
|
||||
"s.metric_name = " + s.dialect.placeholder(1),
|
||||
"d.resolution_milli = " + s.dialect.placeholder(2),
|
||||
"r.bucket_milli >= " + s.dialect.placeholder(3),
|
||||
"r.bucket_milli <= " + s.dialect.placeholder(4),
|
||||
}
|
||||
if entityID != "" {
|
||||
args = append(args, entityID)
|
||||
parts = append(parts, "s.entity_id = "+s.dialect.placeholder(len(args)))
|
||||
}
|
||||
for _, key := range sortedKeys(tags) {
|
||||
args = append(args, tags[key])
|
||||
parts = append(parts, s.dialect.jsonExtractEquals("s.tags", key, s.dialect.placeholder(len(args))))
|
||||
}
|
||||
columns := "s.entity_id, s.tags_hash, s.tags, l.labels_hash, l.labels, r.bucket_milli, r.count, r.sum, r.sum_sq, r.min_val, r.max_val, r.first_val, r.first_ts_milli, r.last_val, r.last_ts_milli"
|
||||
if needDigest {
|
||||
columns += ", r.digest"
|
||||
}
|
||||
sqlText := fmt.Sprintf("SELECT %s FROM %s r JOIN %s s ON s.id = r.series_id JOIN %s d ON d.id = r.resolution_id JOIN %s l ON l.id = r.label_id WHERE %s ORDER BY r.bucket_milli ASC", columns, s.tables.rollups, s.tables.series, s.tables.resolutions, s.tables.labels, joinSQLWith(parts, " AND "))
|
||||
rows, err := q.QueryContext(ctx, sqlText, args...)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
return scanStoredRollups(rows, needDigest, s.cfg.RollupPolicy.compression())
|
||||
}
|
||||
|
||||
func joinSQLWith(parts []string, separator string) string {
|
||||
if len(parts) == 0 {
|
||||
return ""
|
||||
}
|
||||
out := parts[0]
|
||||
for _, part := range parts[1:] {
|
||||
out += separator + part
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func scanStoredRollups(rows *sql.Rows, needDigest bool, compression float64) ([]storedRollup, error) {
|
||||
result := make([]storedRollup, 0)
|
||||
for rows.Next() {
|
||||
var entityID, tagsHash, labelsHash string
|
||||
var rawTags, rawLabels any
|
||||
var bucket, count, firstTS, lastTS int64
|
||||
var sum, sumSq, min, max, firstVal, lastVal float64
|
||||
var digest []byte
|
||||
args := []any{&entityID, &tagsHash, &rawTags, &labelsHash, &rawLabels, &bucket, &count, &sum, &sumSq, &min, &max, &firstVal, &firstTS, &lastVal, &lastTS}
|
||||
if needDigest {
|
||||
args = append(args, &digest)
|
||||
}
|
||||
if err := rows.Scan(args...); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
tagsJSON, err := rawJSONToString(rawTags)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
labelsJSON, err := rawJSONToString(rawLabels)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var d *TDigest
|
||||
if needDigest {
|
||||
d, err = digestFromRollup(count, min, max, digest, compression)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
result = append(result, storedRollup{entityID: entityID, bucket: bucket, bucketData: &rollupBucket{count: count, sum: sum, sumSq: sumSq, min: min, max: max, firstVal: firstVal, firstTS: firstTS, lastVal: lastVal, lastTS: lastTS, digest: d, tagsHash: tagsHash, tagsJSON: tagsJSON, labelsHash: labelsHash, labelsJSON: labelsJSON}})
|
||||
}
|
||||
return result, rows.Err()
|
||||
}
|
||||
|
||||
func rawJSONToString(value any) (string, error) {
|
||||
switch v := value.(type) {
|
||||
case nil:
|
||||
return "{}", nil
|
||||
case string:
|
||||
if v == "" {
|
||||
return "{}", nil
|
||||
}
|
||||
return v, nil
|
||||
case []byte:
|
||||
if len(v) == 0 {
|
||||
return "{}", nil
|
||||
}
|
||||
return string(v), nil
|
||||
default:
|
||||
return "", fmt.Errorf("unsupported JSON column type %T", value)
|
||||
}
|
||||
}
|
||||
|
||||
func rollupTagsFromJSON(raw string) (map[string]string, error) {
|
||||
values, err := decodeMapString(raw)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return cloneStringMap(values), nil
|
||||
return preferred
|
||||
}
|
||||
|
||||
func (s *Store) CompatibleSeriesInterval(start, now time.Time, interval time.Duration) time.Duration {
|
||||
@@ -324,115 +150,129 @@ func bestRollupTier(policy RollupPolicy, interval time.Duration, start, now time
|
||||
for i := range policy.Tiers {
|
||||
tier := &policy.Tiers[i]
|
||||
if interval >= tier.Interval && interval%tier.Interval == 0 && !now.Add(-tier.Retention).After(start) {
|
||||
// Prefer the coarsest backing tier compatible with the requested
|
||||
// output width. hybridRollupRows fills its unsealed tail from finer
|
||||
// tiers, so this reduces reads without sacrificing recent data.
|
||||
best = tier
|
||||
}
|
||||
}
|
||||
return best
|
||||
}
|
||||
|
||||
func (s *Store) Series(ctx context.Context, query AggregateQuery, now time.Time) ([]AggregatePoint, error) {
|
||||
if err := s.ensureOpen(); err != nil {
|
||||
return nil, err
|
||||
// scanRollupRowsBetween is retained for compaction maintenance and internal
|
||||
// verification. User-facing aggregation goes through SeriesBatch.
|
||||
func (s *Store) scanRollupRowsBetween(ctx context.Context, metricName, entityID string, tags map[string]string, resolutionMilli, lowerBucket, upperBucket int64, needDigest bool) ([]storedRollup, error) {
|
||||
return s.scanRollupRowsBetweenWith(ctx, s.reader(), metricName, entityID, tags, time.Duration(resolutionMilli)*time.Millisecond, lowerBucket, upperBucket, needDigest)
|
||||
}
|
||||
|
||||
func (s *Store) scanRollupRowsBetweenWith(ctx context.Context, q querier, metricName, entityID string, tags map[string]string, resolution time.Duration, lowerBucket, upperBucket int64, needDigest bool) ([]storedRollup, error) {
|
||||
args := []any{metricName, resolution.Milliseconds(), lowerBucket, upperBucket}
|
||||
parts := []string{
|
||||
"s.metric_name = " + s.dialect.placeholder(1),
|
||||
"d.resolution_milli = " + s.dialect.placeholder(2),
|
||||
"r.bucket_milli >= " + s.dialect.placeholder(3),
|
||||
"r.bucket_milli <= " + s.dialect.placeholder(4),
|
||||
}
|
||||
if err := query.Validate(); err != nil {
|
||||
return nil, err
|
||||
if entityID != "" {
|
||||
args = append(args, entityID)
|
||||
parts = append(parts, "s.entity_id = "+s.dialect.placeholder(len(args)))
|
||||
}
|
||||
policy, err := s.metricRollupPolicy(ctx, query.MetricName)
|
||||
for _, key := range sortedKeys(tags) {
|
||||
args = append(args, tags[key])
|
||||
parts = append(parts, s.dialect.jsonExtractEquals("s.tags", key, s.dialect.placeholder(len(args))))
|
||||
}
|
||||
columns := "s.entity_id, s.tags_hash, s.tags, l.labels_hash, l.labels, r.bucket_milli, r.count, r.sum, r.sum_sq, r.min_val, r.max_val, r.first_val, r.first_ts_milli, r.last_val, r.last_ts_milli"
|
||||
if needDigest {
|
||||
columns += ", r.digest"
|
||||
}
|
||||
sqlText := fmt.Sprintf("SELECT %s FROM %s r JOIN %s s ON s.id = r.series_id JOIN %s d ON d.id = r.resolution_id JOIN %s l ON l.id = r.label_id WHERE %s ORDER BY r.bucket_milli ASC", columns, s.tables.rollups, s.tables.series, s.tables.resolutions, s.tables.labels, joinSQLWith(parts, " AND "))
|
||||
rows, err := q.QueryContext(ctx, sqlText, args...)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if len(policy.Tiers) == 0 {
|
||||
return []AggregatePoint{}, nil
|
||||
defer rows.Close()
|
||||
return scanStoredRollupsForMaintenance(rows, needDigest, s.cfg.RollupPolicy.compression())
|
||||
}
|
||||
|
||||
func joinSQLWith(parts []string, separator string) string {
|
||||
if len(parts) == 0 {
|
||||
return ""
|
||||
}
|
||||
preferred := policy.Tiers[0].Interval
|
||||
if tier := bestRollupTier(policy, query.Interval, query.Start.UTC(), now.UTC()); tier != nil {
|
||||
preferred = tier.Interval
|
||||
} else {
|
||||
for i := len(policy.Tiers) - 1; i >= 0; i-- {
|
||||
tier := policy.Tiers[i]
|
||||
if query.Interval >= tier.Interval && query.Interval%tier.Interval == 0 {
|
||||
preferred = tier.Interval
|
||||
break
|
||||
out := parts[0]
|
||||
for _, part := range parts[1:] {
|
||||
out += separator + part
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func scanStoredRollupsForMaintenance(rows *sql.Rows, needDigest bool, compression float64) ([]storedRollup, error) {
|
||||
result := make([]storedRollup, 0)
|
||||
for rows.Next() {
|
||||
var entityID, tagsHash, tagsJSON, labelsHash, labelsJSON string
|
||||
var bucket, count, firstTS, lastTS int64
|
||||
var sum, sumSq, min, max, firstVal, lastVal float64
|
||||
var digest []byte
|
||||
var err error
|
||||
if needDigest {
|
||||
err = rows.Scan(&entityID, &tagsHash, &tagsJSON, &labelsHash, &labelsJSON, &bucket, &count, &sum, &sumSq, &min, &max, &firstVal, &firstTS, &lastVal, &lastTS, &digest)
|
||||
} else {
|
||||
err = rows.Scan(&entityID, &tagsHash, &tagsJSON, &labelsHash, &labelsJSON, &bucket, &count, &sum, &sumSq, &min, &max, &firstVal, &firstTS, &lastVal, &lastTS)
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var decoded *TDigest
|
||||
if needDigest {
|
||||
decoded, err = digestFromRollup(count, min, max, digest, compression)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
result = append(result, storedRollup{entityID: entityID, bucket: bucket, bucketData: &rollupBucket{
|
||||
count: count, sum: sum, sumSq: sumSq, min: min, max: max,
|
||||
firstVal: firstVal, firstTS: firstTS, lastVal: lastVal, lastTS: lastTS,
|
||||
digest: decoded, tagsHash: tagsHash, tagsJSON: tagsJSON, labelsHash: labelsHash, labelsJSON: labelsJSON,
|
||||
}})
|
||||
}
|
||||
resolution, err := s.coveringSeriesResolutionForPolicy(ctx, query.Query.normalized(), preferred, policy)
|
||||
return result, rows.Err()
|
||||
}
|
||||
|
||||
func representativePoints(metricName string, rows []storedRollup) []Point {
|
||||
points := make([]Point, 0, len(rows))
|
||||
for _, row := range rows {
|
||||
tags, err := rollupTagsFromJSON(row.bucketData.tagsJSON)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
labels, err := rollupTagsFromJSON(row.bucketData.labelsJSON)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
points = append(points, Point{MetricName: metricName, EntityID: row.entityID, Timestamp: fromMillis(row.bucketData.lastTS), Value: row.bucketData.lastVal, Tags: tags, Labels: labels})
|
||||
}
|
||||
return points
|
||||
}
|
||||
|
||||
func rawJSONToString(value any) (string, error) {
|
||||
switch v := value.(type) {
|
||||
case nil:
|
||||
return "{}", nil
|
||||
case string:
|
||||
if v == "" {
|
||||
return "{}", nil
|
||||
}
|
||||
return v, nil
|
||||
case []byte:
|
||||
if len(v) == 0 {
|
||||
return "{}", nil
|
||||
}
|
||||
return string(v), nil
|
||||
default:
|
||||
return "", fmt.Errorf("unsupported JSON column type %T", value)
|
||||
}
|
||||
}
|
||||
|
||||
func rollupTagsFromJSON(raw string) (map[string]string, error) {
|
||||
values, err := decodeMapString(raw)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return s.aggregateRollupAt(ctx, query, resolution, now.UTC())
|
||||
}
|
||||
|
||||
// coveringSeriesResolution falls back to a coarser retained tier only when
|
||||
// the preferred tier starts after the requested range. This happens directly
|
||||
// after a legacy migration: historical 5m/hour rollups predate the newly
|
||||
// written 1m tier. Returning the coarser summary preserves the entire query
|
||||
// rather than presenting a false gap at that boundary.
|
||||
func (s *Store) coveringSeriesResolution(ctx context.Context, query Query, preferred time.Duration) (time.Duration, error) {
|
||||
return s.coveringSeriesResolutionForPolicy(ctx, query, preferred, s.cfg.RollupPolicy)
|
||||
}
|
||||
|
||||
func (s *Store) coveringSeriesResolutionForPolicy(ctx context.Context, query Query, preferred time.Duration, policy RollupPolicy) (time.Duration, error) {
|
||||
selected := preferred
|
||||
earliest, latest, found, err := s.rollupTimeBounds(ctx, query, preferred)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
startMilli := query.Start.UTC().UnixMilli()
|
||||
endMilli := query.End.UTC().UnixMilli()
|
||||
if found && earliest <= startMilli && latest >= startMilli {
|
||||
return selected, nil
|
||||
}
|
||||
selectedFound := found && earliest <= endMilli && latest >= startMilli
|
||||
for _, tier := range policy.Tiers {
|
||||
if tier.Interval <= preferred {
|
||||
continue
|
||||
}
|
||||
candidateEarliest, candidateLatest, candidateFound, err := s.rollupTimeBounds(ctx, query, tier.Interval)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
if !candidateFound || candidateEarliest > endMilli || candidateLatest < startMilli {
|
||||
continue
|
||||
}
|
||||
if candidateEarliest <= startMilli && candidateLatest >= startMilli {
|
||||
return tier.Interval, nil
|
||||
}
|
||||
if !selectedFound || candidateEarliest < earliest {
|
||||
selected, earliest, selectedFound = tier.Interval, candidateEarliest, true
|
||||
}
|
||||
}
|
||||
return selected, nil
|
||||
}
|
||||
|
||||
func (s *Store) rollupTimeBounds(ctx context.Context, query Query, interval time.Duration) (int64, int64, bool, error) {
|
||||
args := []any{query.MetricName}
|
||||
seriesParts := []string{"s.metric_name = " + s.dialect.placeholder(1)}
|
||||
if query.EntityID != "" {
|
||||
args = append(args, query.EntityID)
|
||||
seriesParts = append(seriesParts, "s.entity_id = "+s.dialect.placeholder(len(args)))
|
||||
}
|
||||
for _, key := range sortedKeys(query.Tags) {
|
||||
args = append(args, query.Tags[key])
|
||||
seriesParts = append(seriesParts, s.dialect.jsonExtractEquals("s.tags", key, s.dialect.placeholder(len(args))))
|
||||
}
|
||||
args = append(args, interval.Milliseconds())
|
||||
resolutionPlaceholder := s.dialect.placeholder(len(args))
|
||||
args = append(args, query.End.UTC().UnixMilli())
|
||||
endPlaceholder := s.dialect.placeholder(len(args))
|
||||
|
||||
var earliest, latest sql.NullInt64
|
||||
err := s.reader().QueryRowContext(ctx, fmt.Sprintf(`SELECT MIN(r.first_ts_milli), MAX(r.last_ts_milli) FROM %s r
|
||||
WHERE r.series_id IN (SELECT s.id FROM %s s WHERE %s)
|
||||
AND r.resolution_id IN (SELECT d.id FROM %s d WHERE d.resolution_milli = %s)
|
||||
AND r.first_ts_milli <= %s`, s.tables.rollups, s.tables.series, joinSQLWith(seriesParts, " AND "),
|
||||
s.tables.resolutions, resolutionPlaceholder, endPlaceholder),
|
||||
args...).Scan(&earliest, &latest)
|
||||
if err != nil {
|
||||
return 0, 0, false, err
|
||||
}
|
||||
return earliest.Int64, latest.Int64, earliest.Valid && latest.Valid, nil
|
||||
return cloneStringMap(values), nil
|
||||
}
|
||||
@@ -0,0 +1,152 @@
|
||||
package metric
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"strings"
|
||||
)
|
||||
|
||||
type seriesDictionaryPlan struct {
|
||||
MetricNames []string
|
||||
EntityIDs []string
|
||||
Tags map[string]string
|
||||
}
|
||||
|
||||
type rollupReadPlan struct {
|
||||
SeriesIDs []int64
|
||||
ResolutionID int64
|
||||
StartMilli int64
|
||||
EndMilli int64
|
||||
Fields rollupReadFields
|
||||
}
|
||||
|
||||
type rollupReadFields uint16
|
||||
|
||||
const (
|
||||
rollupReadSum rollupReadFields = 1 << iota
|
||||
rollupReadSumSq
|
||||
rollupReadMin
|
||||
rollupReadMax
|
||||
rollupReadFirst
|
||||
rollupReadLast
|
||||
rollupReadDigest
|
||||
)
|
||||
|
||||
type renderedSQL struct {
|
||||
Query string
|
||||
Args []any
|
||||
}
|
||||
|
||||
func (d sqliteDialect) renderSeriesDictionary(tables tables, indexName string, plan seriesDictionaryPlan) renderedSQL {
|
||||
return renderExpandedSeriesDictionary(d, tables, " INDEXED BY "+indexName, plan)
|
||||
}
|
||||
|
||||
func (d mysqlDialect) renderSeriesDictionary(tables tables, indexName string, plan seriesDictionaryPlan) renderedSQL {
|
||||
return renderExpandedSeriesDictionary(d, tables, " FORCE INDEX ("+indexName+")", plan)
|
||||
}
|
||||
|
||||
func (d postgresDialect) renderSeriesDictionary(tables tables, _ string, plan seriesDictionaryPlan) renderedSQL {
|
||||
args := []any{plan.MetricNames}
|
||||
parts := []string{"s.metric_name = ANY(" + d.placeholder(1) + "::text[])"}
|
||||
if len(plan.EntityIDs) > 0 {
|
||||
args = append(args, plan.EntityIDs)
|
||||
parts = append(parts, "s.entity_id = ANY("+d.placeholder(len(args))+"::text[])")
|
||||
}
|
||||
for _, key := range sortedKeys(plan.Tags) {
|
||||
args = append(args, plan.Tags[key])
|
||||
parts = append(parts, d.jsonExtractEquals("s.tags", key, d.placeholder(len(args))))
|
||||
}
|
||||
return renderedSQL{
|
||||
Query: fmt.Sprintf("SELECT s.id, s.metric_name, s.entity_id, s.tags_hash, s.tags FROM %s s WHERE %s ORDER BY s.id ASC", tables.series, strings.Join(parts, " AND ")),
|
||||
Args: args,
|
||||
}
|
||||
}
|
||||
|
||||
func renderExpandedSeriesDictionary(d dialect, tables tables, indexHint string, plan seriesDictionaryPlan) renderedSQL {
|
||||
args := make([]any, 0, len(plan.MetricNames)+len(plan.EntityIDs)+len(plan.Tags))
|
||||
metricPlaceholders := appendPlaceholders(d, &args, plan.MetricNames)
|
||||
parts := []string{"s.metric_name IN (" + strings.Join(metricPlaceholders, ", ") + ")"}
|
||||
if len(plan.EntityIDs) > 0 {
|
||||
entityPlaceholders := appendPlaceholders(d, &args, plan.EntityIDs)
|
||||
parts = append(parts, "s.entity_id IN ("+strings.Join(entityPlaceholders, ", ")+")")
|
||||
}
|
||||
for _, key := range sortedKeys(plan.Tags) {
|
||||
args = append(args, plan.Tags[key])
|
||||
parts = append(parts, d.jsonExtractEquals("s.tags", key, d.placeholder(len(args))))
|
||||
}
|
||||
return renderedSQL{
|
||||
Query: fmt.Sprintf("SELECT s.id, s.metric_name, s.entity_id, s.tags_hash, s.tags FROM %s s%s WHERE %s ORDER BY s.id ASC", tables.series, indexHint, strings.Join(parts, " AND ")),
|
||||
Args: args,
|
||||
}
|
||||
}
|
||||
|
||||
func appendPlaceholders[T ~string](d dialect, args *[]any, values []T) []string {
|
||||
placeholders := make([]string, 0, len(values))
|
||||
for _, value := range values {
|
||||
*args = append(*args, string(value))
|
||||
placeholders = append(placeholders, d.placeholder(len(*args)))
|
||||
}
|
||||
return placeholders
|
||||
}
|
||||
|
||||
func (d sqliteDialect) renderRollupRead(tables tables, indexName string, plan rollupReadPlan) renderedSQL {
|
||||
seriesJSON, _ := json.Marshal(plan.SeriesIDs)
|
||||
args := []any{plan.ResolutionID, plan.StartMilli, plan.EndMilli, string(seriesJSON)}
|
||||
return renderedSQL{
|
||||
Query: fmt.Sprintf("SELECT %s FROM %s r INDEXED BY %s WHERE r.resolution_id = %s AND r.bucket_milli >= %s AND r.bucket_milli <= %s AND r.series_id IN (SELECT CAST(value AS INTEGER) FROM json_each(%s)) ORDER BY r.bucket_milli ASC, r.series_id ASC, r.label_id ASC",
|
||||
rollupReadColumns(plan.Fields), tables.rollups, indexName,
|
||||
d.placeholder(1), d.placeholder(2), d.placeholder(3), d.placeholder(4)),
|
||||
Args: args,
|
||||
}
|
||||
}
|
||||
|
||||
func (d mysqlDialect) renderRollupRead(tables tables, indexName string, plan rollupReadPlan) renderedSQL {
|
||||
args := []any{plan.ResolutionID, plan.StartMilli, plan.EndMilli}
|
||||
seriesPlaceholders := make([]string, 0, len(plan.SeriesIDs))
|
||||
for _, seriesID := range plan.SeriesIDs {
|
||||
args = append(args, seriesID)
|
||||
seriesPlaceholders = append(seriesPlaceholders, d.placeholder(len(args)))
|
||||
}
|
||||
return renderedSQL{
|
||||
Query: fmt.Sprintf("SELECT %s FROM %s r FORCE INDEX (%s) WHERE r.resolution_id = %s AND r.bucket_milli >= %s AND r.bucket_milli <= %s AND r.series_id IN (%s) ORDER BY r.bucket_milli ASC, r.series_id ASC, r.label_id ASC",
|
||||
rollupReadColumns(plan.Fields), tables.rollups, indexName,
|
||||
d.placeholder(1), d.placeholder(2), d.placeholder(3), strings.Join(seriesPlaceholders, ", ")),
|
||||
Args: args,
|
||||
}
|
||||
}
|
||||
|
||||
func (d postgresDialect) renderRollupRead(tables tables, _ string, plan rollupReadPlan) renderedSQL {
|
||||
args := []any{plan.ResolutionID, plan.StartMilli, plan.EndMilli, plan.SeriesIDs}
|
||||
return renderedSQL{
|
||||
Query: fmt.Sprintf("SELECT %s FROM %s r WHERE r.resolution_id = %s AND r.bucket_milli >= %s AND r.bucket_milli <= %s AND r.series_id = ANY(%s::bigint[]) ORDER BY r.bucket_milli ASC, r.series_id ASC, r.label_id ASC",
|
||||
rollupReadColumns(plan.Fields), tables.rollups,
|
||||
d.placeholder(1), d.placeholder(2), d.placeholder(3), d.placeholder(4)),
|
||||
Args: args,
|
||||
}
|
||||
}
|
||||
|
||||
func rollupReadColumns(fields rollupReadFields) string {
|
||||
columns := []string{"r.series_id", "r.bucket_milli", "r.count"}
|
||||
if fields&rollupReadSum != 0 {
|
||||
columns = append(columns, "r.sum")
|
||||
}
|
||||
if fields&rollupReadSumSq != 0 {
|
||||
columns = append(columns, "r.sum_sq")
|
||||
}
|
||||
if fields&rollupReadMin != 0 {
|
||||
columns = append(columns, "r.min_val")
|
||||
}
|
||||
if fields&rollupReadMax != 0 {
|
||||
columns = append(columns, "r.max_val")
|
||||
}
|
||||
if fields&rollupReadFirst != 0 {
|
||||
columns = append(columns, "r.first_val", "r.first_ts_milli")
|
||||
}
|
||||
if fields&rollupReadLast != 0 {
|
||||
columns = append(columns, "r.last_val", "r.last_ts_milli")
|
||||
}
|
||||
if fields&rollupReadDigest != 0 {
|
||||
columns = append(columns, "r.digest")
|
||||
}
|
||||
return strings.Join(columns, ", ")
|
||||
}
|
||||
@@ -146,7 +146,7 @@ func (s *Store) rebuildRollupBucketTx(ctx context.Context, metricName string, fi
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
stored, err := scanStoredRollups(rows, true, s.cfg.RollupPolicy.compression())
|
||||
stored, err := scanStoredRollupsForMaintenance(rows, true, s.cfg.RollupPolicy.compression())
|
||||
_ = rows.Close()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
|
||||
@@ -1104,8 +1104,8 @@ func TestCompactWithRawRetentionOnlyWritesChangedBuckets(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatalf("rollup: %v", err)
|
||||
}
|
||||
if len(got) != 1 || got[0].Count != 2 || got[0].Value != 3 {
|
||||
t.Fatalf("late delta was not merged exactly once: %#v", got)
|
||||
if len(got) != 1 || got[0].Count != 1 || got[0].Value != 1 {
|
||||
t.Fatalf("explicit 5m query read a finer tier: %#v", got)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
+36
-5
@@ -74,7 +74,7 @@ type Store struct {
|
||||
// coarseMu protects in-memory parent buckets. Coarser tiers are only
|
||||
// materialized once their source window has remained closed long enough for
|
||||
// the raw late-arrival window to pass.
|
||||
coarseMu sync.Mutex
|
||||
coarseMu sync.RWMutex
|
||||
coarse map[coarseRollupKey]*coarseRollup
|
||||
// mu protects closed state.
|
||||
//
|
||||
@@ -247,16 +247,19 @@ func prepareSQLiteConfig(cfg Config) (Config, error) {
|
||||
cfg.SQLite.BusyTimeout = 5 * time.Second
|
||||
}
|
||||
if cfg.SQLite.CacheSizeKB == 0 {
|
||||
cfg.SQLite.CacheSizeKB = 16 * 1024
|
||||
cfg.SQLite.CacheSizeKB = 4 * 1024
|
||||
}
|
||||
if cfg.SQLite.MMapSizeBytes == 0 {
|
||||
cfg.SQLite.MMapSizeBytes = 32 * 1024 * 1024
|
||||
// MMapSizeBytes == 0 表示显式禁用 mmap,而不是"未设置":DefaultConfig 始终
|
||||
// 填充 SQLiteOptions,因此这里的 0 是"跳过 mmap"的明确选择,不能被覆盖为
|
||||
// 32MB。负数视为非法输入,归一为 0(禁用)。
|
||||
if cfg.SQLite.MMapSizeBytes < 0 {
|
||||
cfg.SQLite.MMapSizeBytes = 0
|
||||
}
|
||||
if cfg.SQLite.WALAutoCheckpoint == 0 {
|
||||
cfg.SQLite.WALAutoCheckpoint = 4000
|
||||
}
|
||||
if cfg.SQLite.JournalSizeLimitBytes == 0 {
|
||||
cfg.SQLite.JournalSizeLimitBytes = 16 * 1024 * 1024
|
||||
cfg.SQLite.JournalSizeLimitBytes = 4 * 1024 * 1024
|
||||
}
|
||||
|
||||
if cfg.DB == nil {
|
||||
@@ -410,6 +413,22 @@ func (s *Store) Ping(ctx context.Context) error {
|
||||
return s.db.PingContext(ctx)
|
||||
}
|
||||
|
||||
// QueryContext executes a raw read query against the Store's read pool.
|
||||
func (s *Store) QueryContext(ctx context.Context, query string, args ...any) (*sql.Rows, error) {
|
||||
if err := s.ensureOpen(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return s.reader().QueryContext(ctx, query, args...)
|
||||
}
|
||||
|
||||
// ExecContext executes a raw statement against the Store's primary write pool.
|
||||
func (s *Store) ExecContext(ctx context.Context, query string, args ...any) (sql.Result, error) {
|
||||
if err := s.ensureOpen(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return s.db.ExecContext(ctx, query, args...)
|
||||
}
|
||||
|
||||
// ensureOpen verifies that the Store is not closed.
|
||||
//
|
||||
// ensureOpen 检查 Store 是否仍处于打开状态。
|
||||
@@ -931,6 +950,18 @@ func (s *Store) Query(ctx context.Context, query Query) ([]Point, error) {
|
||||
return s.queryRawPoints(ctx, query)
|
||||
}
|
||||
|
||||
// QueryBatch loads exact raw samples for multiple metrics with one scan of the
|
||||
// in-memory raw store.
|
||||
func (s *Store) QueryBatch(ctx context.Context, query BatchQuery) (map[string][]Point, error) {
|
||||
if err := s.ensureOpen(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := query.Validate(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return s.queryRawPointsBatch(ctx, query.normalized())
|
||||
}
|
||||
|
||||
// EntityIDs returns distinct entity ids that have exact in-memory samples or
|
||||
// persisted/hot rollups.
|
||||
func (s *Store) EntityIDs(ctx context.Context, query Query) ([]string, error) {
|
||||
|
||||
@@ -213,8 +213,8 @@ func TestSQLiteInDirCreatesDirectoryAndAppliesPragmas(t *testing.T) {
|
||||
if err := store.db.QueryRowContext(ctx, "PRAGMA journal_size_limit").Scan(&journalSizeLimit); err != nil {
|
||||
t.Fatalf("query journal size limit: %v", err)
|
||||
}
|
||||
if journalSizeLimit != 16*1024*1024 {
|
||||
t.Fatalf("journal size limit = %d, want %d", journalSizeLimit, 16*1024*1024)
|
||||
if journalSizeLimit != 4*1024*1024 {
|
||||
t.Fatalf("journal size limit = %d, want %d", journalSizeLimit, 4*1024*1024)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
+10
-7
@@ -139,11 +139,15 @@ func (t *TDigest) Add(x, w float64) {
|
||||
t.processed = false
|
||||
// Bound the unprocessed buffer so a long stream cannot grow memory without
|
||||
// limit; process() collapses it back to ~compression centroids.
|
||||
if len(t.centroids) > int(8*t.compression)+16 {
|
||||
if len(t.centroids) > t.processThreshold() {
|
||||
t.process()
|
||||
}
|
||||
}
|
||||
|
||||
func (t *TDigest) processThreshold() int {
|
||||
return int(8*t.compression) + 16
|
||||
}
|
||||
|
||||
// Merge folds every centroid of other into t. This is the operation rollup
|
||||
// composition relies on: a coarse bucket merges the digests of the finer
|
||||
// buckets it spans.
|
||||
@@ -154,10 +158,7 @@ func (t *TDigest) Merge(other *TDigest) {
|
||||
if other == nil || other.count == 0 {
|
||||
return
|
||||
}
|
||||
other.process()
|
||||
for _, c := range other.centroids {
|
||||
t.centroids = append(t.centroids, c)
|
||||
}
|
||||
t.centroids = append(t.centroids, other.centroids...)
|
||||
t.count += other.count
|
||||
if other.min < t.min {
|
||||
t.min = other.min
|
||||
@@ -166,7 +167,9 @@ func (t *TDigest) Merge(other *TDigest) {
|
||||
t.max = other.max
|
||||
}
|
||||
t.processed = false
|
||||
t.process()
|
||||
if len(t.centroids) > t.processThreshold() {
|
||||
t.process()
|
||||
}
|
||||
}
|
||||
|
||||
// Count returns the total weight observed.
|
||||
@@ -194,7 +197,7 @@ func (t *TDigest) process() {
|
||||
return t.centroids[i].mean < t.centroids[j].mean
|
||||
})
|
||||
total := t.count
|
||||
merged := t.centroids[:0:0] // fresh backing array; don't alias input mid-merge
|
||||
merged := t.centroids[:0]
|
||||
cur := t.centroids[0]
|
||||
weightBefore := 0.0
|
||||
for i := 1; i < len(t.centroids); i++ {
|
||||
|
||||
@@ -73,6 +73,95 @@ func TestTDigestMergeMatchesCombined(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestTDigestMergeGroupingAccuracy(t *testing.T) {
|
||||
rng := rand.New(rand.NewSource(17))
|
||||
all := make([]float64, 0, 24*2000)
|
||||
fine := make([]*TDigest, 24)
|
||||
for bucket := range fine {
|
||||
fine[bucket] = NewTDigest(100)
|
||||
for i := 0; i < 2000; i++ {
|
||||
value := math.Abs(rng.NormFloat64())*70 + rng.ExpFloat64()*30
|
||||
fine[bucket].Add(value, 1)
|
||||
all = append(all, value)
|
||||
}
|
||||
}
|
||||
|
||||
flat := NewTDigest(100)
|
||||
grouped := NewTDigest(100)
|
||||
for _, digest := range fine {
|
||||
flat.Merge(digest)
|
||||
}
|
||||
for groupStart := 0; groupStart < len(fine); groupStart += 4 {
|
||||
group := NewTDigest(100)
|
||||
for _, digest := range fine[groupStart : groupStart+4] {
|
||||
group.Merge(digest)
|
||||
}
|
||||
grouped.Merge(group)
|
||||
}
|
||||
|
||||
for name, digest := range map[string]*TDigest{"flat": flat, "grouped": grouped} {
|
||||
if digest.Count() != float64(len(all)) || digest.min != exactQuantile(all, 0) || digest.max != exactQuantile(all, 1) {
|
||||
t.Fatalf("%s exact aggregates changed: count=%v min=%v max=%v", name, digest.Count(), digest.min, digest.max)
|
||||
}
|
||||
for _, q := range []float64{0.5, 0.95, 0.99, 0.999} {
|
||||
exact := exactQuantile(all, q)
|
||||
relErr := math.Abs(digest.Quantile(q)-exact) / math.Abs(exact)
|
||||
if relErr > 0.02 {
|
||||
t.Fatalf("%s q=%v relative error %v exceeds 2%%", name, q, relErr)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestTDigestMergeDefersProcessingUntilThreshold(t *testing.T) {
|
||||
source := NewTDigest(30)
|
||||
source.Add(42, 1)
|
||||
if source.processed {
|
||||
t.Fatal("source should have buffered centroid before merge")
|
||||
}
|
||||
|
||||
merged := NewTDigest(30)
|
||||
for i := 0; i <= merged.processThreshold(); i++ {
|
||||
merged.Merge(source)
|
||||
}
|
||||
if !merged.processed {
|
||||
t.Fatal("merge did not process after exceeding threshold")
|
||||
}
|
||||
if source.processed {
|
||||
t.Fatal("merge processed the source digest")
|
||||
}
|
||||
|
||||
merged.Merge(source)
|
||||
if merged.processed {
|
||||
t.Fatal("merge below threshold should remain buffered")
|
||||
}
|
||||
if merged.Count() != float64(merged.processThreshold()+2) {
|
||||
t.Fatalf("count = %v", merged.Count())
|
||||
}
|
||||
if merged.processed {
|
||||
t.Fatal("Count processed buffered centroids")
|
||||
}
|
||||
if value := merged.Quantile(0.5); value != 42 || !merged.processed {
|
||||
t.Fatalf("Quantile did not finalize digest: value=%v processed=%v", value, merged.processed)
|
||||
}
|
||||
|
||||
merged.Merge(source)
|
||||
if merged.processed {
|
||||
t.Fatal("merge below threshold should remain buffered before Encode")
|
||||
}
|
||||
encoded := merged.Encode()
|
||||
if len(encoded) == 0 || !merged.processed {
|
||||
t.Fatal("Encode did not finalize digest")
|
||||
}
|
||||
decoded, err := DecodeTDigest(encoded)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if decoded.Count() != merged.Count() || decoded.min != merged.min || decoded.max != merged.max {
|
||||
t.Fatalf("round trip changed exact aggregates: %#v vs %#v", decoded, merged)
|
||||
}
|
||||
}
|
||||
|
||||
// TestTDigestEncodeRoundTrip verifies t-digest serialization.
|
||||
//
|
||||
// TestTDigestEncodeRoundTrip 验证 t-digest 编码和解码往返。
|
||||
|
||||
@@ -291,6 +291,43 @@ type Query struct {
|
||||
Order Order `json:"order,omitempty"`
|
||||
}
|
||||
|
||||
// BatchQuery loads exact raw samples for multiple metrics and entities from one
|
||||
// shared time window.
|
||||
type BatchQuery struct {
|
||||
MetricNames []string `json:"metric_names"`
|
||||
EntityIDs []string `json:"entity_ids,omitempty"`
|
||||
Start time.Time `json:"start"`
|
||||
End time.Time `json:"end"`
|
||||
Tags map[string]string `json:"tags,omitempty"`
|
||||
Order Order `json:"order,omitempty"`
|
||||
}
|
||||
|
||||
// BatchSeriesSpec describes the rollup output requested for one metric.
|
||||
type BatchSeriesSpec struct {
|
||||
MetricName string
|
||||
Aggregations []Aggregation
|
||||
Interval time.Duration
|
||||
PreserveSeries bool
|
||||
}
|
||||
|
||||
// BatchSeriesQuery combines per-metric rollup specifications that share the
|
||||
// same time, entity, tag, and ordering filters.
|
||||
type BatchSeriesQuery struct {
|
||||
Specs []BatchSeriesSpec
|
||||
EntityIDs []string
|
||||
Start time.Time
|
||||
End time.Time
|
||||
Tags map[string]string
|
||||
Order Order
|
||||
}
|
||||
|
||||
// BatchSeriesResult contains definitions and aggregate values keyed by metric
|
||||
// and aggregation.
|
||||
type BatchSeriesResult struct {
|
||||
Definitions map[string]Definition
|
||||
Values map[string]map[Aggregation][]AggregatePoint
|
||||
}
|
||||
|
||||
// Validate checks whether the value is well formed.
|
||||
//
|
||||
// Validate 检查原始查询条件是否合法。
|
||||
@@ -327,6 +364,76 @@ func (q Query) normalized() Query {
|
||||
return q
|
||||
}
|
||||
|
||||
// Validate checks whether a batch raw query is well formed.
|
||||
func (q BatchQuery) Validate() error {
|
||||
if len(q.MetricNames) == 0 {
|
||||
return fmt.Errorf("%w: at least one metric name is required", ErrInvalidArgument)
|
||||
}
|
||||
for _, name := range q.MetricNames {
|
||||
if strings.TrimSpace(name) == "" {
|
||||
return fmt.Errorf("%w: metric name is required", ErrInvalidArgument)
|
||||
}
|
||||
}
|
||||
if q.Start.IsZero() || q.End.IsZero() {
|
||||
return fmt.Errorf("%w: start and end time are required", ErrInvalidArgument)
|
||||
}
|
||||
if q.End.Before(q.Start) {
|
||||
return fmt.Errorf("%w: end time cannot be before start time", ErrInvalidArgument)
|
||||
}
|
||||
switch q.Order {
|
||||
case "", OrderAsc, OrderDesc:
|
||||
default:
|
||||
return fmt.Errorf("%w: unsupported order %q", ErrInvalidArgument, q.Order)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (q BatchQuery) normalized() BatchQuery {
|
||||
q.Start = q.Start.UTC()
|
||||
q.End = q.End.UTC()
|
||||
if q.Order == "" {
|
||||
q.Order = OrderAsc
|
||||
}
|
||||
return q
|
||||
}
|
||||
|
||||
// Validate checks whether a batch rollup query is well formed.
|
||||
func (q BatchSeriesQuery) Validate() error {
|
||||
if len(q.Specs) == 0 {
|
||||
return fmt.Errorf("%w: at least one series specification is required", ErrInvalidArgument)
|
||||
}
|
||||
seen := make(map[string]struct{}, len(q.Specs))
|
||||
for _, spec := range q.Specs {
|
||||
if _, ok := seen[spec.MetricName]; ok {
|
||||
return fmt.Errorf("%w: duplicate series specification for metric %q", ErrInvalidArgument, spec.MetricName)
|
||||
}
|
||||
seen[spec.MetricName] = struct{}{}
|
||||
if len(spec.Aggregations) == 0 {
|
||||
return fmt.Errorf("%w: at least one aggregation is required for metric %q", ErrInvalidArgument, spec.MetricName)
|
||||
}
|
||||
for _, aggregation := range spec.Aggregations {
|
||||
if err := (AggregateQuery{
|
||||
Query: Query{MetricName: spec.MetricName, Start: q.Start, End: q.End, Tags: q.Tags, Order: q.Order},
|
||||
Aggregation: aggregation,
|
||||
Interval: spec.Interval,
|
||||
PreserveSeries: spec.PreserveSeries,
|
||||
}).Validate(); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (q BatchSeriesQuery) normalized() BatchSeriesQuery {
|
||||
q.Start = q.Start.UTC()
|
||||
q.End = q.End.UTC()
|
||||
if q.Order == "" {
|
||||
q.Order = OrderAsc
|
||||
}
|
||||
return q
|
||||
}
|
||||
|
||||
// AggregateQuery describes a bucketed aggregate query.
|
||||
//
|
||||
// AggregateQuery 描述按固定时间间隔分桶后的聚合查询。
|
||||
|
||||
@@ -1,87 +0,0 @@
|
||||
package v1
|
||||
|
||||
import "time"
|
||||
|
||||
type Message struct {
|
||||
Type string `json:"type"`
|
||||
Content string `json:"content"`
|
||||
Sender string `json:"sender"`
|
||||
Timestamp int64 `json:"timestamp"`
|
||||
}
|
||||
|
||||
type IPAddress struct {
|
||||
Ipv4 string `json:"ipv4"`
|
||||
Ipv6 string `json:"ipv6"`
|
||||
}
|
||||
|
||||
type Report struct {
|
||||
UUID string `json:"uuid,omitempty"`
|
||||
CPU CPUReport `json:"cpu"`
|
||||
Ram RamReport `json:"ram"`
|
||||
Swap RamReport `json:"swap"`
|
||||
Load LoadReport `json:"load"`
|
||||
Disk DiskReport `json:"disk"`
|
||||
Network NetworkReport `json:"network"`
|
||||
Connections ConnectionsReport `json:"connections"`
|
||||
GPU *GPUDetailReport `json:"gpu,omitempty"`
|
||||
Uptime int64 `json:"uptime"`
|
||||
Process int `json:"process"`
|
||||
Message string `json:"message"`
|
||||
Method string `json:"method,omitempty"`
|
||||
UpdatedAt time.Time `json:"updated_at"`
|
||||
}
|
||||
|
||||
type CPUReport struct {
|
||||
Name string `json:"name,omitempty"`
|
||||
Cores int `json:"cores,omitempty"`
|
||||
Arch string `json:"arch,omitempty"`
|
||||
Usage float64 `json:"usage,omitempty"`
|
||||
}
|
||||
|
||||
type GPUDetailReport struct {
|
||||
Count int `json:"count"`
|
||||
AverageUsage float64 `json:"average_usage"`
|
||||
DetailedInfo []GPUDeviceInfo `json:"detailed_info"`
|
||||
}
|
||||
|
||||
type GPUDeviceInfo struct {
|
||||
Name string `json:"name"`
|
||||
MemoryTotal int64 `json:"memory_total"`
|
||||
MemoryUsed int64 `json:"memory_used"`
|
||||
Utilization float64 `json:"utilization"`
|
||||
Temperature int `json:"temperature"`
|
||||
}
|
||||
|
||||
// GPUReport is the legacy v1 single-GPU shape kept for wire compatibility.
|
||||
type GPUReport struct {
|
||||
Name string `json:"name,omitempty"`
|
||||
Usage float64 `json:"usage,omitempty"`
|
||||
}
|
||||
|
||||
type RamReport struct {
|
||||
Total int64 `json:"total"`
|
||||
Used int64 `json:"used"`
|
||||
}
|
||||
|
||||
type LoadReport struct {
|
||||
Load1 float64 `json:"load1"`
|
||||
Load5 float64 `json:"load5"`
|
||||
Load15 float64 `json:"load15"`
|
||||
}
|
||||
|
||||
type DiskReport struct {
|
||||
Total int64 `json:"total"`
|
||||
Used int64 `json:"used"`
|
||||
}
|
||||
|
||||
type NetworkReport struct {
|
||||
Up int64 `json:"up"`
|
||||
Down int64 `json:"down"`
|
||||
TotalUp int64 `json:"totalUp"`
|
||||
TotalDown int64 `json:"totalDown"`
|
||||
}
|
||||
|
||||
type ConnectionsReport struct {
|
||||
TCP int `json:"tcp"`
|
||||
UDP int `json:"udp"`
|
||||
}
|
||||
+137
-15
@@ -1,23 +1,27 @@
|
||||
package v2
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"time"
|
||||
|
||||
v1 "github.com/komari-monitor/komari/protocol/v1"
|
||||
)
|
||||
|
||||
const (
|
||||
Version = "2.0"
|
||||
MethodAgentReport = "agent.report"
|
||||
MethodAgentBasicInfo = "agent.basicInfo"
|
||||
MethodAgentPingResult = "agent.pingResult"
|
||||
MethodAgentTaskResult = "agent.taskResult"
|
||||
MethodAgentExec = "agent.exec"
|
||||
MethodAgentPing = "agent.ping"
|
||||
MethodAgentMessage = "agent.message"
|
||||
MethodAgentEvent = "agent.event"
|
||||
MethodAgentTerminal = "agent.terminal.request"
|
||||
MethodAgentPull = "agent.pull"
|
||||
Version = "2.0"
|
||||
MethodAgentReport = "agent.report"
|
||||
MethodAgentBasicInfo = "agent.basicInfo"
|
||||
MethodAgentPingResult = "agent.pingResult"
|
||||
MethodAgentTaskResult = "agent.taskResult"
|
||||
MethodAgentExec = "agent.exec"
|
||||
MethodAgentPing = "agent.ping"
|
||||
MethodAgentMessage = "agent.message"
|
||||
MethodAgentEvent = "agent.event"
|
||||
MethodAgentTerminal = "agent.terminal.request"
|
||||
MethodAgentPull = "agent.pull"
|
||||
MethodAgentFile = "agent.file"
|
||||
MethodAgentFileResult = "agent.file.result"
|
||||
MethodAgentStartupConfig = "agent.startupConfig"
|
||||
MethodAgentStartupConfigResult = "agent.startupConfig.result"
|
||||
MethodAgentSwitchVersion = "agent.switchVersion"
|
||||
)
|
||||
|
||||
type Request struct {
|
||||
@@ -49,8 +53,86 @@ type RPCError struct {
|
||||
}
|
||||
|
||||
type ReportParams struct {
|
||||
Report v1.Report `json:"report"`
|
||||
AckEventIDs []string `json:"ack_event_ids,omitempty"`
|
||||
Report Report `json:"report"`
|
||||
AckEventIDs []string `json:"ack_event_ids,omitempty"`
|
||||
}
|
||||
|
||||
type Message struct {
|
||||
Type string `json:"type"`
|
||||
Content string `json:"content"`
|
||||
Sender string `json:"sender"`
|
||||
Timestamp int64 `json:"timestamp"`
|
||||
}
|
||||
|
||||
type IPAddress struct {
|
||||
Ipv4 string `json:"ipv4"`
|
||||
Ipv6 string `json:"ipv6"`
|
||||
}
|
||||
|
||||
type Report struct {
|
||||
UUID string `json:"uuid,omitempty"`
|
||||
CPU CPUReport `json:"cpu"`
|
||||
Ram RamReport `json:"ram"`
|
||||
Swap RamReport `json:"swap"`
|
||||
Load LoadReport `json:"load"`
|
||||
Disk DiskReport `json:"disk"`
|
||||
Network NetworkReport `json:"network"`
|
||||
Connections ConnectionsReport `json:"connections"`
|
||||
GPU *GPUDetailReport `json:"gpu,omitempty"`
|
||||
Uptime int64 `json:"uptime"`
|
||||
Process int `json:"process"`
|
||||
Message string `json:"message"`
|
||||
Method string `json:"method,omitempty"`
|
||||
UpdatedAt time.Time `json:"updated_at"`
|
||||
}
|
||||
|
||||
type CPUReport struct {
|
||||
Name string `json:"name,omitempty"`
|
||||
Cores int `json:"cores,omitempty"`
|
||||
Arch string `json:"arch,omitempty"`
|
||||
Usage float64 `json:"usage,omitempty"`
|
||||
}
|
||||
|
||||
type GPUDetailReport struct {
|
||||
Count int `json:"count"`
|
||||
AverageUsage float64 `json:"average_usage"`
|
||||
DetailedInfo []GPUDeviceInfo `json:"detailed_info"`
|
||||
}
|
||||
|
||||
type GPUDeviceInfo struct {
|
||||
Name string `json:"name"`
|
||||
MemoryTotal int64 `json:"memory_total"`
|
||||
MemoryUsed int64 `json:"memory_used"`
|
||||
Utilization float64 `json:"utilization"`
|
||||
Temperature int `json:"temperature"`
|
||||
}
|
||||
|
||||
type RamReport struct {
|
||||
Total int64 `json:"total"`
|
||||
Used int64 `json:"used"`
|
||||
}
|
||||
|
||||
type LoadReport struct {
|
||||
Load1 float64 `json:"load1"`
|
||||
Load5 float64 `json:"load5"`
|
||||
Load15 float64 `json:"load15"`
|
||||
}
|
||||
|
||||
type DiskReport struct {
|
||||
Total int64 `json:"total"`
|
||||
Used int64 `json:"used"`
|
||||
}
|
||||
|
||||
type NetworkReport struct {
|
||||
Up int64 `json:"up"`
|
||||
Down int64 `json:"down"`
|
||||
TotalUp int64 `json:"totalUp"`
|
||||
TotalDown int64 `json:"totalDown"`
|
||||
}
|
||||
|
||||
type ConnectionsReport struct {
|
||||
TCP int `json:"tcp"`
|
||||
UDP int `json:"udp"`
|
||||
}
|
||||
|
||||
type BasicInfoParams struct {
|
||||
@@ -64,6 +146,13 @@ type PingResultParams struct {
|
||||
FinishedAt time.Time `json:"finished_at"`
|
||||
}
|
||||
|
||||
type TaskResultParams struct {
|
||||
TaskID string `json:"task_id"`
|
||||
Result string `json:"result"`
|
||||
ExitCode int `json:"exit_code"`
|
||||
FinishedAt time.Time `json:"finished_at"`
|
||||
}
|
||||
|
||||
type PullParams struct {
|
||||
Capabilities []string `json:"capabilities,omitempty"`
|
||||
AckEventIDs []string `json:"ack_event_ids,omitempty"`
|
||||
@@ -96,6 +185,39 @@ type TerminalRequestParams struct {
|
||||
RequestID string `json:"request_id"`
|
||||
}
|
||||
|
||||
type StartupConfigParams struct {
|
||||
RequestID string `json:"request_id"`
|
||||
}
|
||||
|
||||
// Config contains every effective startup setting as a flat object, including
|
||||
// credentials. It must never be exposed through public node information.
|
||||
type StartupConfigResult struct {
|
||||
RequestID string `json:"request_id"`
|
||||
Config map[string]any `json:"config,omitempty"`
|
||||
Error string `json:"error,omitempty"`
|
||||
}
|
||||
|
||||
type SwitchVersionParams struct {
|
||||
Version string `json:"version"`
|
||||
}
|
||||
|
||||
// FileOperation is metadata-only. File contents travel through the dedicated
|
||||
// HTTP transfer endpoint rather than through JSON-RPC.
|
||||
type FileOperation struct {
|
||||
UUID string `json:"uuid"`
|
||||
RequestID string `json:"request_id"`
|
||||
Op string `json:"op"`
|
||||
Args map[string]any `json:"args,omitempty"`
|
||||
}
|
||||
|
||||
type FileResult struct {
|
||||
UUID string `json:"uuid"`
|
||||
RequestID string `json:"request_id"`
|
||||
OK bool `json:"ok"`
|
||||
Result json.RawMessage `json:"result,omitempty"`
|
||||
Error string `json:"error,omitempty"`
|
||||
}
|
||||
|
||||
func Success(id any, result any) Response {
|
||||
return Response{JSONRPC: Version, ID: id, Result: result}
|
||||
}
|
||||
|
||||
@@ -1,15 +0,0 @@
|
||||
package messageSender
|
||||
|
||||
import (
|
||||
_ "github.com/komari-monitor/komari/utils/messageSender/bark"
|
||||
_ "github.com/komari-monitor/komari/utils/messageSender/email"
|
||||
_ "github.com/komari-monitor/komari/utils/messageSender/empty"
|
||||
_ "github.com/komari-monitor/komari/utils/messageSender/javascript"
|
||||
_ "github.com/komari-monitor/komari/utils/messageSender/serverchan3"
|
||||
_ "github.com/komari-monitor/komari/utils/messageSender/serverchanturbo"
|
||||
_ "github.com/komari-monitor/komari/utils/messageSender/telegram"
|
||||
_ "github.com/komari-monitor/komari/utils/messageSender/webhook"
|
||||
)
|
||||
|
||||
func All() {
|
||||
}
|
||||
@@ -1,118 +0,0 @@
|
||||
package bark
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strings"
|
||||
|
||||
"github.com/komari-monitor/komari/utils/messageSender/factory"
|
||||
)
|
||||
|
||||
type BarkSender struct {
|
||||
Addition
|
||||
}
|
||||
|
||||
func (b *BarkSender) GetName() string {
|
||||
return "bark"
|
||||
}
|
||||
|
||||
func (b *BarkSender) GetConfiguration() factory.Configuration {
|
||||
return &b.Addition
|
||||
}
|
||||
|
||||
func (b *BarkSender) Init() error {
|
||||
// 验证 ServerURL 格式
|
||||
if b.Addition.ServerURL != "" {
|
||||
if _, err := url.Parse(b.Addition.ServerURL); err != nil {
|
||||
return fmt.Errorf("invalid server URL: %v", err)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (b *BarkSender) Destroy() error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (b *BarkSender) SendTextMessage(message, title string) error {
|
||||
if b.Addition.DeviceKey == "" {
|
||||
return fmt.Errorf("device key is required")
|
||||
}
|
||||
|
||||
if message == "" {
|
||||
return fmt.Errorf("message is empty")
|
||||
}
|
||||
|
||||
// 准备请求数据
|
||||
payload := map[string]interface{}{
|
||||
"body": message,
|
||||
"device_key": b.Addition.DeviceKey,
|
||||
}
|
||||
|
||||
// 如果有标题,添加标题
|
||||
if title != "" {
|
||||
payload["title"] = title
|
||||
}
|
||||
|
||||
// 添加可选参数
|
||||
if b.Addition.Icon != "" {
|
||||
payload["icon"] = b.Addition.Icon
|
||||
}
|
||||
|
||||
if b.Addition.Level != "" {
|
||||
payload["level"] = b.Addition.Level
|
||||
}
|
||||
|
||||
// 序列化 JSON
|
||||
jsonData, err := json.Marshal(payload)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to marshal payload: %v", err)
|
||||
}
|
||||
|
||||
// 构建请求 URL
|
||||
serverURL := b.Addition.ServerURL
|
||||
if serverURL == "" {
|
||||
serverURL = "https://api.day.app"
|
||||
}
|
||||
|
||||
// 确保 URL 以正确的格式结尾
|
||||
serverURL = strings.TrimRight(serverURL, "/")
|
||||
requestURL := fmt.Sprintf("%s/push", serverURL)
|
||||
|
||||
// 发送 HTTP POST 请求
|
||||
resp, err := http.Post(requestURL, "application/json", bytes.NewBuffer(jsonData))
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to send request: %v", err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
// 检查响应状态
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return fmt.Errorf("bark API returned non-OK status: %d", resp.StatusCode)
|
||||
}
|
||||
|
||||
// 解析响应
|
||||
var result struct {
|
||||
Code int `json:"code"`
|
||||
Message string `json:"message"`
|
||||
Data interface{} `json:"data"`
|
||||
}
|
||||
|
||||
if err := json.NewDecoder(resp.Body).Decode(&result); err != nil {
|
||||
// 如果解析失败,认为发送成功(某些 Bark 服务器可能返回纯文本)
|
||||
return nil
|
||||
}
|
||||
|
||||
// 检查 Bark API 响应
|
||||
if result.Code != 200 {
|
||||
return fmt.Errorf("bark API error (code: %d): %s", result.Code, result.Message)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// 确保实现了 IMessageSender 接口
|
||||
var _ factory.IMessageSender = (*BarkSender)(nil)
|
||||
@@ -1,18 +0,0 @@
|
||||
package bark
|
||||
|
||||
import (
|
||||
"github.com/komari-monitor/komari/utils/messageSender/factory"
|
||||
)
|
||||
|
||||
type Addition struct {
|
||||
ServerURL string `json:"server_url" required:"true" default:"https://api.day.app" help:"Bark server URL, e.g., https://api.day.app or your self-hosted server address"`
|
||||
DeviceKey string `json:"device_key" required:"true" help:"Your Bark device key, which can be found in the Bark App"`
|
||||
Icon string `json:"icon" help:"Push notification icon, supports URL or system icon name"`
|
||||
Level string `json:"level" type:"option" default:"timeSensitive" options:"active,timeSensitive,passive,critical" help:"Push notification level: active, timeSensitive (default), passive, critical"`
|
||||
}
|
||||
|
||||
func init() {
|
||||
factory.RegisterMessageSender(func() factory.IMessageSender {
|
||||
return &BarkSender{}
|
||||
})
|
||||
}
|
||||
@@ -1,289 +0,0 @@
|
||||
package email
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"crypto/tls"
|
||||
"fmt"
|
||||
"mime"
|
||||
"mime/quotedprintable"
|
||||
"net/mail"
|
||||
"net/smtp"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/komari-monitor/komari/utils/messageSender/factory"
|
||||
)
|
||||
|
||||
type EmailSender struct {
|
||||
Addition
|
||||
}
|
||||
|
||||
// loginAuth 是一个更宽松的 SMTP 认证实现,支持更多 SMTP 服务器
|
||||
// 它不会严格验证服务器主机名,从而兼容微软邮箱、网易邮箱等服务
|
||||
type loginAuth struct {
|
||||
username string
|
||||
password string
|
||||
host string
|
||||
}
|
||||
|
||||
func (a *loginAuth) Start(server *smtp.ServerInfo) (string, []byte, error) {
|
||||
return "LOGIN", []byte{}, nil
|
||||
}
|
||||
|
||||
func (a *loginAuth) Next(fromServer []byte, more bool) ([]byte, error) {
|
||||
if more {
|
||||
switch string(fromServer) {
|
||||
case "Username:", "User Name":
|
||||
return []byte(a.username), nil
|
||||
case "Password:", "password:":
|
||||
return []byte(a.password), nil
|
||||
default:
|
||||
// 某些服务器可能发送 base64 编码的提示
|
||||
// 尝试返回用户名或密码
|
||||
prompt := strings.ToLower(strings.TrimSpace(string(fromServer)))
|
||||
if strings.Contains(prompt, "user") {
|
||||
return []byte(a.username), nil
|
||||
}
|
||||
return []byte(a.password), nil
|
||||
}
|
||||
}
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
// plainAuthWithoutCheck 是一个不检查主机名的 PlainAuth 实现
|
||||
// 用于解决某些 SMTP 服务器主机名与配置不匹配的问题
|
||||
type plainAuthWithoutCheck struct {
|
||||
identity string
|
||||
username string
|
||||
password string
|
||||
host string
|
||||
}
|
||||
|
||||
func (a *plainAuthWithoutCheck) Start(server *smtp.ServerInfo) (string, []byte, error) {
|
||||
resp := []byte(a.identity + "\x00" + a.username + "\x00" + a.password)
|
||||
return "PLAIN", resp, nil
|
||||
}
|
||||
|
||||
func (a *plainAuthWithoutCheck) Next(fromServer []byte, more bool) ([]byte, error) {
|
||||
if more {
|
||||
return nil, fmt.Errorf("unexpected server challenge")
|
||||
}
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
func (e *EmailSender) GetName() string {
|
||||
return "email"
|
||||
}
|
||||
|
||||
func (e *EmailSender) GetConfiguration() factory.Configuration {
|
||||
return &e.Addition
|
||||
}
|
||||
|
||||
func (e *EmailSender) Init() error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (e *EmailSender) Destroy() error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (e *EmailSender) SendTextMessage(message, title string) error {
|
||||
|
||||
if e.Addition.Host == "" || e.Addition.Sender == "" || e.Addition.Username == "" || e.Addition.Password == "" || e.Addition.Receiver == "" {
|
||||
return fmt.Errorf("email sending is not fully configured")
|
||||
}
|
||||
|
||||
// 使用更宽松的认证方式,优先尝试 PLAIN,如果失败则尝试 LOGIN
|
||||
// 这样可以兼容更多的 SMTP 服务器,包括微软邮箱、网易邮箱等
|
||||
var auth smtp.Auth
|
||||
if e.Addition.UseLoginAuth {
|
||||
// 使用 LOGIN 认证(适用于某些旧的或特殊的 SMTP 服务器)
|
||||
auth = &loginAuth{
|
||||
username: e.Addition.Username,
|
||||
password: e.Addition.Password,
|
||||
host: e.Addition.Host,
|
||||
}
|
||||
} else {
|
||||
// 使用不检查主机名的 PLAIN 认证(适用于大多数现代 SMTP 服务器)
|
||||
auth = &plainAuthWithoutCheck{
|
||||
identity: "",
|
||||
username: e.Addition.Username,
|
||||
password: e.Addition.Password,
|
||||
host: e.Addition.Host,
|
||||
}
|
||||
}
|
||||
|
||||
// Parse sender address (for MAIL FROM and header)
|
||||
var senderAddr string
|
||||
var senderHeader string
|
||||
if addr, err := mail.ParseAddress(e.Addition.Sender); err == nil {
|
||||
senderAddr = addr.Address
|
||||
senderHeader = addr.String()
|
||||
} else {
|
||||
// Fallback: use raw string
|
||||
senderAddr = e.Addition.Sender
|
||||
senderHeader = e.Addition.Sender
|
||||
}
|
||||
|
||||
// Parse recipients (support comma-separated list)
|
||||
var rcptList []string
|
||||
var rcptHeaderParts []string
|
||||
if addrs, err := mail.ParseAddressList(e.Addition.Receiver); err == nil {
|
||||
for _, a := range addrs {
|
||||
rcptList = append(rcptList, a.Address)
|
||||
rcptHeaderParts = append(rcptHeaderParts, a.String())
|
||||
}
|
||||
} else {
|
||||
// Fallback simple split
|
||||
parts := strings.Split(e.Addition.Receiver, ",")
|
||||
for _, p := range parts {
|
||||
p = strings.TrimSpace(p)
|
||||
if p == "" {
|
||||
continue
|
||||
}
|
||||
if a, err := mail.ParseAddress(p); err == nil {
|
||||
rcptList = append(rcptList, a.Address)
|
||||
rcptHeaderParts = append(rcptHeaderParts, a.String())
|
||||
} else {
|
||||
rcptList = append(rcptList, p)
|
||||
rcptHeaderParts = append(rcptHeaderParts, p)
|
||||
}
|
||||
}
|
||||
}
|
||||
if len(rcptList) == 0 {
|
||||
return fmt.Errorf("no valid recipient address parsed")
|
||||
}
|
||||
|
||||
// RFC 2047 encode subject if non-ASCII
|
||||
encodedSubject := mime.QEncoding.Encode("UTF-8", title)
|
||||
|
||||
// Encode body as quoted-printable to be safe with UTF-8 on servers lacking 8BITMIME
|
||||
var bodyBuf bytes.Buffer
|
||||
qp := quotedprintable.NewWriter(&bodyBuf)
|
||||
if _, err := qp.Write([]byte(message)); err != nil {
|
||||
return fmt.Errorf("failed to encode body: %w", err)
|
||||
}
|
||||
if err := qp.Close(); err != nil {
|
||||
return fmt.Errorf("failed to finalize body encoding: %w", err)
|
||||
}
|
||||
|
||||
contentType := "text/plain; charset=UTF-8"
|
||||
// 检测模板是否包含HTML
|
||||
trimmedMsg := strings.TrimSpace(message)
|
||||
if strings.Contains(strings.ToLower(trimmedMsg), "<html") ||
|
||||
strings.Contains(strings.ToLower(trimmedMsg), "<!doctype") ||
|
||||
(strings.Contains(trimmedMsg, "<div") && strings.Contains(trimmedMsg, "</div>")) {
|
||||
contentType = "text/html; charset=UTF-8"
|
||||
}
|
||||
|
||||
// Compose headers
|
||||
now := time.Now().UTC()
|
||||
headers := []string{
|
||||
"To: " + strings.Join(rcptHeaderParts, ", "),
|
||||
"From: " + senderHeader,
|
||||
"Subject: " + encodedSubject,
|
||||
"MIME-Version: 1.0",
|
||||
"Content-Type: " + contentType,
|
||||
"Content-Transfer-Encoding: quoted-printable",
|
||||
"Date: " + now.Format(time.RFC1123Z),
|
||||
fmt.Sprintf("Message-ID: <%d@%s>", now.UnixNano(), e.Addition.Host),
|
||||
}
|
||||
|
||||
fullMsg := []byte(strings.Join(headers, "\r\n") + "\r\n\r\n" + bodyBuf.String())
|
||||
|
||||
addr := e.Addition.Host + ":" + strconv.Itoa(e.Addition.Port)
|
||||
|
||||
if e.Addition.UseSSL {
|
||||
// Use TLS. If port is 465, prefer implicit TLS. Otherwise, use STARTTLS.
|
||||
if e.Addition.Port == 465 {
|
||||
// Implicit TLS (SMTPS)
|
||||
tlsCfg := &tls.Config{ServerName: e.Addition.Host}
|
||||
conn, err := tls.Dial("tcp", addr, tlsCfg)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to establish implicit TLS connection: %w", err)
|
||||
}
|
||||
defer conn.Close()
|
||||
|
||||
c, err := smtp.NewClient(conn, e.Addition.Host)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to create SMTP client over TLS: %w", err)
|
||||
}
|
||||
defer c.Close()
|
||||
|
||||
if err = c.Auth(auth); err != nil {
|
||||
return fmt.Errorf("failed to authenticate: %w", err)
|
||||
}
|
||||
|
||||
if err = c.Mail(senderAddr); err != nil {
|
||||
return fmt.Errorf("failed to set sender: %w", err)
|
||||
}
|
||||
for _, rcpt := range rcptList {
|
||||
if err = c.Rcpt(rcpt); err != nil {
|
||||
return fmt.Errorf("failed to add recipient %s: %w", rcpt, err)
|
||||
}
|
||||
}
|
||||
|
||||
w, err := c.Data()
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to get data writer: %w", err)
|
||||
}
|
||||
if _, err = w.Write(fullMsg); err != nil {
|
||||
return fmt.Errorf("failed to write message: %w", err)
|
||||
}
|
||||
if err = w.Close(); err != nil {
|
||||
return fmt.Errorf("failed to close data writer: %w", err)
|
||||
}
|
||||
return c.Quit()
|
||||
} else {
|
||||
// STARTTLS
|
||||
c, err := smtp.Dial(addr)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to dial SMTP server: %w", err)
|
||||
}
|
||||
defer c.Close()
|
||||
|
||||
if err = c.StartTLS(&tls.Config{ServerName: e.Addition.Host}); err != nil {
|
||||
return fmt.Errorf("failed to start TLS: %w", err)
|
||||
}
|
||||
|
||||
if err = c.Auth(auth); err != nil {
|
||||
return fmt.Errorf("failed to authenticate: %w", err)
|
||||
}
|
||||
|
||||
if err = c.Mail(senderAddr); err != nil {
|
||||
return fmt.Errorf("failed to set sender: %w", err)
|
||||
}
|
||||
for _, rcpt := range rcptList {
|
||||
if err = c.Rcpt(rcpt); err != nil {
|
||||
return fmt.Errorf("failed to add recipient %s: %w", rcpt, err)
|
||||
}
|
||||
}
|
||||
|
||||
w, err := c.Data()
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to get data writer: %w", err)
|
||||
}
|
||||
if _, err = w.Write(fullMsg); err != nil {
|
||||
return fmt.Errorf("failed to write message: %w", err)
|
||||
}
|
||||
if err = w.Close(); err != nil {
|
||||
return fmt.Errorf("failed to close data writer: %w", err)
|
||||
}
|
||||
|
||||
return c.Quit()
|
||||
}
|
||||
} else {
|
||||
// Send without SSL/TLS (less secure). We still reuse the composed message and parsed addresses.
|
||||
return smtp.SendMail(
|
||||
addr,
|
||||
auth,
|
||||
senderAddr,
|
||||
rcptList,
|
||||
fullMsg,
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
// 确保实现了 IMessageSender 接口
|
||||
var _ factory.IMessageSender = (*EmailSender)(nil)
|
||||
@@ -1,22 +0,0 @@
|
||||
package email
|
||||
|
||||
import (
|
||||
"github.com/komari-monitor/komari/utils/messageSender/factory"
|
||||
)
|
||||
|
||||
type Addition struct {
|
||||
Host string `json:"host" required:"true"`
|
||||
Port int `json:"port" required:"true" default:"587"`
|
||||
Username string `json:"username"`
|
||||
Password string `json:"password"`
|
||||
Sender string `json:"sender"`
|
||||
Receiver string `json:"receiver"`
|
||||
UseSSL bool `json:"use_ssl" default:"false"`
|
||||
UseLoginAuth bool `json:"use_login_auth" default:"false" help:"Use LOGIN authentication method instead of PLAIN. Enable this if you encounter authentication errors with Microsoft (Outlook/Office365), NetEase (163.com), or other email providers"`
|
||||
}
|
||||
|
||||
func init() {
|
||||
factory.RegisterMessageSender(func() factory.IMessageSender {
|
||||
return &EmailSender{}
|
||||
})
|
||||
}
|
||||
@@ -1,43 +0,0 @@
|
||||
package empty
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
|
||||
"github.com/komari-monitor/komari/utils/messageSender/factory"
|
||||
)
|
||||
|
||||
type Addition struct {
|
||||
}
|
||||
|
||||
type EmptyProvider struct {
|
||||
Addition
|
||||
}
|
||||
|
||||
func (e *EmptyProvider) GetName() string {
|
||||
return "empty"
|
||||
}
|
||||
|
||||
func (e *EmptyProvider) GetConfiguration() factory.Configuration {
|
||||
return &e.Addition
|
||||
}
|
||||
|
||||
func (e *EmptyProvider) Init() error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (e *EmptyProvider) Destroy() error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (e *EmptyProvider) SendTextMessage(message, title string) error {
|
||||
return fmt.Errorf("you are using an empty message sender, please check your configuration")
|
||||
}
|
||||
|
||||
func init() {
|
||||
factory.RegisterMessageSender(func() factory.IMessageSender {
|
||||
return &EmptyProvider{}
|
||||
})
|
||||
}
|
||||
|
||||
// 确保实现了 IMessageSender 接口
|
||||
var _ factory.IMessageSender = (*EmptyProvider)(nil)
|
||||
@@ -1,60 +0,0 @@
|
||||
package factory
|
||||
|
||||
import (
|
||||
logger "github.com/komari-monitor/komari/utils/log"
|
||||
|
||||
"github.com/komari-monitor/komari/utils/item"
|
||||
)
|
||||
|
||||
var (
|
||||
senders = make(map[string]IMessageSender)
|
||||
senderConstructor = make(map[string]MessageSenderConstructor)
|
||||
sendersAdditionalItems = make(map[string][]item.Item)
|
||||
)
|
||||
|
||||
func RegisterMessageSender(constructor MessageSenderConstructor) {
|
||||
sender := constructor()
|
||||
senderConstructor[sender.GetName()] = constructor
|
||||
if sender == nil {
|
||||
panic("Message sender constructor returned nil")
|
||||
}
|
||||
if _, exists := senders[sender.GetName()]; exists {
|
||||
logger.InfoArgs("message-sender", "Message sender already registered: "+sender.GetName())
|
||||
}
|
||||
senders[sender.GetName()] = sender
|
||||
|
||||
// 使用反射来提取提供程序的配置字段
|
||||
config := sender.GetConfiguration()
|
||||
items := item.Parse(config)
|
||||
|
||||
sendersAdditionalItems[sender.GetName()] = items
|
||||
}
|
||||
|
||||
func GetSenderConfigs() map[string][]item.Item {
|
||||
return sendersAdditionalItems
|
||||
}
|
||||
|
||||
func GetAllMessageSenders() map[string]IMessageSender {
|
||||
return senders
|
||||
}
|
||||
|
||||
func GetConstructor(name string) (MessageSenderConstructor, bool) {
|
||||
constructor, exists := senderConstructor[name]
|
||||
return constructor, exists
|
||||
}
|
||||
|
||||
func GetAllMessageSenderNames() []string {
|
||||
names := make([]string, 0, len(senders))
|
||||
for name := range senders {
|
||||
names = append(names, name)
|
||||
}
|
||||
return names
|
||||
}
|
||||
|
||||
func Initialize() {
|
||||
for _, sender := range senders {
|
||||
if err := sender.Init(); err != nil {
|
||||
logger.Errorf("message-sender", "Failed to initialize message sender %s: %v", sender.GetName(), err)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,21 +0,0 @@
|
||||
package factory
|
||||
|
||||
import "github.com/komari-monitor/komari/database/models"
|
||||
|
||||
type IMessageSender interface {
|
||||
GetName() string
|
||||
// 请务必返回 &Configuration{} 的指针
|
||||
GetConfiguration() Configuration
|
||||
SendTextMessage(message, title string) error
|
||||
Init() error
|
||||
Destroy() error
|
||||
}
|
||||
|
||||
// IEventMessageSender 是可选接口,如果实现则可以接收结构化的事件消息
|
||||
type IEventMessageSender interface {
|
||||
SendEvent(event models.EventMessage) error
|
||||
}
|
||||
|
||||
type Configuration interface{}
|
||||
|
||||
type MessageSenderConstructor func() IMessageSender
|
||||
@@ -1,117 +0,0 @@
|
||||
package javascript
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"github.com/komari-monitor/komari/database/models"
|
||||
"github.com/komari-monitor/komari/pkg/jsruntime"
|
||||
"github.com/komari-monitor/komari/utils/messageSender/factory"
|
||||
)
|
||||
|
||||
type JavaScriptSender struct {
|
||||
Addition
|
||||
runtime *jsruntime.Runtime
|
||||
}
|
||||
|
||||
func (j *JavaScriptSender) GetName() string {
|
||||
return "Javascript"
|
||||
}
|
||||
|
||||
func (j *JavaScriptSender) GetConfiguration() factory.Configuration {
|
||||
return &j.Addition
|
||||
}
|
||||
|
||||
func (j *JavaScriptSender) Init() error {
|
||||
runtime, err := jsruntime.New(j.Addition.Script, jsruntime.Options{})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !runtime.HasFunction("sendMessage") {
|
||||
runtime.Close()
|
||||
return errors.New("sendMessage function not defined or not callable in script")
|
||||
}
|
||||
if j.runtime != nil {
|
||||
j.runtime.Close()
|
||||
}
|
||||
j.runtime = runtime
|
||||
return nil
|
||||
}
|
||||
|
||||
func (j *JavaScriptSender) Destroy() error {
|
||||
if j.runtime != nil {
|
||||
j.runtime.Close()
|
||||
j.runtime = nil
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (j *JavaScriptSender) SendTextMessage(message, title string) error {
|
||||
if err := j.ensureRuntime(); err != nil {
|
||||
return err
|
||||
}
|
||||
return j.runtime.Call("sendMessage", message, title)
|
||||
}
|
||||
|
||||
func (j *JavaScriptSender) SendEvent(event models.EventMessage) error {
|
||||
if err := j.ensureRuntime(); err != nil {
|
||||
return err
|
||||
}
|
||||
if !j.runtime.HasFunction("sendEvent") {
|
||||
return j.fallbackToTextMessage(event)
|
||||
}
|
||||
|
||||
eventJSON, err := json.Marshal(event)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to marshal event: %v", err)
|
||||
}
|
||||
var eventMap map[string]interface{}
|
||||
if err := json.Unmarshal(eventJSON, &eventMap); err != nil {
|
||||
return fmt.Errorf("failed to unmarshal event: %v", err)
|
||||
}
|
||||
return j.runtime.Call("sendEvent", eventMap)
|
||||
}
|
||||
|
||||
func (j *JavaScriptSender) ensureRuntime() error {
|
||||
if j.runtime != nil {
|
||||
return nil
|
||||
}
|
||||
return j.Init()
|
||||
}
|
||||
|
||||
// fallbackToTextMessage formats structured events for scripts that only
|
||||
// implement sendMessage.
|
||||
func (j *JavaScriptSender) fallbackToTextMessage(event models.EventMessage) error {
|
||||
message := fmt.Sprintf("%v%v%v\nEvent: %v\nMessage: %v\nTime: %s",
|
||||
event.Emoji, event.Emoji, event.Emoji,
|
||||
event.Event,
|
||||
event.Message,
|
||||
event.Time.UTC().Format(time.RFC3339Nano))
|
||||
if len(event.Clients) > 0 {
|
||||
clientNames := make([]string, 0, len(event.Clients))
|
||||
for _, client := range event.Clients {
|
||||
name := client.Name
|
||||
if name == "" {
|
||||
name = client.UUID
|
||||
}
|
||||
clientNames = append(clientNames, name)
|
||||
}
|
||||
message = fmt.Sprintf("%v%v%v\nEvent: %v\nClients: %s\nMessage: %v\nTime: %s",
|
||||
event.Emoji, event.Emoji, event.Emoji,
|
||||
event.Event,
|
||||
clientNames,
|
||||
event.Message,
|
||||
event.Time.UTC().Format(time.RFC3339Nano))
|
||||
}
|
||||
return j.SendTextMessage(message, fmt.Sprint(event.Event))
|
||||
}
|
||||
|
||||
func init() {
|
||||
factory.RegisterMessageSender(func() factory.IMessageSender {
|
||||
return &JavaScriptSender{}
|
||||
})
|
||||
}
|
||||
|
||||
var _ factory.IMessageSender = (*JavaScriptSender)(nil)
|
||||
@@ -1,5 +0,0 @@
|
||||
package javascript
|
||||
|
||||
type Addition struct {
|
||||
Script string `json:"script" required:"true" type:"richtext" help:"JavaScript code (ES6 supported) that implements sendMessage(message, title) and optionally sendEvent(event) functions. Both should return a Promise or boolean. Available APIs: fetch(), xhr(), console.log()."`
|
||||
}
|
||||
@@ -1,29 +0,0 @@
|
||||
package messageSender
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
|
||||
"github.com/komari-monitor/komari/utils/messageSender/factory"
|
||||
)
|
||||
|
||||
func LoadProvider(name string, addition string) error {
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
constructor, exists := factory.GetConstructor(name)
|
||||
if !exists {
|
||||
return fmt.Errorf("message sender provider not found: %s", name)
|
||||
}
|
||||
|
||||
provider := constructor()
|
||||
err := json.Unmarshal([]byte(addition), provider.GetConfiguration())
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to load config for provider %s: %w", name, err)
|
||||
}
|
||||
provider.Init()
|
||||
if currentProvider != nil {
|
||||
currentProvider.Destroy()
|
||||
}
|
||||
currentProvider = provider
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,19 @@
|
||||
package messageSender
|
||||
|
||||
import (
|
||||
"os"
|
||||
"testing"
|
||||
|
||||
"github.com/komari-monitor/komari/cmd/flags"
|
||||
"github.com/komari-monitor/komari/database/dbcore"
|
||||
)
|
||||
|
||||
func TestMain(m *testing.M) {
|
||||
flags.DatabaseType = flags.DatabaseTypeSQLite
|
||||
flags.DatabaseFile = "file:komari_message_sender_test?mode=memory&cache=shared"
|
||||
db := dbcore.GetDBInstance()
|
||||
if sqlDB, err := db.DB(); err == nil {
|
||||
sqlDB.SetMaxOpenConns(1)
|
||||
}
|
||||
os.Exit(m.Run())
|
||||
}
|
||||
@@ -0,0 +1,289 @@
|
||||
package messageSender
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"reflect"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
|
||||
"github.com/komari-monitor/komari/database"
|
||||
"github.com/komari-monitor/komari/database/models"
|
||||
"github.com/komari-monitor/komari/internal/managedconfig"
|
||||
"gorm.io/gorm"
|
||||
)
|
||||
|
||||
// Notification is the message passed to a notification channel.
|
||||
type Notification struct {
|
||||
Event models.EventMessage `json:"event"`
|
||||
Title string `json:"title"`
|
||||
Message string `json:"message"`
|
||||
}
|
||||
|
||||
// NotificationChannel is implemented by built-in and plugin notification
|
||||
// channels.
|
||||
type NotificationChannel interface {
|
||||
Send(context.Context, Notification, map[string]any) error
|
||||
Unload() error
|
||||
}
|
||||
|
||||
// NotificationChannelInfo describes a registered notification channel.
|
||||
type NotificationChannelInfo struct {
|
||||
ID string `json:"id"`
|
||||
Configuration models.Configuration `json:"configuration"`
|
||||
}
|
||||
|
||||
type notificationChannelEntry struct {
|
||||
configuration models.Configuration
|
||||
channel NotificationChannel
|
||||
}
|
||||
|
||||
var notificationChannels = struct {
|
||||
sync.RWMutex
|
||||
entries map[string]*notificationChannelEntry
|
||||
}{
|
||||
entries: make(map[string]*notificationChannelEntry),
|
||||
}
|
||||
|
||||
// RegisterNotificationChannel registers a channel. Registration fails for an
|
||||
// empty or already-used ID.
|
||||
func RegisterNotificationChannel(id string, configuration models.Configuration, notificationChannel NotificationChannel) error {
|
||||
return registerNotificationChannel(id, configuration, notificationChannel, nil)
|
||||
}
|
||||
|
||||
func registerNotificationChannel(
|
||||
id string,
|
||||
configuration models.Configuration,
|
||||
notificationChannel NotificationChannel,
|
||||
afterRegister func() error,
|
||||
) error {
|
||||
id = strings.TrimSpace(id)
|
||||
if id == "" {
|
||||
return errors.New("notification channel id is required")
|
||||
}
|
||||
if notificationChannel == nil {
|
||||
return fmt.Errorf("notification channel %q is nil", id)
|
||||
}
|
||||
if configuration.Type == "" {
|
||||
configuration.Type = models.ThemeConfigurationManaged
|
||||
}
|
||||
|
||||
notificationChannels.Lock()
|
||||
defer notificationChannels.Unlock()
|
||||
if _, exists := notificationChannels.entries[id]; exists {
|
||||
return fmt.Errorf("notification channel %q is already registered", id)
|
||||
}
|
||||
notificationChannels.entries[id] = ¬ificationChannelEntry{
|
||||
configuration: configuration,
|
||||
channel: notificationChannel,
|
||||
}
|
||||
if afterRegister != nil {
|
||||
if err := afterRegister(); err != nil {
|
||||
delete(notificationChannels.entries, id)
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// UnregisterNotificationChannel removes a channel after all calls using it
|
||||
// have completed, then unloads it.
|
||||
func UnregisterNotificationChannel(id string) error {
|
||||
id = strings.TrimSpace(id)
|
||||
notificationChannels.Lock()
|
||||
found, exists := notificationChannels.entries[id]
|
||||
if exists {
|
||||
delete(notificationChannels.entries, id)
|
||||
}
|
||||
notificationChannels.Unlock()
|
||||
if !exists {
|
||||
return nil
|
||||
}
|
||||
return found.channel.Unload()
|
||||
}
|
||||
|
||||
// CallNotificationChannel invokes one registered channel with the latest
|
||||
// saved, defaulted, and resolved configuration.
|
||||
func CallNotificationChannel(ctx context.Context, id string, notification Notification) error {
|
||||
id = strings.TrimSpace(id)
|
||||
notificationChannels.RLock()
|
||||
found := notificationChannels.entries[id]
|
||||
if found == nil {
|
||||
notificationChannels.RUnlock()
|
||||
return fmt.Errorf("notification channel %q is not registered", id)
|
||||
}
|
||||
configuration := found.configuration
|
||||
notificationChannel := found.channel
|
||||
defer notificationChannels.RUnlock()
|
||||
|
||||
values, err := resolveNotificationChannelConfiguration(id, configuration)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return notificationChannel.Send(ctx, notification, values)
|
||||
}
|
||||
|
||||
// ListNotificationChannels returns a snapshot sorted by channel ID.
|
||||
func ListNotificationChannels() []NotificationChannelInfo {
|
||||
notificationChannels.RLock()
|
||||
defer notificationChannels.RUnlock()
|
||||
result := make([]NotificationChannelInfo, 0, len(notificationChannels.entries))
|
||||
for id, found := range notificationChannels.entries {
|
||||
result = append(result, NotificationChannelInfo{
|
||||
ID: id,
|
||||
Configuration: found.configuration,
|
||||
})
|
||||
}
|
||||
sort.Slice(result, func(i, j int) bool { return result[i].ID < result[j].ID })
|
||||
return result
|
||||
}
|
||||
|
||||
func NotificationChannelRegistered(id string) bool {
|
||||
return notificationChannelRegistered(id)
|
||||
}
|
||||
|
||||
func GetNotificationChannelConfiguration(id string) (models.Configuration, map[string]any, bool, error) {
|
||||
configuration, exists := notificationChannelConfiguration(id)
|
||||
if !exists {
|
||||
return models.Configuration{}, nil, false, nil
|
||||
}
|
||||
values, err := resolveNotificationChannelConfiguration(id, configuration)
|
||||
if err != nil {
|
||||
return models.Configuration{}, nil, true, err
|
||||
}
|
||||
return configuration, values, true, nil
|
||||
}
|
||||
|
||||
func notificationChannelRegistered(id string) bool {
|
||||
notificationChannels.RLock()
|
||||
defer notificationChannels.RUnlock()
|
||||
return notificationChannels.entries[strings.TrimSpace(id)] != nil
|
||||
}
|
||||
|
||||
func notificationChannelConfiguration(id string) (models.Configuration, bool) {
|
||||
notificationChannels.RLock()
|
||||
defer notificationChannels.RUnlock()
|
||||
found := notificationChannels.entries[strings.TrimSpace(id)]
|
||||
if found == nil {
|
||||
return models.Configuration{}, false
|
||||
}
|
||||
return found.configuration, true
|
||||
}
|
||||
|
||||
func resolveNotificationChannelConfiguration(id string, configuration models.Configuration) (map[string]any, error) {
|
||||
values := map[string]any{}
|
||||
cfg, err := database.GetMessageSenderConfigByName(id)
|
||||
if err != nil && !errors.Is(err, gorm.ErrRecordNotFound) {
|
||||
return nil, fmt.Errorf("read notification channel %q configuration: %w", id, err)
|
||||
}
|
||||
if err == nil {
|
||||
if err := json.Unmarshal([]byte(cfg.Addition), &values); err != nil {
|
||||
return nil, fmt.Errorf("parse notification channel %q configuration: %w", id, err)
|
||||
}
|
||||
}
|
||||
|
||||
items := managedconfig.Items(configuration)
|
||||
for _, item := range items {
|
||||
if item.Key == "" {
|
||||
continue
|
||||
}
|
||||
if _, exists := values[item.Key]; !exists {
|
||||
values[item.Key] = managedconfig.DefaultValue(item)
|
||||
}
|
||||
}
|
||||
if err := managedconfig.ResolveForOutput(values, items); err != nil {
|
||||
return nil, fmt.Errorf("resolve notification channel %q configuration: %w", id, err)
|
||||
}
|
||||
return values, nil
|
||||
}
|
||||
|
||||
// ManagedConfiguration builds a managed configuration declaration from a
|
||||
// built-in channel's tagged configuration struct.
|
||||
func ManagedConfiguration(name any, configuration any) models.Configuration {
|
||||
value := reflect.ValueOf(configuration)
|
||||
if value.Kind() == reflect.Pointer {
|
||||
value = value.Elem()
|
||||
}
|
||||
items := make([]models.ManagedThemeConfigurationItem, 0, value.NumField())
|
||||
for i := 0; i < value.NumField(); i++ {
|
||||
field := value.Type().Field(i)
|
||||
key := strings.Split(field.Tag.Get("json"), ",")[0]
|
||||
if key == "" || key == "-" {
|
||||
continue
|
||||
}
|
||||
itemType := field.Tag.Get("type")
|
||||
switch {
|
||||
case itemType == "option":
|
||||
itemType = "select"
|
||||
case itemType == "richtext":
|
||||
itemType = "richtext"
|
||||
case field.Type.Kind() == reflect.Bool:
|
||||
itemType = "switch"
|
||||
case field.Type.Kind() >= reflect.Int && field.Type.Kind() <= reflect.Int64:
|
||||
itemType = "number"
|
||||
case field.Type.Kind() >= reflect.Uint && field.Type.Kind() <= reflect.Uint64:
|
||||
itemType = "number"
|
||||
case field.Type.Kind() == reflect.Float32 || field.Type.Kind() == reflect.Float64:
|
||||
itemType = "number"
|
||||
default:
|
||||
itemType = "string"
|
||||
}
|
||||
items = append(items, models.ManagedThemeConfigurationItem{
|
||||
Key: key,
|
||||
Name: field.Tag.Get("name"),
|
||||
Required: field.Tag.Get("required") == "true",
|
||||
Type: itemType,
|
||||
Options: field.Tag.Get("options"),
|
||||
Default: parseConfigurationDefault(field, field.Tag.Get("default")),
|
||||
Help: field.Tag.Get("help"),
|
||||
})
|
||||
}
|
||||
return models.Configuration{
|
||||
Type: models.ThemeConfigurationManaged,
|
||||
Name: name,
|
||||
Data: items,
|
||||
}
|
||||
}
|
||||
|
||||
// DecodeConfiguration decodes resolved configuration values into a built-in
|
||||
// channel's typed configuration struct.
|
||||
func DecodeConfiguration(values map[string]any, target any) error {
|
||||
raw, err := json.Marshal(values)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return json.Unmarshal(raw, target)
|
||||
}
|
||||
|
||||
func parseConfigurationDefault(field reflect.StructField, raw string) any {
|
||||
if raw == "" {
|
||||
return nil
|
||||
}
|
||||
switch field.Type.Kind() {
|
||||
case reflect.Bool:
|
||||
value, err := strconv.ParseBool(raw)
|
||||
if err == nil {
|
||||
return value
|
||||
}
|
||||
case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
|
||||
value, err := strconv.ParseInt(raw, 10, field.Type.Bits())
|
||||
if err == nil {
|
||||
return value
|
||||
}
|
||||
case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64:
|
||||
value, err := strconv.ParseUint(raw, 10, field.Type.Bits())
|
||||
if err == nil {
|
||||
return value
|
||||
}
|
||||
case reflect.Float32, reflect.Float64:
|
||||
value, err := strconv.ParseFloat(raw, field.Type.Bits())
|
||||
if err == nil {
|
||||
return value
|
||||
}
|
||||
}
|
||||
return raw
|
||||
}
|
||||
@@ -0,0 +1,243 @@
|
||||
package messageSender
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/komari-monitor/komari/database/dbcore"
|
||||
"github.com/komari-monitor/komari/database/models"
|
||||
)
|
||||
|
||||
type testChannel struct {
|
||||
mu sync.Mutex
|
||||
sends int
|
||||
config map[string]any
|
||||
unloaded bool
|
||||
err error
|
||||
}
|
||||
|
||||
func (c *testChannel) Send(_ context.Context, _ Notification, config map[string]any) error {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
c.sends++
|
||||
c.config = config
|
||||
return c.err
|
||||
}
|
||||
|
||||
func (c *testChannel) Unload() error {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
c.unloaded = true
|
||||
return nil
|
||||
}
|
||||
|
||||
func TestRegistryRegisterListCallAndUnregister(t *testing.T) {
|
||||
clearNotificationChannels()
|
||||
t.Cleanup(clearNotificationChannels)
|
||||
|
||||
config := models.Configuration{
|
||||
Type: models.ThemeConfigurationManaged,
|
||||
Name: "Test",
|
||||
Data: []models.ManagedThemeConfigurationItem{
|
||||
{Key: "endpoint", Type: "string", Default: "default-endpoint"},
|
||||
},
|
||||
}
|
||||
sender := &testChannel{}
|
||||
if err := RegisterNotificationChannel("test", config, sender); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := RegisterNotificationChannel("", config, sender); err == nil {
|
||||
t.Fatal("empty id was accepted")
|
||||
}
|
||||
if err := RegisterNotificationChannel("test", config, sender); err == nil {
|
||||
t.Fatal("duplicate id was accepted")
|
||||
}
|
||||
list := ListNotificationChannels()
|
||||
if len(list) != 1 || list[0].ID != "test" || list[0].Configuration.Name != "Test" {
|
||||
t.Fatalf("ListNotificationChannels() = %#v", list)
|
||||
}
|
||||
if err := CallNotificationChannel(context.Background(), "test", Notification{Title: "hello"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if sender.config["endpoint"] != "default-endpoint" {
|
||||
t.Fatalf("config = %#v", sender.config)
|
||||
}
|
||||
if err := UnregisterNotificationChannel("test"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !sender.unloaded {
|
||||
t.Fatal("Unload was not called")
|
||||
}
|
||||
if err := CallNotificationChannel(context.Background(), "test", Notification{}); err == nil {
|
||||
t.Fatal("unregistered channel was callable")
|
||||
}
|
||||
}
|
||||
|
||||
func TestRegistryUnregisterWaitsForInFlightCall(t *testing.T) {
|
||||
clearNotificationChannels()
|
||||
t.Cleanup(clearNotificationChannels)
|
||||
|
||||
blocking := &blockingChannel{started: make(chan struct{}), release: make(chan struct{})}
|
||||
if err := RegisterNotificationChannel("blocking", models.Configuration{Type: "managed"}, blocking); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
callDone := make(chan error, 1)
|
||||
go func() { callDone <- CallNotificationChannel(context.Background(), "blocking", Notification{}) }()
|
||||
<-blocking.started
|
||||
|
||||
unloadDone := make(chan error, 1)
|
||||
go func() { unloadDone <- UnregisterNotificationChannel("blocking") }()
|
||||
select {
|
||||
case <-unloadDone:
|
||||
t.Fatal("UnregisterNotificationChannel returned before the in-flight call finished")
|
||||
case <-time.After(20 * time.Millisecond):
|
||||
}
|
||||
close(blocking.release)
|
||||
if err := <-callDone; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := <-unloadDone; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
type blockingChannel struct {
|
||||
started chan struct{}
|
||||
release chan struct{}
|
||||
}
|
||||
|
||||
func (c *blockingChannel) Send(context.Context, Notification, map[string]any) error {
|
||||
close(c.started)
|
||||
<-c.release
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *blockingChannel) Unload() error { return nil }
|
||||
|
||||
func TestRegistryUsesSavedConfigurationAndResolvesSelectors(t *testing.T) {
|
||||
clearNotificationChannels()
|
||||
t.Cleanup(clearNotificationChannels)
|
||||
db := dbcore.GetDBInstance()
|
||||
if err := db.AutoMigrate(&models.MessageSenderProvider{}, &models.Client{}, &models.PingTask{}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
const nodeID = "notification-registry-node"
|
||||
const taskID = uint(43)
|
||||
if err := db.Where("uuid = ?", nodeID).Delete(&models.Client{}).Error; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := db.Create(&models.Client{UUID: nodeID, Token: nodeID, Name: "node"}).Error; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := db.Where("id = ?", taskID).Delete(&models.PingTask{}).Error; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := db.Create(&models.PingTask{Id: taskID, Name: "task"}).Error; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := db.Where("name = ?", "configured").Delete(&models.MessageSenderProvider{}).Error; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := db.Create(&models.MessageSenderProvider{
|
||||
Name: "configured",
|
||||
Addition: `{"nodes":"[\"notification-registry-node\",\"missing\"]","tasks":"[43,7]","token":"saved"}`,
|
||||
}).Error; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
sender := &testChannel{}
|
||||
config := models.Configuration{
|
||||
Type: "managed",
|
||||
Data: []models.ManagedThemeConfigurationItem{
|
||||
{Key: "token", Type: "string", Default: "default"},
|
||||
{Key: "nodes", Type: "nodes"},
|
||||
{Key: "tasks", Type: "pingtasks"},
|
||||
{Key: "missing", Type: "number"},
|
||||
},
|
||||
}
|
||||
if err := RegisterNotificationChannel("configured", config, sender); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := CallNotificationChannel(context.Background(), "configured", Notification{}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if sender.config["token"] != "saved" {
|
||||
t.Fatalf("token = %#v", sender.config["token"])
|
||||
}
|
||||
if sender.config["missing"] != float64(0) {
|
||||
t.Fatalf("missing default = %#v", sender.config["missing"])
|
||||
}
|
||||
nodes, ok := sender.config["nodes"].([]string)
|
||||
if !ok || len(nodes) != 1 || nodes[0] != nodeID {
|
||||
t.Fatalf("nodes = %#v", sender.config["nodes"])
|
||||
}
|
||||
tasks, ok := sender.config["tasks"].([]uint)
|
||||
if !ok || len(tasks) != 1 || tasks[0] != taskID {
|
||||
t.Fatalf("tasks = %#v", sender.config["tasks"])
|
||||
}
|
||||
}
|
||||
|
||||
func TestRegistryUsesLatestSavedConfiguration(t *testing.T) {
|
||||
clearNotificationChannels()
|
||||
t.Cleanup(clearNotificationChannels)
|
||||
db := dbcore.GetDBInstance()
|
||||
if err := db.AutoMigrate(&models.MessageSenderProvider{}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := db.Where("name = ?", "latest").Delete(&models.MessageSenderProvider{}).Error; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
sender := &testChannel{}
|
||||
if err := RegisterNotificationChannel("latest", models.Configuration{Type: "managed"}, sender); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := db.Create(&models.MessageSenderProvider{
|
||||
Name: "latest",
|
||||
Addition: `{"token":"first"}`,
|
||||
}).Error; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := CallNotificationChannel(context.Background(), "latest", Notification{}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if sender.config["token"] != "first" {
|
||||
t.Fatalf("first config = %#v", sender.config)
|
||||
}
|
||||
if err := db.Model(&models.MessageSenderProvider{}).
|
||||
Where("name = ?", "latest").
|
||||
Update("addition", `{"token":"second"}`).Error; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := CallNotificationChannel(context.Background(), "latest", Notification{}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if sender.config["token"] != "second" {
|
||||
t.Fatalf("second config = %#v", sender.config)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRegistryDoesNotRetryFailedCall(t *testing.T) {
|
||||
clearNotificationChannels()
|
||||
t.Cleanup(clearNotificationChannels)
|
||||
sender := &testChannel{err: errors.New("send failed")}
|
||||
if err := RegisterNotificationChannel("failed", models.Configuration{Type: "managed"}, sender); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := CallNotificationChannel(context.Background(), "failed", Notification{}); err == nil {
|
||||
t.Fatal("failed call returned nil")
|
||||
}
|
||||
if sender.sends != 1 {
|
||||
t.Fatalf("sends = %d, want 1", sender.sends)
|
||||
}
|
||||
}
|
||||
|
||||
func clearNotificationChannels() {
|
||||
notificationChannels.Lock()
|
||||
notificationChannels.entries = make(map[string]*notificationChannelEntry)
|
||||
notificationChannels.Unlock()
|
||||
}
|
||||
|
||||
var _ NotificationChannel = (*testChannel)(nil)
|
||||
var _ NotificationChannel = (*blockingChannel)(nil)
|
||||
+16
-119
@@ -1,111 +1,18 @@
|
||||
package messageSender
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"context"
|
||||
"fmt"
|
||||
logger "github.com/komari-monitor/komari/utils/log"
|
||||
"reflect"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/komari-monitor/komari/database"
|
||||
"github.com/komari-monitor/komari/database/auditlog"
|
||||
"github.com/komari-monitor/komari/database/clients"
|
||||
"github.com/komari-monitor/komari/database/models"
|
||||
"github.com/komari-monitor/komari/internal/config"
|
||||
"github.com/komari-monitor/komari/utils/messageSender/factory"
|
||||
)
|
||||
|
||||
var (
|
||||
currentProvider factory.IMessageSender
|
||||
mu = sync.Mutex{}
|
||||
once = sync.Once{}
|
||||
)
|
||||
|
||||
func CurrentProvider() factory.IMessageSender {
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
return currentProvider
|
||||
}
|
||||
|
||||
// Shutdown 销毁当前消息发送 provider,释放其持有的资源。供关闭流程调用。
|
||||
func Shutdown() error {
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
if currentProvider == nil {
|
||||
return nil
|
||||
}
|
||||
err := currentProvider.Destroy()
|
||||
currentProvider = nil
|
||||
return err
|
||||
}
|
||||
|
||||
func Initialize() {
|
||||
go func() {
|
||||
once.Do(func() {
|
||||
all := factory.GetAllMessageSenders()
|
||||
for _, provider := range all {
|
||||
if _, err := database.GetMessageSenderConfigByName(provider.GetName()); err == nil {
|
||||
continue
|
||||
}
|
||||
// 如果数据库中没有该提供者的配置,则保存默认配置
|
||||
config := provider.GetConfiguration()
|
||||
configBytes, err := json.Marshal(config)
|
||||
if err != nil {
|
||||
logger.Errorf("message-sender", "Failed to marshal config for provider %s: %v", provider.GetName(), err)
|
||||
return
|
||||
}
|
||||
if err := database.SaveMessageSenderConfig(&models.MessageSenderProvider{
|
||||
Name: provider.GetName(),
|
||||
Addition: string(configBytes),
|
||||
}); err != nil {
|
||||
logger.Errorf("message-sender", "Failed to save default config for provider %s: %v", provider.GetName(), err)
|
||||
return
|
||||
}
|
||||
}
|
||||
})
|
||||
}()
|
||||
NotificationMethod, _ := config.GetAs[string](config.NotificationMethodKey, "none")
|
||||
|
||||
if NotificationMethod == "" || NotificationMethod == "none" {
|
||||
LoadProvider("empty", "{}")
|
||||
return
|
||||
}
|
||||
|
||||
// 尝试从数据库加载配置
|
||||
senderConfig, err := database.GetMessageSenderConfigByName(NotificationMethod)
|
||||
if err != nil {
|
||||
// 如果没有找到配置,使用empty provider
|
||||
LoadProvider("empty", "{}")
|
||||
return
|
||||
}
|
||||
LoadProvider(NotificationMethod, senderConfig.Addition)
|
||||
}
|
||||
|
||||
func SendTextMessage(message string, title string) error {
|
||||
if CurrentProvider() == nil {
|
||||
return fmt.Errorf("message sender provider is not initialized")
|
||||
}
|
||||
var err error
|
||||
NotificationEnabled, err := config.GetAs[bool](config.NotificationEnabledKey, false)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !NotificationEnabled {
|
||||
return nil
|
||||
}
|
||||
for i := 0; i < 3; i++ {
|
||||
err = CurrentProvider().SendTextMessage(message, title)
|
||||
if err == nil {
|
||||
auditlog.Log("", "", "Message sent: "+title, "info")
|
||||
return nil
|
||||
}
|
||||
}
|
||||
auditlog.Log("", "", "Failed to send message after 3 attempts: "+err.Error()+","+title, "error")
|
||||
return err
|
||||
}
|
||||
|
||||
// SendNotification 是通知发送的统一实现:解析事件中的客户端 UUID(外部传入可只含
|
||||
// UUID 字段)后委托 SendEvent。内部调用与 admin:sendNotification RPC 共用此实现。
|
||||
func SendNotification(event models.EventMessage) error {
|
||||
@@ -128,9 +35,6 @@ func SendNotification(event models.EventMessage) error {
|
||||
}
|
||||
|
||||
func SendEvent(event models.EventMessage) error {
|
||||
if CurrentProvider() == nil {
|
||||
return fmt.Errorf("message sender provider is not initialized")
|
||||
}
|
||||
if event.Time.IsZero() {
|
||||
event.Time = time.Now().UTC()
|
||||
} else {
|
||||
@@ -148,33 +52,26 @@ func SendEvent(event models.EventMessage) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// 检查提供者是否实现了 IEventMessageSender 接口
|
||||
if eventSender, ok := CurrentProvider().(factory.IEventMessageSender); ok {
|
||||
// 如果实现了,直接调用 SendEvent
|
||||
for i := 0; i < 3; i++ {
|
||||
err = eventSender.SendEvent(event)
|
||||
if err == nil || err.Error() == "short response: \x00\x00\x00\x1a\x00\x00\x00" {
|
||||
auditlog.Log("", "", "Event message sent: "+fmt.Sprint(event.Event), "info")
|
||||
return nil
|
||||
}
|
||||
}
|
||||
auditlog.Log("", "", "Failed to send event message after 3 attempts: "+err.Error()+","+fmt.Sprint(event.Event), "error")
|
||||
method, err := config.GetAs[string](config.NotificationMethodKey, "none")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// 如果没有实现,使用模板格式化为文本消息
|
||||
messageTemplate := cfg[config.NotificationTemplateKey].(string)
|
||||
|
||||
messageTemplate = parseTemplate(messageTemplate, event)
|
||||
|
||||
for i := 0; i < 3; i++ {
|
||||
err = CurrentProvider().SendTextMessage(messageTemplate, fmt.Sprint(event.Event))
|
||||
if err == nil || err.Error() == "short response: \x00\x00\x00\x1a\x00\x00\x00" { // QQ 会返回这个错误,但实际上消息是发送成功的
|
||||
auditlog.Log("", "", "Event message sent: "+fmt.Sprint(event.Event), "info")
|
||||
return nil
|
||||
}
|
||||
if method == "" || method == "none" {
|
||||
return nil
|
||||
}
|
||||
auditlog.Log("", "", "Failed to send event message after 3 attempts: "+err.Error()+","+fmt.Sprint(event.Event), "error")
|
||||
err = CallNotificationChannel(context.Background(), method, Notification{
|
||||
Event: event,
|
||||
Title: fmt.Sprint(event.Event),
|
||||
Message: messageTemplate,
|
||||
})
|
||||
if err == nil {
|
||||
auditlog.Log("", "", "Event message sent: "+fmt.Sprint(event.Event), "info")
|
||||
return nil
|
||||
}
|
||||
auditlog.Log("", "", "Failed to send event message: "+err.Error()+","+fmt.Sprint(event.Event), "error")
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -232,7 +129,7 @@ func formatTemplateField(fieldName string, v reflect.Value) string {
|
||||
}
|
||||
case reflect.Struct:
|
||||
if t, ok := v.Interface().(time.Time); ok {
|
||||
return t.In(time.Local).Format(time.RFC3339Nano)
|
||||
return t.In(time.Local).Format("2006-01-02 15:04:05")
|
||||
}
|
||||
}
|
||||
return ""
|
||||
|
||||
Loaded 100 of 182 files, more files were not shown because too many files have changed in this diff.
Show more
Reference in new issue
Block a user