mirror of
https://github.com/komari-monitor/komari.git
synced 2026-11-12 09:24:35 +00:00
Compare commits
65
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
e5d67686db | ||
|
|
96462528c8 | ||
|
|
d92f60f757 | ||
|
|
cba8153d4a | ||
|
|
08067bc8f8 | ||
|
|
a714637fee | ||
|
|
956b9334d0 | ||
|
|
996dd90386 | ||
|
|
28abafb5b2 | ||
|
|
5488189e5f | ||
|
|
d9713aa696 | ||
|
|
eab03a1da5 | ||
|
|
1f139ba3bf | ||
|
|
4040a5d9c7 | ||
|
|
01c4816d1c | ||
|
|
c27c21d5c9 | ||
|
|
f967ba07f9 | ||
|
|
91d5f92c55 | ||
|
|
965afb3f77 | ||
|
|
68bf4ed148 | ||
|
|
8985adcf67 | ||
|
|
91cbeb3f82 | ||
|
|
91ae9eda1a | ||
|
|
54851bd1cf | ||
|
|
3c5838283e | ||
|
|
df6ffba18c | ||
|
|
4077201f09 | ||
|
|
05a91adc53 | ||
|
|
1c75cb8be0 | ||
|
|
d69effee7a | ||
|
|
ebe5278183 | ||
|
|
16d6f2670d | ||
|
|
abc53b3cc0 | ||
|
|
a1ae82169c | ||
|
|
8c849aa423 | ||
|
|
0d699951b7 | ||
|
|
1ffe413eb7 | ||
|
|
c3c3ed8ecf | ||
|
|
0103b99651 | ||
|
|
b8c100823f | ||
|
|
c7819cd96e | ||
|
|
17aa37c64f | ||
|
|
045f24ef85 | ||
|
|
6ab9c6724e | ||
|
|
6efc72ce31 | ||
|
|
1f6bd1391a | ||
|
|
5b93181d94 | ||
|
|
f07bd749c4 | ||
|
|
887381ad4f | ||
|
|
11617b46b3 | ||
|
|
5a8488926c | ||
|
|
db2ed8e424 | ||
|
|
312834f921 | ||
|
|
9d149ab660 | ||
|
|
9d7e24ab78 | ||
|
|
99c684e75e | ||
|
|
a402cc6189 | ||
|
|
0bac110d8a | ||
|
|
7a955467fd | ||
|
|
eb7ecc6fe4 | ||
|
|
f59e0e813c | ||
|
|
b4d3977871 | ||
|
|
b55436842f | ||
|
|
cdfd5bd13f | ||
|
|
5a66f28e81 |
No files matched your search
@@ -8,6 +8,9 @@ inputs:
|
||||
repository:
|
||||
description: komari-web repository URL.
|
||||
default: https://github.com/komari-monitor/komari-web
|
||||
ref:
|
||||
description: Optional komari-web ref to build. Uses the repository default branch when empty.
|
||||
default: ""
|
||||
artifact-name:
|
||||
description: Optional artifact name for the packed frontend theme.
|
||||
default: ""
|
||||
@@ -25,7 +28,11 @@ runs:
|
||||
run: |
|
||||
set -euxo pipefail
|
||||
rm -rf komari-web
|
||||
git clone --depth=1 "${{ inputs.repository }}" komari-web
|
||||
if [ -n "${{ inputs.ref }}" ]; then
|
||||
git clone --depth=1 --branch "${{ inputs.ref }}" "${{ inputs.repository }}" komari-web
|
||||
else
|
||||
git clone --depth=1 "${{ inputs.repository }}" komari-web
|
||||
fi
|
||||
cd komari-web
|
||||
npm install
|
||||
npm run build
|
||||
|
||||
@@ -0,0 +1,159 @@
|
||||
name: Rebuild Existing Release
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
release_tag:
|
||||
description: Existing GitHub Release tag whose assets and version image will be replaced.
|
||||
required: true
|
||||
type: string
|
||||
server_ref:
|
||||
description: Server commit, branch, or tag to build.
|
||||
required: true
|
||||
type: string
|
||||
frontend_ref:
|
||||
description: komari-web commit, branch, or tag to build.
|
||||
required: true
|
||||
type: string
|
||||
|
||||
permissions:
|
||||
contents: write
|
||||
packages: write
|
||||
|
||||
env:
|
||||
REGISTRY: ghcr.io
|
||||
IMAGE_NAME: ${{ github.repository }}
|
||||
|
||||
jobs:
|
||||
build-frontend:
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
steps:
|
||||
- name: Checkout workflow source
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Build selected frontend artifact
|
||||
uses: ./.github/actions/build-frontend
|
||||
with:
|
||||
artifact-name: komari-frontend
|
||||
ref: ${{ inputs.frontend_ref }}
|
||||
|
||||
build-and-attach-binaries:
|
||||
needs: build-frontend
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
strategy:
|
||||
matrix:
|
||||
include:
|
||||
- goos: windows
|
||||
goarch: amd64
|
||||
zig_target_triple: x86_64-windows-gnu
|
||||
binary_extension: .exe
|
||||
- goos: windows
|
||||
goarch: arm64
|
||||
zig_target_triple: aarch64-windows-gnu
|
||||
binary_extension: .exe
|
||||
- goos: windows
|
||||
goarch: 386
|
||||
zig_target_triple: x86-windows-gnu
|
||||
binary_extension: .exe
|
||||
- goos: linux
|
||||
goarch: amd64
|
||||
zig_target_triple: x86_64-linux-musl
|
||||
binary_extension: ""
|
||||
- goos: linux
|
||||
goarch: arm64
|
||||
zig_target_triple: aarch64-linux-musl
|
||||
binary_extension: ""
|
||||
- goos: linux
|
||||
goarch: 386
|
||||
zig_target_triple: x86-linux-musl
|
||||
binary_extension: ""
|
||||
- goos: linux
|
||||
goarch: riscv64
|
||||
zig_target_triple: riscv64-linux-musl
|
||||
binary_extension: ""
|
||||
|
||||
steps:
|
||||
- name: Checkout server source
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
ref: ${{ inputs.server_ref }}
|
||||
|
||||
- name: Restore frontend artifact
|
||||
uses: komari-monitor/komari/.github/actions/restore-frontend@main
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v5
|
||||
with:
|
||||
go-version: "1.23"
|
||||
|
||||
- name: Install Zig cross-compiler toolchain
|
||||
uses: komari-monitor/komari/.github/actions/setup-zig@main
|
||||
|
||||
- name: Build binary
|
||||
env:
|
||||
GOOS: ${{ matrix.goos }}
|
||||
GOARCH: ${{ matrix.goarch }}
|
||||
CGO_ENABLED: 1
|
||||
CC: zig cc -target ${{ matrix.zig_target_triple }}
|
||||
GIN_MODE: release
|
||||
RELEASE_TAG: ${{ inputs.release_tag }}
|
||||
run: |
|
||||
BINARY_NAME=komari-${{ matrix.goos }}-${{ matrix.goarch }}${{ matrix.binary_extension }}
|
||||
VERSION_HASH="$(git rev-parse HEAD)"
|
||||
go build -trimpath -ldflags="-s -w -X github.com/komari-monitor/komari/utils.CurrentVersion=${RELEASE_TAG} -X github.com/komari-monitor/komari/utils.VersionHash=${VERSION_HASH}" -o "$BINARY_NAME"
|
||||
|
||||
- name: Upload binary to release
|
||||
env:
|
||||
GH_TOKEN: ${{ github.token }}
|
||||
RELEASE_TAG: ${{ inputs.release_tag }}
|
||||
run: |
|
||||
BINARY_NAME=komari-${{ matrix.goos }}-${{ matrix.goarch }}${{ matrix.binary_extension }}
|
||||
gh release upload "$RELEASE_TAG" "$BINARY_NAME" --clobber --repo "$GITHUB_REPOSITORY"
|
||||
|
||||
- name: Upload binary as artifact
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: komari-${{ matrix.goos }}-${{ matrix.goarch }}
|
||||
path: komari-${{ matrix.goos }}-${{ matrix.goarch }}${{ matrix.binary_extension }}
|
||||
|
||||
build-and-push-docker:
|
||||
needs: build-and-attach-binaries
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
steps:
|
||||
- name: Checkout server source
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
ref: ${{ inputs.server_ref }}
|
||||
|
||||
- name: Download Docker binaries
|
||||
uses: actions/download-artifact@v4
|
||||
with:
|
||||
pattern: komari-linux-*
|
||||
path: .
|
||||
merge-multiple: true
|
||||
|
||||
- name: Set up QEMU
|
||||
uses: docker/setup-qemu-action@v3
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@v3
|
||||
|
||||
- name: Log in to the Container registry
|
||||
uses: docker/login-action@v3
|
||||
with:
|
||||
registry: ${{ env.REGISTRY }}
|
||||
username: ${{ github.actor }}
|
||||
password: ${{ github.token }}
|
||||
|
||||
- name: Build and push version Docker image
|
||||
uses: docker/build-push-action@v5
|
||||
with:
|
||||
context: .
|
||||
push: true
|
||||
platforms: linux/amd64,linux/arm64
|
||||
tags: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ inputs.release_tag }}
|
||||
no-cache: true
|
||||
provenance: false
|
||||
@@ -107,7 +107,6 @@ jobs:
|
||||
images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
|
||||
tags: |
|
||||
type=raw,value=${{ github.event.inputs.version }}
|
||||
type=raw,value=latest
|
||||
|
||||
- name: Build and push Docker image
|
||||
uses: docker/build-push-action@v5
|
||||
|
||||
@@ -13,7 +13,24 @@ env:
|
||||
IMAGE_NAME: ${{ github.repository }}
|
||||
|
||||
jobs:
|
||||
resolve-release:
|
||||
runs-on: ubuntu-latest
|
||||
outputs:
|
||||
frontend_ref: ${{ steps.release.outputs.frontend_ref }}
|
||||
|
||||
steps:
|
||||
- id: release
|
||||
env:
|
||||
TAG: ${{ github.event.release.tag_name }}
|
||||
run: |
|
||||
frontend_ref=""
|
||||
if [[ "$TAG" =~ ^(.+)-fix[0-9]+$ ]]; then
|
||||
frontend_ref="${BASH_REMATCH[1]}"
|
||||
fi
|
||||
echo "frontend_ref=$frontend_ref" >> "$GITHUB_OUTPUT"
|
||||
|
||||
build-frontend:
|
||||
needs: resolve-release
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
steps:
|
||||
@@ -24,6 +41,7 @@ jobs:
|
||||
uses: ./.github/actions/build-frontend
|
||||
with:
|
||||
artifact-name: komari-frontend
|
||||
ref: ${{ needs.resolve-release.outputs.frontend_ref }}
|
||||
|
||||
build-and-attach-binaries:
|
||||
needs: build-frontend
|
||||
@@ -155,6 +173,20 @@ jobs:
|
||||
username: ${{ github.actor }}
|
||||
password: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
- name: Check whether this is the latest GitHub release
|
||||
id: release
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
TAG: ${{ github.event.release.tag_name }}
|
||||
run: |
|
||||
is_latest="$(gh api graphql \
|
||||
-f 'query=query($owner: String!, $name: String!, $tag: String!) { repository(owner: $owner, name: $name) { release(tagName: $tag) { isLatest } } }' \
|
||||
-F owner="${{ github.repository_owner }}" \
|
||||
-F name="${{ github.event.repository.name }}" \
|
||||
-F tag="$TAG" \
|
||||
--jq '.data.repository.release.isLatest')"
|
||||
echo "is_latest=$is_latest" >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Extract metadata
|
||||
id: meta
|
||||
uses: docker/metadata-action@v5
|
||||
@@ -162,7 +194,6 @@ jobs:
|
||||
images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
|
||||
tags: |
|
||||
type=raw,value=${{ github.event.release.tag_name }}
|
||||
type=raw,value=latest
|
||||
|
||||
- name: Build and push Docker image
|
||||
uses: docker/build-push-action@v5
|
||||
@@ -174,3 +205,9 @@ jobs:
|
||||
labels: ${{ steps.meta.outputs.labels }}
|
||||
no-cache: true
|
||||
provenance: false
|
||||
|
||||
- name: Promote Docker latest tag
|
||||
if: steps.release.outputs.is_latest == 'true'
|
||||
env:
|
||||
TAG: ${{ github.event.release.tag_name }}
|
||||
run: docker buildx imagetools create --tag "${REGISTRY}/${IMAGE_NAME}:latest" "${REGISTRY}/${IMAGE_NAME}:${TAG}"
|
||||
@@ -34,4 +34,5 @@ go.work.sum
|
||||
|
||||
# build
|
||||
komari
|
||||
/dist/
|
||||
.codex
|
||||
@@ -5,7 +5,7 @@
|
||||
|
||||

|
||||
|
||||
[简体中文](./docs/README_zh.md) | [繁體中文](./docs/README_zh-TW.md) | [日本語](./docs/README_ja.md)
|
||||
[简体中文](./README_zh-cn.md)
|
||||
|
||||
Komari is a lightweight, self-hosted server monitoring tool designed to provide a simple and efficient solution for monitoring server performance. It supports viewing server status through a web interface and collects data through a lightweight agent.
|
||||
|
||||
@@ -14,7 +14,7 @@ Komari is a lightweight, self-hosted server monitoring tool designed to provide
|
||||
> Users are solely responsible for how they deploy and operate Komari. The developers do not accept responsibility for unauthorized or abusive use, or for any resulting consequences.
|
||||
> On Windows, when remote control is enabled, the client displays a Windows notification at each user login to remind the user that Komari is remote control software.
|
||||
|
||||
[Documentation](https://komari-document.pages.dev/) | [文档(镜像站 By Geekertao)](https://www.komari.wiki) | [Telegram Group](https://t.me/komari_monitor)
|
||||
[文档/Documentation](https://komari-document.pages.dev/) | [文档(镜像站 By Geekertao)](https://www.komari.wiki) | [Telegram Group](https://t.me/komari_monitor)
|
||||
|
||||
## Features
|
||||
|
||||
@@ -22,9 +22,7 @@ Komari is a lightweight, self-hosted server monitoring tool designed to provide
|
||||
- **Self-hosted**: Complete control over data privacy, easy to deploy.
|
||||
- **Web Interface**: Intuitive monitoring dashboard, easy to use.
|
||||
|
||||
## Quick Start
|
||||
|
||||
### 0. One-click Deployment with Cloud Hosting
|
||||
## One-click Deployment
|
||||
|
||||
- Rainyun - CNY 4.5/month
|
||||
|
||||
@@ -34,87 +32,7 @@ Komari is a lightweight, self-hosted server monitoring tool designed to provide
|
||||
|
||||
Available on 1Panel App Store. Install via **App Store > Utilities > Komari**.
|
||||
|
||||
### 1. Use the One-click Install Script
|
||||
|
||||
Suitable for distributions using systemd (Ubuntu, Debian...).
|
||||
|
||||
```bash
|
||||
curl -fsSL https://raw.githubusercontent.com/komari-monitor/komari/main/install-komari.sh -o install-komari.sh
|
||||
chmod +x install-komari.sh
|
||||
sudo ./install-komari.sh
|
||||
```
|
||||
|
||||
### 2. Docker Deployment
|
||||
|
||||
1. Create a data directory:
|
||||
```bash
|
||||
mkdir -p ./data
|
||||
```
|
||||
2. Run the Docker container:
|
||||
```bash
|
||||
docker run -d \
|
||||
-p 25774:25774 \
|
||||
-v $(pwd)/data:/app/data \
|
||||
--name komari \
|
||||
ghcr.io/komari-monitor/komari:latest
|
||||
```
|
||||
3. View the default username and password:
|
||||
```bash
|
||||
docker logs komari
|
||||
```
|
||||
4. Access `http://<your_server_ip>:25774` in your browser.
|
||||
|
||||
> [!NOTE]
|
||||
> You can also customize the initial username and password through the environment variables `ADMIN_USERNAME` and `ADMIN_PASSWORD`.
|
||||
|
||||
### 3. Binary File Deployment
|
||||
|
||||
1. Visit Komari's [GitHub Release page](https://github.com/komari-monitor/komari/releases) to download the latest binary for your operating system.
|
||||
2. Run Komari:
|
||||
```bash
|
||||
./komari server -l 0.0.0.0:25774
|
||||
```
|
||||
3. Access `http://<your_server_ip>:25774` in your browser. The default port is `25774`.
|
||||
4. The default username and password can be found in the startup logs or set via the environment variables `ADMIN_USERNAME` and `ADMIN_PASSWORD`.
|
||||
|
||||
> [!NOTE]
|
||||
> Ensure the binary has execute permissions (`chmod +x komari`). Data will be saved in the `data` folder in the running directory.
|
||||
|
||||
### Manual Build
|
||||
|
||||
#### Dependencies
|
||||
|
||||
- Go 1.18+ and Node.js 20+ (for manual build)
|
||||
|
||||
1. Build the frontend static files:
|
||||
```bash
|
||||
git clone https://github.com/komari-monitor/komari-web
|
||||
cd komari-web
|
||||
npm install
|
||||
npm run build
|
||||
```
|
||||
2. Build the backend:
|
||||
```bash
|
||||
git clone https://github.com/komari-monitor/komari
|
||||
cd komari
|
||||
```
|
||||
Copy the static files generated in step 1 to the `/web/public/defaultTheme/dist` folder in the root of the `komari` project, and copy `komari-theme.json` + `preview.png`/`perview.png` to `/web/public/defaultTheme`.
|
||||
```bash
|
||||
go build -o komari
|
||||
```
|
||||
3. Run:
|
||||
```bash
|
||||
./komari server -l 0.0.0.0:25774
|
||||
```
|
||||
The default listening port is `25774`. Access `http://localhost:25774`.
|
||||
|
||||
## Frontend Development Guide
|
||||
|
||||
[Komari Theme Development Guide | Komari](https://komari-document.pages.dev/dev/theme.html)
|
||||
|
||||
## Client Agent Development Guide
|
||||
|
||||
[Komari Agent Information Reporting and Event Handling Documentation](https://komari-document.pages.dev/dev/agent.html)
|
||||
For Docker, binary, source-build, and update instructions, see the [installation guide](https://komari-document.pages.dev/en/install/quick-start).
|
||||
|
||||
## Contributing
|
||||
|
||||
@@ -122,26 +40,8 @@ Issues and Pull Requests are welcome!
|
||||
|
||||
## Acknowledgements
|
||||
|
||||
### 破碎工坊云
|
||||
|
||||
[破碎工坊云 - 专业云计算服务平台,提供高效、稳定、安全的高防服务器与CDN解决方案](https://www.crash.work/)
|
||||
|
||||
### DreamCloud
|
||||
|
||||
[DreamCloud - 极高性价比解锁直连亚太高防](https://as211392.com/)
|
||||
|
||||
### 🚀 Sponsored by SharonNetworks
|
||||
|
||||
[](https://sharon.io)
|
||||
|
||||
SharonNetworks 为您的业务起飞保驾护航!
|
||||
|
||||
亚太数据中心提供顶级的中国优化网络接入 · 低延时&高带宽&提供Tbps级本地清洗高防服务, 为您的业务保驾护航, 为您的客户提供极致体验. 加入社区 [Telegram群组](https://t.me/SharonNetwork) 可参与公益募捐或群内抽奖免费使用
|
||||
|
||||
### The open source software community
|
||||
|
||||
All the developers who submitted PRs and created themes
|
||||
|
||||
## Star History
|
||||
|
||||
[](https://www.star-history.com/#komari-monitor/komari&Date)
|
||||
| Acknowledgement | Details |
|
||||
| ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
|
||||
| <a href="https://whmcs.as211392.com/aff.php?aff=110"><img src="https://raw.githubusercontent.com/komari-monitor/public/refs/heads/main/images/dreamcloud.png" alt="Dream Cloud" width="180"></a> | 极高性价比解锁直连亚太高防,真高防,不虚标,打死退款 |
|
||||
| <a href="https://sharon.io"><img src="https://raw.githubusercontent.com/komari-monitor/public/refs/heads/main/images/sharon-networks.webp" alt="Sharon Networks" width="180"></a> | 亚太数据中心提供顶级的中国优化网络接入 · 低延时&高带宽&提供Tbps级本地清洗高防服务, 为您的业务保驾护航, 为您的客户提供极致体验. 加入社区 [Telegram群组](https://t.me/SharonNetwork) 可参与公益募捐或群内抽奖免费使用 |
|
||||
| Open-source community | Everyone who has submitted pull requests or created themes. |
|
||||
@@ -0,0 +1,49 @@
|
||||
# Komari
|
||||
|
||||

|
||||
[](https://deepwiki.com/komari-monitor/komari)
|
||||
|
||||

|
||||
|
||||
[简体中文](./README_zh-cn.md)
|
||||
|
||||
Komari 是一款轻量级的自托管服务器监控工具,旨在提供简单、高效的服务器性能监控解决方案。它支持通过 Web 界面查看服务器状态,并通过轻量级 Agent 收集数据。
|
||||
|
||||
> [!WARNING]
|
||||
> Komari 是一款自托管的监控/控制程序,仅应部署在你拥有或已获得授权管理的系统上。请勿将 Komari 武器化,或在未获授权的情况下部署、访问、持久化、执行命令及从事其他滥用行为。关于现实中的滥用风险,可参考 Huntress 的分析:[Komari C2 agent abuse](https://www.huntress.com/blog/komari-c2-agent-abuse)。
|
||||
> 用户需要自行承担部署和使用 Komari 的责任。开发者不对未经授权或滥用行为及其后果承担责任。
|
||||
> 在 Windows 端开启远程控制后,客户端会在每次用户登录时通过 Windows 通知提醒用户 Komari 是一款远程控制软件。
|
||||
|
||||
[文档/Documentation](https://www.komari.wiki/) | [Telegram 群](https://t.me/komari_monitor)
|
||||
|
||||
## 特性
|
||||
|
||||
- **轻量高效**:低资源占用,适合各种规模的服务器。
|
||||
- **自托管**:完全掌控数据隐私,部署简单。
|
||||
- **Web 界面**:直观的监控仪表盘,易于使用。
|
||||
|
||||
## 快速开始
|
||||
|
||||
### 0. 容器云一键部署
|
||||
|
||||
- 雨云云应用 - CNY 4.5/月
|
||||
|
||||
[](https://app.rainyun.com/apps/rca/store/6780/NzYxNzAz_)
|
||||
|
||||
- 1Panel 应用商店
|
||||
|
||||
已上架1Panel应用商店,应用商店-实用工具-Komari 即可安装
|
||||
|
||||
Docker、二进制文件、源码构建和更新说明,请参阅 [安装指南](https://www.komari.wiki/install/quick-start).
|
||||
|
||||
## 贡献
|
||||
|
||||
欢迎提交 Issue 或 Pull Request!
|
||||
|
||||
## 鸣谢
|
||||
|
||||
| 赞助商 | 描述 |
|
||||
| ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
|
||||
| <a href="https://whmcs.as211392.com/aff.php?aff=110"><img src="https://raw.githubusercontent.com/komari-monitor/public/refs/heads/main/images/dreamcloud.png" alt="Dream Cloud" width="180"></a> | 极高性价比解锁直连亚太高防,真高防,不虚标,打死退款 |
|
||||
| <a href="https://sharon.io"><img src="https://raw.githubusercontent.com/komari-monitor/public/refs/heads/main/images/sharon-networks.webp" alt="Sharon Networks" width="180"></a> | 亚太数据中心提供顶级的中国优化网络接入 · 低延时&高带宽&提供Tbps级本地清洗高防服务, 为您的业务保驾护航, 为您的客户提供极致体验. 加入社区 [Telegram群组](https://t.me/SharonNetwork) 可参与公益募捐或群内抽奖免费使用 |
|
||||
| 开源社区 | 提交 PR、制作主题的各位开发者 |
|
||||
-498
@@ -1,498 +0,0 @@
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"log"
|
||||
"net/http"
|
||||
"os"
|
||||
"os/signal"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
"github.com/gin-gonic/gin"
|
||||
|
||||
"github.com/komari-monitor/komari/cmd/flags"
|
||||
"github.com/komari-monitor/komari/database"
|
||||
"github.com/komari-monitor/komari/database/accounts"
|
||||
"github.com/komari-monitor/komari/database/auditlog"
|
||||
"github.com/komari-monitor/komari/database/dbcore"
|
||||
"github.com/komari-monitor/komari/database/metricstore"
|
||||
"github.com/komari-monitor/komari/database/models"
|
||||
d_notification "github.com/komari-monitor/komari/database/notification"
|
||||
"github.com/komari-monitor/komari/database/tasks"
|
||||
"github.com/komari-monitor/komari/pkg/config"
|
||||
"github.com/komari-monitor/komari/pkg/corn"
|
||||
"github.com/komari-monitor/komari/pkg/migrations"
|
||||
"github.com/komari-monitor/komari/utils"
|
||||
"github.com/komari-monitor/komari/utils/geoip"
|
||||
logutil "github.com/komari-monitor/komari/utils/log"
|
||||
"github.com/komari-monitor/komari/utils/messageSender"
|
||||
"github.com/komari-monitor/komari/utils/notifier"
|
||||
"github.com/komari-monitor/komari/web/api"
|
||||
"github.com/komari-monitor/komari/web/oauth"
|
||||
frontendpublic "github.com/komari-monitor/komari/web/public"
|
||||
"github.com/komari-monitor/komari/web/router"
|
||||
"github.com/komari-monitor/komari/web/security"
|
||||
upgradeweb "github.com/komari-monitor/komari/web/update"
|
||||
)
|
||||
|
||||
// cleanupFunc 是一个关闭阶段执行的清理函数。
|
||||
type cleanupFunc struct {
|
||||
name string
|
||||
fn func(ctx context.Context) error
|
||||
}
|
||||
|
||||
// App 显式建模服务端的启动生命周期。
|
||||
//
|
||||
// 过去 RunServer 把目录创建、数据库、metric store、GeoIP、定时任务、通知、OAuth、
|
||||
// Gin 中间件、路由、HTTP 启动与 shutdown 全部混在一个函数里,
|
||||
// 启动顺序只能靠通读整段代码推断,异步初始化过早且吞掉错误,关闭也不完整。
|
||||
//
|
||||
// App 把这些拆成有序阶段:
|
||||
//
|
||||
// Bootstrap 基础设施:目录、数据库、默认管理员、配置快照
|
||||
// InitStores 存储:metric store
|
||||
// InitProviders 外部 provider:OAuth(同步,路由依赖)、GeoIP、消息发送
|
||||
// StartBackground 后台:定时任务
|
||||
// BuildRouter 构建 Gin 引擎与路由
|
||||
// Run 启动 HTTP 服务并阻塞直到收到信号
|
||||
// Shutdown 反序执行已登记的清理函数
|
||||
//
|
||||
// 每个阶段返回错误即可让上层决定是否中止启动;各资源在创建时把对应清理登记到
|
||||
// cleanup 栈,关闭时按后进先出(LIFO)顺序释放。
|
||||
type App struct {
|
||||
settings *config.Settings
|
||||
engine *gin.Engine
|
||||
server *http.Server
|
||||
reload *ReloadManager
|
||||
dbReady bool
|
||||
oauthReady bool
|
||||
|
||||
cleanups []cleanupFunc
|
||||
}
|
||||
|
||||
// NewApp 构造一个空的 App。真正的初始化在各阶段方法中完成。
|
||||
func NewApp() *App {
|
||||
return &App{
|
||||
reload: NewReloadManager(),
|
||||
}
|
||||
}
|
||||
|
||||
// addCleanup 登记一个关闭阶段的清理函数(后进先出执行)。
|
||||
func (a *App) addCleanup(name string, fn func(ctx context.Context) error) {
|
||||
a.cleanups = append(a.cleanups, cleanupFunc{name: name, fn: fn})
|
||||
}
|
||||
|
||||
// Bootstrap 初始化基础设施:数据目录、数据库、默认管理员账号、配置快照。
|
||||
func (a *App) Bootstrap() error {
|
||||
if err := os.MkdirAll("./data/theme", os.ModePerm); err != nil {
|
||||
return fmt.Errorf("failed to create theme directory: %w", err)
|
||||
}
|
||||
|
||||
// 注入版本标识,供 dbcore 在检测到版本升级时自动备份 ./data。
|
||||
// 必须在 Initialize() 之前设置。
|
||||
dbcore.SetVersionID(utils.CurrentVersion + "-" + utils.VersionHash)
|
||||
|
||||
// 显式初始化数据库(返回错误而非在 getter 里 log.Fatal)。
|
||||
if err := dbcore.Initialize(); err != nil {
|
||||
return fmt.Errorf("failed to initialize database: %w", err)
|
||||
}
|
||||
|
||||
a.dbReady = true
|
||||
a.addCleanup("database", func(context.Context) error {
|
||||
return dbcore.Close()
|
||||
})
|
||||
|
||||
if utils.VersionHash != "unknown" {
|
||||
gin.SetMode(gin.ReleaseMode)
|
||||
}
|
||||
|
||||
// 首次启动创建默认管理员账号。
|
||||
if err := ensureDefaultAdmin(); err != nil {
|
||||
return fmt.Errorf("failed to ensure default admin account: %w", err)
|
||||
}
|
||||
|
||||
conf, err := config.GetManyAs[config.Settings]()
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to load settings: %w", err)
|
||||
}
|
||||
a.settings = conf
|
||||
return nil
|
||||
}
|
||||
|
||||
// InitStores 初始化独立存储组件(metric store)并执行 metrics 迁移。
|
||||
//
|
||||
// metric store 现在始终启用(旧的 metric_store_enabled 开关已废弃):
|
||||
// 未显式配置时使用 SQLite(./data/metrics.db),否则使用配置的 MySQL/PostgreSQL。
|
||||
// 初始化失败即启动失败,不再静默 fallback 到旧 records 表。
|
||||
//
|
||||
// 初始化成功后先执行需要 metric store 的一次性迁移,再执行启动迁移:当 metrics
|
||||
// 存储后端发生变化(例如从默认 SQLite 切换到 MySQL/PostgreSQL)时,把上一个
|
||||
// metrics 目标库的数据搬运到当前目标。迁移失败同样让启动失败,并打印明确错误。
|
||||
func (a *App) InitStores() error {
|
||||
if err := metricstore.InitializeStore(); err != nil {
|
||||
auditlog.EventLog("error", fmt.Sprintf("Failed to initialize metric store: %v", err))
|
||||
return fmt.Errorf("failed to initialize metric store: %w", err)
|
||||
}
|
||||
a.addCleanup("metric-store", func(ctx context.Context) error {
|
||||
return metricstore.CloseStoreContext(ctx)
|
||||
})
|
||||
|
||||
if err := migrations.RunMetricStoreMigrations(migrations.MetricContext{
|
||||
DB: dbcore.GetDBInstance(),
|
||||
Store: metricstore.GetStore(),
|
||||
}); err != nil {
|
||||
auditlog.EventLog("error", fmt.Sprintf("Metric store one-shot migrations failed: %v", err))
|
||||
return fmt.Errorf("metric store one-shot migrations failed: %w", err)
|
||||
}
|
||||
|
||||
// 存储后端切换时把上一个 metrics 目标库的数据搬运到当前目标(失败即启动失败)。
|
||||
if err := metricstore.RunStartupMigration(); err != nil {
|
||||
auditlog.EventLog("error", fmt.Sprintf("Metrics startup migration failed: %v", err))
|
||||
|
||||
return fmt.Errorf("metrics startup migration failed: %w", err)
|
||||
}
|
||||
metricstore.StartReportBatcher()
|
||||
a.addCleanup("metric-report-batcher", func(ctx context.Context) error {
|
||||
return metricstore.StopReportBatcher(ctx)
|
||||
})
|
||||
return nil
|
||||
}
|
||||
|
||||
// InitProviders 初始化外部 provider。
|
||||
//
|
||||
// OAuth 必须在 HTTP 服务开始接收请求之前同步完成,否则 oauth.CurrentProvider()
|
||||
// 存在空指针风险;GeoIP 与消息发送允许后台初始化。
|
||||
func (a *App) InitProviders() error {
|
||||
a.initOAuth()
|
||||
|
||||
// GeoIP:可能涉及下载/加载 mmdb,放后台执行避免拖慢启动。
|
||||
go geoip.InitGeoIp()
|
||||
a.addCleanup("geoip", func(context.Context) error {
|
||||
return geoip.Shutdown()
|
||||
})
|
||||
|
||||
// 消息发送 provider。
|
||||
messageSender.Initialize()
|
||||
a.addCleanup("message-sender", func(context.Context) error {
|
||||
return messageSender.Shutdown()
|
||||
})
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// initOAuth initializes the provider once. The restricted upgrade server also
|
||||
// needs OAuth login, so provider setup may happen before the normal provider
|
||||
// stage without registering duplicate cleanup work.
|
||||
func (a *App) initOAuth() {
|
||||
if a.oauthReady {
|
||||
return
|
||||
}
|
||||
if err := oauth.Initialize(); err != nil {
|
||||
// Keep password login available when an OAuth provider is misconfigured.
|
||||
log.Printf("Failed to initialize OAuth provider: %v", err)
|
||||
auditlog.EventLog("error", fmt.Sprintf("Failed to initialize OAuth provider: %v", err))
|
||||
}
|
||||
a.oauthReady = true
|
||||
a.addCleanup("oauth", func(context.Context) error {
|
||||
return oauth.Shutdown()
|
||||
})
|
||||
}
|
||||
|
||||
func (a *App) LegacyUpgradeRequired() (bool, migrations.LegacyMonitoringSummary, error) {
|
||||
return migrations.LegacyMonitoringMigrationRequired(dbcore.GetDBInstance())
|
||||
}
|
||||
|
||||
// RunLegacyUpgrade starts a restricted HTTP server on the normal listen
|
||||
// address. It returns only after the migration finishes and the restricted
|
||||
// listener has released the port, or after shutdown/interruption.
|
||||
func (a *App) RunLegacyUpgrade(summary migrations.LegacyMonitoringSummary) (bool, error) {
|
||||
a.initOAuth()
|
||||
controller := upgradeweb.NewController(dbcore.GetDBInstance(), summary)
|
||||
controller.Activate()
|
||||
defer controller.Deactivate()
|
||||
|
||||
r := gin.New()
|
||||
r.Use(logutil.GinLogger())
|
||||
r.Use(logutil.GinRecovery())
|
||||
cors := security.NewCorsController(a.settings.CorsOriginCheckEnabled, a.settings.CorsAllowedOrigins)
|
||||
r.Use(cors.Middleware())
|
||||
r.Use(api.IdentityMiddleware())
|
||||
r.Use(func(c *gin.Context) {
|
||||
if strings.HasPrefix(c.Request.URL.Path, "/api") {
|
||||
c.Header("Cache-Control", "no-store")
|
||||
}
|
||||
c.Next()
|
||||
})
|
||||
|
||||
controller.Register(r)
|
||||
frontendpublic.Static(r.Group("/"), func(handlers ...gin.HandlerFunc) {
|
||||
r.NoRoute(func(c *gin.Context) {
|
||||
requestPath := c.Request.URL.Path
|
||||
if strings.HasPrefix(requestPath, "/api") {
|
||||
api.RespondError(c, http.StatusNotFound, "Not found in upgrade mode")
|
||||
return
|
||||
}
|
||||
if c.Request.Method == http.MethodGet && requestPath != upgradeweb.PagePath && filepath.Ext(requestPath) == "" {
|
||||
c.Redirect(http.StatusTemporaryRedirect, upgradeweb.PagePath)
|
||||
return
|
||||
}
|
||||
for _, handler := range handlers {
|
||||
handler(c)
|
||||
if c.IsAborted() {
|
||||
return
|
||||
}
|
||||
}
|
||||
})
|
||||
})
|
||||
|
||||
a.engine = r
|
||||
a.server = &http.Server{Addr: flags.Listen, Handler: r}
|
||||
serverErr := make(chan error, 1)
|
||||
log.Printf("Legacy monitoring data requires the 1.2.7 upgrade wizard on %s", flags.Listen)
|
||||
go func() {
|
||||
if err := a.server.ListenAndServe(); err != nil && err != http.ErrServerClosed {
|
||||
serverErr <- err
|
||||
}
|
||||
}()
|
||||
|
||||
quit := make(chan os.Signal, 1)
|
||||
signal.Notify(quit, os.Interrupt, syscall.SIGTERM)
|
||||
defer signal.Stop(quit)
|
||||
|
||||
select {
|
||||
case err := <-serverErr:
|
||||
return false, fmt.Errorf("listen in upgrade mode: %w", err)
|
||||
case <-quit:
|
||||
return false, a.Shutdown()
|
||||
case <-controller.Done():
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||
defer cancel()
|
||||
if err := a.server.Shutdown(ctx); err != nil {
|
||||
return false, fmt.Errorf("stop upgrade server: %w", err)
|
||||
}
|
||||
a.server = nil
|
||||
a.engine = nil
|
||||
return true, nil
|
||||
}
|
||||
}
|
||||
|
||||
// StartBackground 启动后台工作:定时任务。
|
||||
func (a *App) StartBackground() error {
|
||||
registerScheduledWork()
|
||||
a.addCleanup("scheduler", func(context.Context) error {
|
||||
corn.StopAll()
|
||||
return nil
|
||||
})
|
||||
return nil
|
||||
}
|
||||
|
||||
// registerReloadHandlers 把此前散落各处的 config.Subscribe 统一登记到 reload 管理器。
|
||||
func (a *App) registerReloadHandlers(cors *security.CorsController) {
|
||||
// OAuth provider 切换。
|
||||
a.reload.Register("oauth-provider", func(event config.ConfigEvent) {
|
||||
if ok, t := config.IsChangedT[string](event, config.OAuthProviderKey); ok {
|
||||
if t == "" || t == "none" {
|
||||
t = "github"
|
||||
}
|
||||
oidcProvider, err := database.GetOidcConfigByName(t)
|
||||
if err != nil {
|
||||
log.Printf("Failed to get OIDC provider config: %v", err)
|
||||
return
|
||||
}
|
||||
log.Printf("Using %s as OIDC provider", oidcProvider.Name)
|
||||
if err := oauth.LoadProvider(oidcProvider.Name, oidcProvider.Addition); err != nil {
|
||||
auditlog.EventLog("error", fmt.Sprintf("Failed to load OIDC provider: %v", err))
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
// GeoIP provider 切换。
|
||||
a.reload.Register("geoip-provider", func(event config.ConfigEvent) {
|
||||
if event.IsChanged(config.GeoIpProviderKey) {
|
||||
go geoip.InitGeoIp()
|
||||
}
|
||||
})
|
||||
|
||||
// 消息发送方式切换。
|
||||
a.reload.Register("message-sender", func(event config.ConfigEvent) {
|
||||
if event.IsChanged(config.NotificationMethodKey) {
|
||||
go messageSender.Initialize()
|
||||
}
|
||||
})
|
||||
|
||||
// CORS 配置热更新。
|
||||
a.reload.Register("cors", func(event config.ConfigEvent) {
|
||||
cors.Update(event)
|
||||
})
|
||||
}
|
||||
|
||||
// BuildRouter 构建 Gin 引擎、中间件与全部路由,并登记热重载处理器。
|
||||
func (a *App) BuildRouter() error {
|
||||
r := gin.New()
|
||||
r.Use(logutil.GinLogger())
|
||||
r.Use(logutil.GinRecovery())
|
||||
|
||||
cors := security.NewCorsController(a.settings.CorsOriginCheckEnabled, a.settings.CorsAllowedOrigins)
|
||||
r.Use(cors.Middleware())
|
||||
|
||||
r.Use(api.IdentityMiddleware())
|
||||
r.Use(api.PrivateSiteMiddleware())
|
||||
|
||||
r.Use(func(c *gin.Context) {
|
||||
if len(c.Request.URL.Path) >= 4 && c.Request.URL.Path[:4] == "/api" {
|
||||
c.Header("Cache-Control", "no-store")
|
||||
}
|
||||
c.Next()
|
||||
})
|
||||
|
||||
router.Register(r)
|
||||
|
||||
// 集中登记并启动热重载订阅。
|
||||
a.registerReloadHandlers(cors)
|
||||
a.reload.Start()
|
||||
|
||||
a.engine = r
|
||||
return nil
|
||||
}
|
||||
|
||||
// Run 启动 HTTP 服务并阻塞直到收到中断信号或服务异常退出。
|
||||
func (a *App) Run() error {
|
||||
a.server = &http.Server{
|
||||
Addr: flags.Listen,
|
||||
Handler: a.engine,
|
||||
}
|
||||
|
||||
serverErr := make(chan error, 1)
|
||||
log.Printf("Starting server on %s ...", flags.Listen)
|
||||
go func() {
|
||||
if err := a.server.ListenAndServe(); err != nil && err != http.ErrServerClosed {
|
||||
serverErr <- err
|
||||
}
|
||||
}()
|
||||
|
||||
quit := make(chan os.Signal, 1)
|
||||
signal.Notify(quit, os.Interrupt, syscall.SIGTERM)
|
||||
|
||||
select {
|
||||
case err := <-serverErr:
|
||||
a.onFatal(err)
|
||||
return fmt.Errorf("listen: %w", err)
|
||||
case <-quit:
|
||||
return a.Shutdown()
|
||||
}
|
||||
}
|
||||
|
||||
// Shutdown 优雅关闭:先停止接收新请求,再反序执行已登记的清理函数。
|
||||
func (a *App) Shutdown() error {
|
||||
if a.dbReady {
|
||||
auditlog.Log("", "", "server is shutting down", "info")
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||
defer cancel()
|
||||
|
||||
// 先关闭 HTTP 服务,停止接收新请求。
|
||||
if a.server != nil {
|
||||
if err := a.server.Shutdown(ctx); err != nil {
|
||||
log.Printf("HTTP server forced to shutdown: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// 反序释放资源(后进先出)。
|
||||
for i := len(a.cleanups) - 1; i >= 0; i-- {
|
||||
c := a.cleanups[i]
|
||||
if err := c.fn(ctx); err != nil {
|
||||
log.Printf("cleanup %q failed: %v", c.name, err)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// onFatal 处理 HTTP 服务致命错误:尽力释放已登记资源。
|
||||
func (a *App) onFatal(err error) {
|
||||
if a.dbReady {
|
||||
auditlog.Log("", "", "server encountered a fatal error: "+err.Error(), "error")
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||
defer cancel()
|
||||
for i := len(a.cleanups) - 1; i >= 0; i-- {
|
||||
c := a.cleanups[i]
|
||||
if cerr := c.fn(ctx); cerr != nil {
|
||||
log.Printf("cleanup %q failed: %v", c.name, cerr)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ensureDefaultAdmin 首次启动(无任何用户)时创建默认管理员账号。
|
||||
func ensureDefaultAdmin() error {
|
||||
var count int64
|
||||
dbcore.GetDBInstance().Model(&models.User{}).Count(&count)
|
||||
if count != 0 {
|
||||
return nil
|
||||
}
|
||||
user, passwd, err := accounts.CreateDefaultAdminAccount()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
log.Println("Default admin account created. Username:", user, ", Password:", passwd)
|
||||
return nil
|
||||
}
|
||||
|
||||
// registerScheduledWork 注册所有定时任务与首启动同步逻辑。
|
||||
func registerScheduledWork() {
|
||||
if err := tasks.ReloadPingSchedule(); err != nil {
|
||||
log.Println("Failed to reload ping schedule:", err)
|
||||
}
|
||||
if err := d_notification.ReloadLoadNotificationSchedule(); err != nil {
|
||||
log.Println("Failed to reload load notification schedule:", err)
|
||||
}
|
||||
|
||||
if err := corn.AddFunc("records:cleanup", "@every 30m", cleanupScheduledData); err != nil {
|
||||
log.Println("Failed to add cleanup scheduled task:", err)
|
||||
}
|
||||
if err := corn.AddContextFunc("metrics:compact", "@every 5m", true, compactMetricStore); err != nil {
|
||||
log.Println("Failed to add metric compact scheduled task:", err)
|
||||
}
|
||||
if err := corn.AddFunc("notifier:traffic", "@every 1m", notifier.CheckTraffic); err != nil {
|
||||
log.Println("Failed to add traffic notification task:", err)
|
||||
}
|
||||
if err := corn.AddFunc("notifier:expire", "0 0 9 * * *", notifier.CheckExpireScheduledWork); err != nil {
|
||||
log.Println("Failed to add expire notification scheduled task:", err)
|
||||
}
|
||||
|
||||
notifier.InitTrafficReportSchedule()
|
||||
}
|
||||
|
||||
const taskResultRetentionDays = 30
|
||||
|
||||
func cleanupScheduledData() {
|
||||
before := time.Now().UTC().Add(-24 * time.Hour * taskResultRetentionDays)
|
||||
if err := tasks.ClearTaskResultsByTimeBefore(before); err != nil {
|
||||
log.Printf("Failed to clean expired task results: %v", err)
|
||||
}
|
||||
|
||||
auditlog.RemoveOldLogs()
|
||||
accounts.RemoveExpiredSessions()
|
||||
}
|
||||
|
||||
func compactMetricStore(ctx context.Context) {
|
||||
compactCtx, cancel := context.WithTimeout(ctx, 15*time.Minute)
|
||||
defer cancel()
|
||||
|
||||
written, err := metricstore.Compact(compactCtx, time.Now().UTC())
|
||||
if errors.Is(err, metricstore.ErrCompactInProgress) {
|
||||
return
|
||||
}
|
||||
if err != nil {
|
||||
log.Printf("Failed to compact metric store after writing %d rollup buckets: %v", written, err)
|
||||
return
|
||||
}
|
||||
if written > 0 {
|
||||
log.Printf("Metric store compacted %d rollup buckets", written)
|
||||
}
|
||||
}
|
||||
@@ -4,7 +4,7 @@ import (
|
||||
"os"
|
||||
|
||||
"github.com/komari-monitor/komari/database/dbcore"
|
||||
"github.com/komari-monitor/komari/pkg/config"
|
||||
"github.com/komari-monitor/komari/internal/config"
|
||||
"github.com/spf13/cobra"
|
||||
)
|
||||
|
||||
|
||||
+59
-12
@@ -1,7 +1,11 @@
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"log"
|
||||
"errors"
|
||||
"os"
|
||||
|
||||
appserver "github.com/komari-monitor/komari/internal/server"
|
||||
logger "github.com/komari-monitor/komari/utils/log"
|
||||
|
||||
"github.com/komari-monitor/komari/cmd/flags"
|
||||
"github.com/spf13/cobra"
|
||||
@@ -25,32 +29,71 @@ func init() {
|
||||
|
||||
// RunServer 按显式的生命周期阶段启动服务端。
|
||||
//
|
||||
// 具体各阶段的职责与顺序见 App(cmd/app.go)。这里只负责串联:
|
||||
// 具体各阶段的职责与顺序见 internal/server.App。这里只负责串联:
|
||||
// 任一初始化阶段失败即中止启动,避免在半初始化状态下对外提供服务。
|
||||
func RunServer() {
|
||||
app := NewApp()
|
||||
app := appserver.New(appserver.Options{ListenAddr: flags.Listen})
|
||||
if err := app.Bootstrap(); err != nil {
|
||||
_ = app.Shutdown()
|
||||
log.Fatalf("server startup failed at %q: %v", "bootstrap", err)
|
||||
logger.Fatalf("server", "server startup failed at %q: %v", "bootstrap", err)
|
||||
}
|
||||
|
||||
required, summary, err := app.LegacyUpgradeRequired()
|
||||
installRequired, err := app.InstallRequired()
|
||||
if err != nil {
|
||||
_ = app.Shutdown()
|
||||
log.Fatalf("server startup failed at %q: %v", "detect-1.2.7-upgrade", err)
|
||||
logger.Fatalf("server", "server startup failed at %q: %v", "detect-first-run-install", err)
|
||||
}
|
||||
if required {
|
||||
completed, err := app.RunLegacyUpgrade(summary)
|
||||
if installRequired {
|
||||
completed, err := app.RunInstallGuide()
|
||||
if err != nil {
|
||||
_ = app.Shutdown()
|
||||
log.Fatalf("server startup failed at %q: %v", "run-1.2.7-upgrade", err)
|
||||
logger.Fatalf("server", "server startup failed at %q: %v", "run-first-run-install", err)
|
||||
}
|
||||
if !completed {
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
// 初始化阶段:任一步失败都不应继续对外服务。
|
||||
for {
|
||||
requirement, err := app.DatabaseMigrationRequired()
|
||||
if err != nil {
|
||||
completed, recoveryErr := app.RunMetricStoreRecovery(err)
|
||||
if recoveryErr != nil {
|
||||
_ = app.Shutdown()
|
||||
logger.Fatalf("server", "server startup failed at %q: %v", "database-migration-detection-recovery", recoveryErr)
|
||||
}
|
||||
if !completed {
|
||||
return
|
||||
}
|
||||
continue
|
||||
}
|
||||
if !requirement.Required() {
|
||||
break
|
||||
}
|
||||
completed, err := app.RunDatabaseMigration(requirement)
|
||||
if err != nil {
|
||||
_ = app.Shutdown()
|
||||
logger.Fatalf("server", "server startup failed at %q: %v", "run-database-migration", err)
|
||||
}
|
||||
if !completed {
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
// Metric store 是唯一允许进入恢复向导的启动阶段:主库已经在
|
||||
// Bootstrap 中就绪,因此可以保留登录能力并让管理员修正 DSN。
|
||||
if err := app.ConnectMetricStoreWithRetry(); err != nil {
|
||||
completed, recoveryErr := app.RunMetricStoreRecovery(err)
|
||||
if recoveryErr != nil {
|
||||
_ = app.Shutdown()
|
||||
logger.Fatalf("server", "server startup failed at %q: %v", "metric-store-recovery", recoveryErr)
|
||||
}
|
||||
if !completed {
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
// 其余初始化阶段:任一步失败都不应继续对外服务。
|
||||
type stage struct {
|
||||
name string
|
||||
fn func() error
|
||||
@@ -65,11 +108,15 @@ func RunServer() {
|
||||
if err := s.fn(); err != nil {
|
||||
// 已登记的资源尽力回收,再退出。
|
||||
_ = app.Shutdown()
|
||||
log.Fatalf("server startup failed at %q: %v", s.name, err)
|
||||
logger.Fatalf("server", "server startup failed at %q: %v", s.name, err)
|
||||
}
|
||||
}
|
||||
|
||||
if err := app.Run(); err != nil {
|
||||
log.Fatalf("server exited with error: %v", err)
|
||||
if errors.Is(err, appserver.ErrRestartRequested) {
|
||||
logger.Infof("server", "Service stopped for restart: %v", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
logger.Fatalf("server", "server exited with error: %v", err)
|
||||
}
|
||||
}
|
||||
@@ -4,14 +4,12 @@ import (
|
||||
"crypto/sha256"
|
||||
"encoding/base64"
|
||||
"fmt"
|
||||
"os"
|
||||
"time"
|
||||
|
||||
"github.com/google/uuid"
|
||||
"github.com/komari-monitor/komari/database/dbcore"
|
||||
"github.com/komari-monitor/komari/database/models"
|
||||
"github.com/komari-monitor/komari/utils"
|
||||
|
||||
"github.com/google/uuid"
|
||||
"gorm.io/gorm"
|
||||
)
|
||||
|
||||
const constantSalt = "06Wm4Jv1Hkxx"
|
||||
@@ -56,7 +54,10 @@ func hashPasswd(passwd string) string {
|
||||
}
|
||||
|
||||
func CreateAccount(username, passwd string) (user models.User, err error) {
|
||||
db := dbcore.GetDBInstance()
|
||||
return CreateAccountWithDB(dbcore.GetDBInstance(), username, passwd)
|
||||
}
|
||||
|
||||
func CreateAccountWithDB(db *gorm.DB, username, passwd string) (user models.User, err error) {
|
||||
hashedPassword := hashPasswd(passwd)
|
||||
user = models.User{
|
||||
UUID: uuid.New().String(),
|
||||
@@ -71,7 +72,10 @@ func CreateAccount(username, passwd string) (user models.User, err error) {
|
||||
}
|
||||
|
||||
func DeleteAccountByUsername(username string) (err error) {
|
||||
db := dbcore.GetDBInstance()
|
||||
return DeleteAccountByUsernameWithDB(dbcore.GetDBInstance(), username)
|
||||
}
|
||||
|
||||
func DeleteAccountByUsernameWithDB(db *gorm.DB, username string) (err error) {
|
||||
err = db.Where("username = ?", username).Delete(&models.User{}).Error
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -79,39 +83,6 @@ func DeleteAccountByUsername(username string) (err error) {
|
||||
return nil
|
||||
}
|
||||
|
||||
// 创建默认管理员账户,使用环境变量 ADMIN_USERNAME 作为用户名,环境变量 ADMIN_PASSWORD 作为密码
|
||||
func CreateDefaultAdminAccount() (username, passwd string, err error) {
|
||||
db := dbcore.GetDBInstance()
|
||||
|
||||
username = os.Getenv("ADMIN_USERNAME")
|
||||
if username == "" {
|
||||
username = "admin"
|
||||
}
|
||||
|
||||
passwd = os.Getenv("ADMIN_PASSWORD")
|
||||
if passwd == "" {
|
||||
passwd = utils.GeneratePassword()
|
||||
}
|
||||
|
||||
hashedPassword := hashPasswd(passwd)
|
||||
|
||||
user := models.User{
|
||||
UUID: uuid.New().String(),
|
||||
Username: username,
|
||||
Passwd: hashedPassword,
|
||||
SSOID: "",
|
||||
CreatedAt: time.Now().UTC(),
|
||||
UpdatedAt: time.Now().UTC(),
|
||||
}
|
||||
|
||||
err = db.Create(&user).Error
|
||||
if err != nil {
|
||||
return "", "", err
|
||||
}
|
||||
|
||||
return username, passwd, nil
|
||||
}
|
||||
|
||||
func GetUserByUUID(uuid string) (user models.User, err error) {
|
||||
db := dbcore.GetDBInstance()
|
||||
err = db.Where("uuid = ?", uuid).First(&user).Error
|
||||
|
||||
@@ -9,7 +9,7 @@ import (
|
||||
"github.com/komari-monitor/komari/database/dbcore"
|
||||
"github.com/komari-monitor/komari/database/models"
|
||||
messageevent "github.com/komari-monitor/komari/database/models/messageEvent"
|
||||
"github.com/komari-monitor/komari/pkg/config"
|
||||
"github.com/komari-monitor/komari/internal/config"
|
||||
"github.com/komari-monitor/komari/utils"
|
||||
"github.com/komari-monitor/komari/utils/geoip"
|
||||
"github.com/komari-monitor/komari/utils/messageSender"
|
||||
@@ -48,7 +48,7 @@ func CreateSession(uuid string, expires int, userAgent, ip, login_method string)
|
||||
if ipinfo != nil && ipinfo.Name != "" {
|
||||
loc = ipinfo.Name
|
||||
}
|
||||
messageSender.SendEvent(models.EventMessage{
|
||||
_ = messageSender.SendNotification(models.EventMessage{
|
||||
Event: messageevent.Login,
|
||||
Time: time.Now().UTC(),
|
||||
Message: fmt.Sprintf("%s: %s (%s)\n%s", login_method, ip, loc, userAgent),
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
package auditlog
|
||||
|
||||
import (
|
||||
"log"
|
||||
logger "github.com/komari-monitor/komari/utils/log"
|
||||
"time"
|
||||
|
||||
"github.com/komari-monitor/komari/database/dbcore"
|
||||
@@ -17,7 +17,9 @@ func Log(ip, uuid, message, msgType string) {
|
||||
MsgType: msgType,
|
||||
Time: time.Now().UTC(),
|
||||
}
|
||||
db.Create(logEntry)
|
||||
if err := db.Create(logEntry).Error; err != nil {
|
||||
logger.Error("audit", "failed to persist audit event", "error", err, "type", msgType)
|
||||
}
|
||||
}
|
||||
|
||||
func EventLog(eventType, message string) {
|
||||
@@ -29,6 +31,6 @@ func RemoveOldLogs() {
|
||||
db := dbcore.GetDBInstance()
|
||||
threshold := time.Now().UTC().AddDate(0, 0, -30)
|
||||
if err := db.Where("time < ?", threshold).Delete(&models.Log{}).Error; err != nil {
|
||||
log.Println("Failed to remove old logs:", err)
|
||||
logger.ErrorArgs("audit", "Failed to remove old logs:", err)
|
||||
}
|
||||
}
|
||||
@@ -1,15 +1,13 @@
|
||||
package clients
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"log"
|
||||
logger "github.com/komari-monitor/komari/utils/log"
|
||||
"math"
|
||||
"time"
|
||||
|
||||
"github.com/komari-monitor/komari/database/dbcore"
|
||||
"github.com/komari-monitor/komari/database/metricstore"
|
||||
"github.com/komari-monitor/komari/database/models"
|
||||
"github.com/komari-monitor/komari/database/tasks"
|
||||
"github.com/komari-monitor/komari/utils"
|
||||
@@ -18,11 +16,6 @@ import (
|
||||
)
|
||||
|
||||
func DeleteClient(clientUuid string) error {
|
||||
// Clean the independent metric store before removing the client from the main DB.
|
||||
if err := metricstore.DeleteEntity(context.Background(), clientUuid); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
db := dbcore.GetDBInstance()
|
||||
err := db.Delete(&models.Client{}, "uuid = ?", clientUuid).Error
|
||||
if err != nil {
|
||||
@@ -147,7 +140,7 @@ func CreateClient() (clientUUID, token string, err error) {
|
||||
return "", "", err
|
||||
}
|
||||
if err := tasks.AddDefaultOnClientUUID(clientUUID); err != nil {
|
||||
log.Println("Failed to apply default-on ping tasks to new client:", err)
|
||||
logger.ErrorArgs("clients", "Failed to apply default-on ping tasks to new client:", err)
|
||||
}
|
||||
return clientUUID, token, nil
|
||||
}
|
||||
@@ -172,7 +165,7 @@ func CreateClientWithName(name string) (clientUUID, token string, err error) {
|
||||
return "", "", err
|
||||
}
|
||||
if err := tasks.AddDefaultOnClientUUID(clientUUID); err != nil {
|
||||
log.Println("Failed to apply default-on ping tasks to new client:", err)
|
||||
logger.ErrorArgs("clients", "Failed to apply default-on ping tasks to new client:", err)
|
||||
}
|
||||
return clientUUID, token, nil
|
||||
}
|
||||
|
||||
+76
-65
@@ -4,7 +4,6 @@ import (
|
||||
"archive/zip"
|
||||
"fmt"
|
||||
"io"
|
||||
"log"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
@@ -13,9 +12,10 @@ import (
|
||||
|
||||
"github.com/komari-monitor/komari/cmd/flags"
|
||||
"github.com/komari-monitor/komari/database/models"
|
||||
"github.com/komari-monitor/komari/pkg/config"
|
||||
"github.com/komari-monitor/komari/pkg/migrations"
|
||||
logutil "github.com/komari-monitor/komari/utils/log"
|
||||
"github.com/komari-monitor/komari/internal/config"
|
||||
"github.com/komari-monitor/komari/internal/migrations"
|
||||
"github.com/komari-monitor/komari/internal/sqlitetune"
|
||||
logger "github.com/komari-monitor/komari/utils/log"
|
||||
"gorm.io/driver/sqlite"
|
||||
"gorm.io/gorm"
|
||||
)
|
||||
@@ -176,6 +176,13 @@ var (
|
||||
// 也避免额外的裸文件依赖。
|
||||
const SystemVersionKey = "system_version"
|
||||
|
||||
const (
|
||||
mainSQLiteBusyTimeout = 5 * time.Second
|
||||
mainSQLiteCacheSizeKB = 8 * 1024
|
||||
mainSQLiteWALAutoCheckpoint = 256
|
||||
mainSQLiteJournalSizeLimit = 1 << 20
|
||||
)
|
||||
|
||||
// versionID 是当前构建的版本标识,由 SetVersionID 在 Initialize 前注入。
|
||||
var versionID string
|
||||
|
||||
@@ -200,7 +207,7 @@ func resolveDatabaseFile() string {
|
||||
}
|
||||
|
||||
// backupOnVersionUpgrade 在检测到版本升级时,把当前 ./data 打包到
|
||||
// ./backup/upgrade-{time}.zip,便于升级(含 metrics 迁移)异常时回滚。
|
||||
// ./data/backup/upgrade-{time}.zip,便于升级(含 metrics 迁移)异常时回滚。
|
||||
//
|
||||
// 版本标识存放于配置库(configs 表,键 system_version),因此本函数必须在
|
||||
// config.SetDb 之后、一次性 metrics 迁移(InitStores)之前调用。
|
||||
@@ -240,18 +247,19 @@ func backupOnVersionUpgrade() {
|
||||
instance.Exec("PRAGMA wal_checkpoint(TRUNCATE);")
|
||||
}
|
||||
|
||||
if err := os.MkdirAll("./backup", 0755); err != nil {
|
||||
log.Printf("[upgrade-backup] failed to create backup dir: %v", err)
|
||||
backupDir := filepath.Join(".", "data", "backup")
|
||||
if err := os.MkdirAll(backupDir, 0755); err != nil {
|
||||
logger.Errorf("dbcore", "[upgrade-backup] failed to create backup dir: %v", err)
|
||||
return
|
||||
}
|
||||
tsName := time.Now().UTC().Format("20060102-150405")
|
||||
bakPath := filepath.Join("./backup", fmt.Sprintf("upgrade-%s.zip", tsName))
|
||||
bakPath := filepath.Join(backupDir, fmt.Sprintf("upgrade-%s.zip", tsName))
|
||||
backupZipPath := filepath.Join(".", "data", "backup.zip")
|
||||
if zipErr := zipDirectoryExcluding("./data", bakPath, map[string]struct{}{backupZipPath: {}}); zipErr != nil {
|
||||
log.Printf("[upgrade-backup] failed to backup ./data before upgrade (from %q to %q): %v", prevVersion, versionID, zipErr)
|
||||
if zipErr := zipDirectoryExcluding("./data", bakPath, map[string]struct{}{backupZipPath: {}, backupDir: {}}); zipErr != nil {
|
||||
logger.Errorf("dbcore", "[upgrade-backup] failed to backup ./data before upgrade (from %q to %q): %v", prevVersion, versionID, zipErr)
|
||||
return
|
||||
}
|
||||
log.Printf("[upgrade-backup] ./data backed up to %s before upgrade (from %q to %q)", bakPath, prevVersion, versionID)
|
||||
logger.Infof("dbcore", "[upgrade-backup] ./data backed up to %s before upgrade (from %q to %q)", bakPath, prevVersion, versionID)
|
||||
|
||||
writeVersionMarker()
|
||||
}
|
||||
@@ -259,7 +267,7 @@ func backupOnVersionUpgrade() {
|
||||
// writeVersionMarker 将当前 versionID 写入配置库。
|
||||
func writeVersionMarker() {
|
||||
if err := config.Set(SystemVersionKey, versionID); err != nil {
|
||||
log.Printf("[upgrade-backup] failed to persist version marker: %v", err)
|
||||
logger.Errorf("dbcore", "[upgrade-backup] failed to persist version marker: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -268,7 +276,7 @@ func buildSQLiteDSN(databaseFile string) string {
|
||||
databaseFile = "./data/komari.db"
|
||||
}
|
||||
|
||||
params := "_busy_timeout=5000&_txlock=immediate&_journal_mode=WAL&_synchronous=NORMAL"
|
||||
params := fmt.Sprintf("_busy_timeout=%d&_txlock=immediate", mainSQLiteBusyTimeout.Milliseconds())
|
||||
separator := "?"
|
||||
if strings.Contains(databaseFile, "?") {
|
||||
separator = "&"
|
||||
@@ -285,6 +293,19 @@ func buildSQLiteDSN(databaseFile string) string {
|
||||
return "file:" + filepath.ToSlash(databaseFile) + separator + params
|
||||
}
|
||||
|
||||
func mainSQLiteOptions() sqlitetune.Options {
|
||||
return sqlitetune.Options{
|
||||
BusyTimeout: mainSQLiteBusyTimeout,
|
||||
CacheSizeKB: mainSQLiteCacheSizeKB,
|
||||
MMapSizeBytes: 0,
|
||||
TempStoreMemory: false,
|
||||
CacheSpill: true,
|
||||
WALAutoCheckpoint: mainSQLiteWALAutoCheckpoint,
|
||||
JournalSizeLimitBytes: mainSQLiteJournalSizeLimit,
|
||||
Synchronous: sqlitetune.SynchronousNormal,
|
||||
}
|
||||
}
|
||||
|
||||
// Initialize 显式初始化数据库连接与表结构,仅执行一次。
|
||||
// 与 GetDBInstance 不同,Initialize 返回错误而非直接退出进程,
|
||||
// 便于启动生命周期统一处理错误、以及在测试/CLI 命令中做隔离。
|
||||
@@ -300,7 +321,7 @@ func Initialize() error {
|
||||
// 需要错误处理的启动流程应优先调用 Initialize()。
|
||||
func GetDBInstance() *gorm.DB {
|
||||
if err := Initialize(); err != nil {
|
||||
log.Fatalf("Failed to initialize database: %v", err)
|
||||
logger.Fatalf("dbcore", "Failed to initialize database: %v", err)
|
||||
}
|
||||
return instance
|
||||
}
|
||||
@@ -324,42 +345,43 @@ func doInitialize() error {
|
||||
func() {
|
||||
backupZipPath := filepath.Join(".", "data", "backup.zip")
|
||||
if _, statErr := os.Stat(backupZipPath); statErr == nil {
|
||||
// 4. 把除了 ./data/backup.zip 之外的所有文件压缩到 ./backup/{time}.zip
|
||||
if err := os.MkdirAll("./backup", 0755); err != nil {
|
||||
log.Printf("[restore] failed to create backup dir: %v", err)
|
||||
// 4. 将当前数据快照保存到 ./data/backup/,并保留已有归档。
|
||||
backupDir := filepath.Join(".", "data", "backup")
|
||||
if err := os.MkdirAll(backupDir, 0755); err != nil {
|
||||
logger.Errorf("dbcore", "[restore] failed to create backup dir: %v", err)
|
||||
} else {
|
||||
tsName := time.Now().UTC().Format("20060102-150405")
|
||||
bakPath := filepath.Join("./backup", fmt.Sprintf("%s.zip", tsName))
|
||||
if zipErr := zipDirectoryExcluding("./data", bakPath, map[string]struct{}{backupZipPath: {}}); zipErr != nil {
|
||||
log.Printf("[restore] failed to zip current data: %v", zipErr)
|
||||
bakPath := filepath.Join(backupDir, fmt.Sprintf("pre-restore-%s.zip", tsName))
|
||||
if zipErr := zipDirectoryExcluding("./data", bakPath, map[string]struct{}{backupZipPath: {}, backupDir: {}}); zipErr != nil {
|
||||
logger.Errorf("dbcore", "[restore] failed to zip current data: %v", zipErr)
|
||||
} else {
|
||||
log.Printf("[restore] current data zipped to %s", bakPath)
|
||||
logger.Infof("dbcore", "[restore] current data zipped to %s", bakPath)
|
||||
}
|
||||
}
|
||||
|
||||
// 5. 删除除了 ./data/backup.zip 之外的所有文件
|
||||
if delErr := removeAllInDirExcept("./data", map[string]struct{}{backupZipPath: {}}); delErr != nil {
|
||||
log.Printf("[restore] failed to cleanup data dir: %v", delErr)
|
||||
// 5. 删除数据文件,但保留归档目录和待恢复的 backup.zip。
|
||||
if delErr := removeAllInDirExcept("./data", map[string]struct{}{backupZipPath: {}, backupDir: {}}); delErr != nil {
|
||||
logger.Errorf("dbcore", "[restore] failed to cleanup data dir: %v", delErr)
|
||||
}
|
||||
|
||||
// 6. 解压 ./data/backup.zip 到 ./data
|
||||
if unzipErr := unzipToDir(backupZipPath, "./data"); unzipErr != nil {
|
||||
log.Printf("[restore] failed to unzip backup into data: %v", unzipErr)
|
||||
logger.Errorf("dbcore", "[restore] failed to unzip backup into data: %v", unzipErr)
|
||||
} else {
|
||||
log.Printf("[restore] backup.zip extracted to ./data")
|
||||
logger.Infof("dbcore", "[restore] backup.zip extracted to ./data")
|
||||
}
|
||||
|
||||
// 7. 删除 ./data/backup.zip
|
||||
if rmErr := os.Remove(backupZipPath); rmErr != nil {
|
||||
log.Printf("[restore] failed to remove backup.zip: %v", rmErr)
|
||||
logger.Errorf("dbcore", "[restore] failed to remove backup.zip: %v", rmErr)
|
||||
} else {
|
||||
log.Printf("[restore] backup.zip removed")
|
||||
logger.Infof("dbcore", "[restore] backup.zip removed")
|
||||
}
|
||||
// 8. 删除标记
|
||||
if rmErr := os.Remove("./data/komari-backup-markup"); rmErr != nil {
|
||||
log.Printf("[restore] failed to remove komari-backup-markup: %v", rmErr)
|
||||
logger.Errorf("dbcore", "[restore] failed to remove komari-backup-markup: %v", rmErr)
|
||||
} else {
|
||||
log.Printf("[restore] komari-backup-markup removed")
|
||||
logger.Infof("dbcore", "[restore] komari-backup-markup removed")
|
||||
}
|
||||
}
|
||||
}()
|
||||
@@ -372,48 +394,36 @@ func doInitialize() error {
|
||||
}
|
||||
|
||||
logConfig := &gorm.Config{
|
||||
Logger: logutil.NewGormLogger(),
|
||||
Logger: logger.NewGormLogger(),
|
||||
NowFunc: func() time.Time { return time.Now().UTC() },
|
||||
}
|
||||
|
||||
// 根据数据库类型选择不同的连接方式
|
||||
switch flags.ApplyDatabaseTypeNormalization() {
|
||||
case flags.DatabaseTypeSQLite:
|
||||
// SQLite 连接
|
||||
// 通过 DSN 传入 _busy_timeout / _txlock 等参数,确保连接池中的每一条连接
|
||||
// 都生效:
|
||||
// - _busy_timeout=5000:遇到写锁时最多等待 5s 再返回,避免瞬时
|
||||
// "database is locked" 直接失败(仅靠后续 PRAGMA Exec 只作用于
|
||||
// 当时执行该语句的单条连接,池内其它连接不生效)。
|
||||
// - _txlock=immediate:事务一开始即获取写锁,避免「先 SELECT 后写」
|
||||
// 的锁升级在并发写入下产生死锁式的立即 SQLITE_BUSY。
|
||||
// - _journal_mode=WAL / _synchronous=NORMAL:与下方 PRAGMA 保持一致,
|
||||
// 在 DSN 层为所有连接预设。
|
||||
// _txlock=immediate lets writes acquire their lock before reads can
|
||||
// turn into a lock-upgrade conflict. sqlitetune applies the remaining
|
||||
// per-connection PRAGMAs whenever database/sql opens a connection.
|
||||
dsn := buildSQLiteDSN(flags.DatabaseFile)
|
||||
instance, err = gorm.Open(sqlite.Open(dsn), logConfig)
|
||||
sqlDB, dbErr := sqlitetune.Open(dsn, mainSQLiteOptions())
|
||||
if dbErr != nil {
|
||||
return fmt.Errorf("open SQLite connection pool: %w", dbErr)
|
||||
}
|
||||
// SQLite has one writer. Keeping exactly one durable connection also
|
||||
// keeps connection-local cache and WAL limits stable for the main DB.
|
||||
sqlDB.SetMaxOpenConns(1)
|
||||
sqlDB.SetMaxIdleConns(1)
|
||||
sqlDB.SetConnMaxLifetime(0)
|
||||
sqlDB.SetConnMaxIdleTime(0)
|
||||
|
||||
instance, err = gorm.Open(sqlite.New(sqlite.Config{Conn: sqlDB}), logConfig)
|
||||
if err != nil {
|
||||
_ = sqlDB.Close()
|
||||
return fmt.Errorf("failed to connect to SQLite3 database: %w", err)
|
||||
}
|
||||
log.Printf("Using SQLite database file: %s", flags.DatabaseFile)
|
||||
if sqlDB, dbErr := instance.DB(); dbErr == nil {
|
||||
// SQLite 同一时刻只允许一个写者;限制连接数可避免连接池层面的写写竞争。
|
||||
// 负载历史每分钟会执行包含读和写的事务,若连接池允许多个连接,容易与
|
||||
// ping 结果等短写入撞锁并导致整批负载记录回滚。
|
||||
sqlDB.SetMaxOpenConns(1)
|
||||
sqlDB.SetMaxIdleConns(1)
|
||||
sqlDB.SetConnMaxLifetime(0)
|
||||
} else {
|
||||
log.Printf("Failed to access underlying sql.DB for SQLite tuning: %v", dbErr)
|
||||
if err := instance.Exec("PRAGMA wal_checkpoint(TRUNCATE)").Error; err != nil {
|
||||
logger.Errorf("dbcore", "Failed to checkpoint SQLite WAL at startup: %v", err)
|
||||
}
|
||||
instance.Exec("PRAGMA wal = ON;")
|
||||
if err := instance.Exec("PRAGMA journal_mode = WAL;").Error; err != nil {
|
||||
log.Printf("Failed to enable WAL mode for SQLite: %v", err)
|
||||
}
|
||||
instance.Exec("PRAGMA synchronous = NORMAL;")
|
||||
instance.Exec("PRAGMA busy_timeout = 5000;")
|
||||
instance.Exec("PRAGMA cache_size = -65536;")
|
||||
instance.Exec("PRAGMA temp_store = MEMORY;")
|
||||
instance.Exec("PRAGMA wal_checkpoint(TRUNCATE);")
|
||||
default:
|
||||
return fmt.Errorf("unsupported database type: %s (supported: %s)", flags.DatabaseType, flags.SupportedDatabaseTypes())
|
||||
}
|
||||
@@ -422,7 +432,7 @@ func doInitialize() error {
|
||||
}
|
||||
config.SetDb(instance)
|
||||
|
||||
// 配置库就绪后、执行后续 AutoMigrate 与一次性 metrics 迁移之前:
|
||||
// 配置库就绪后、执行后续 AutoMigrate 之前:
|
||||
// 基于配置中的版本标记检测升级并自动备份 ./data,便于回滚。
|
||||
backupOnVersionUpgrade()
|
||||
|
||||
@@ -431,7 +441,7 @@ func doInitialize() error {
|
||||
// 注意:负载/GPU/ping 历史监控数据运行期全部走 metric store(默认 SQLite
|
||||
// ./data/metrics.db,或配置的 MySQL/PostgreSQL)。旧的 records /
|
||||
// records_long_term / gpu_records / ping_records 表不再建表、不再写入。
|
||||
// 若升级时旧表仍存在,会在 pkg/migrations.RunMetricStoreMigrations 中先导入再清理。
|
||||
// 若升级时旧表仍存在,管理员可通过升级向导显式导入并清理。
|
||||
// models.Record / models.PingRecord / models.GPURecord 结构体仍作为
|
||||
// metric store 的读写 DTO 和旧表导入 DTO 保留在 models 包中。
|
||||
|
||||
@@ -447,6 +457,7 @@ func doInitialize() error {
|
||||
&models.OidcProvider{},
|
||||
&models.MessageSenderProvider{},
|
||||
&models.ThemeConfiguration{},
|
||||
&models.PluginConfiguration{},
|
||||
)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to create tables: %w", err)
|
||||
@@ -454,13 +465,13 @@ func doInitialize() error {
|
||||
if err := instance.AutoMigrate(
|
||||
&models.Session{},
|
||||
); err != nil {
|
||||
log.Printf("Failed to create Session table, it may already exist: %v", err)
|
||||
logger.Errorf("dbcore", "Failed to create Session table, it may already exist: %v", err)
|
||||
}
|
||||
if err := instance.AutoMigrate(
|
||||
&models.Task{},
|
||||
&models.TaskResult{},
|
||||
); err != nil {
|
||||
log.Printf("Failed to create Task and TaskResult table, it may already exist: %v", err)
|
||||
logger.Errorf("dbcore", "Failed to create Task and TaskResult table, it may already exist: %v", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
|
||||
@@ -0,0 +1,99 @@
|
||||
package dbcore
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/komari-monitor/komari/internal/sqlitetune"
|
||||
)
|
||||
|
||||
func TestMainSQLiteConnectorAppliesBoundedSettings(t *testing.T) {
|
||||
databasePath := filepath.Join(t.TempDir(), "komari.db")
|
||||
db, err := sqlitetune.Open(buildSQLiteDSN(databasePath), mainSQLiteOptions())
|
||||
if err != nil {
|
||||
t.Fatalf("open SQLite connector: %v", err)
|
||||
}
|
||||
defer db.Close()
|
||||
|
||||
db.SetMaxOpenConns(1)
|
||||
db.SetMaxIdleConns(1)
|
||||
|
||||
if got := sqlitePragmaInt(t, db, "busy_timeout"); got != mainSQLiteBusyTimeout.Milliseconds() {
|
||||
t.Fatalf("busy_timeout = %d, want %d", got, mainSQLiteBusyTimeout.Milliseconds())
|
||||
}
|
||||
if got := sqlitePragmaInt(t, db, "synchronous"); got != 1 {
|
||||
t.Fatalf("synchronous = %d, want NORMAL (1)", got)
|
||||
}
|
||||
if got := sqlitePragmaInt(t, db, "cache_size"); got != -mainSQLiteCacheSizeKB {
|
||||
t.Fatalf("cache_size = %d, want %d", got, -mainSQLiteCacheSizeKB)
|
||||
}
|
||||
if got := sqlitePragmaInt(t, db, "mmap_size"); got != 0 {
|
||||
t.Fatalf("mmap_size = %d, want 0", got)
|
||||
}
|
||||
if got := sqlitePragmaInt(t, db, "temp_store"); got != 1 {
|
||||
t.Fatalf("temp_store = %d, want FILE (1)", got)
|
||||
}
|
||||
if got := sqlitePragmaInt(t, db, "cache_spill"); got == 0 {
|
||||
t.Fatal("cache_spill is disabled")
|
||||
}
|
||||
if got := sqlitePragmaInt(t, db, "wal_autocheckpoint"); got != mainSQLiteWALAutoCheckpoint {
|
||||
t.Fatalf("wal_autocheckpoint = %d, want %d", got, mainSQLiteWALAutoCheckpoint)
|
||||
}
|
||||
if got := sqlitePragmaInt(t, db, "journal_size_limit"); got != mainSQLiteJournalSizeLimit {
|
||||
t.Fatalf("journal_size_limit = %d, want %d", got, mainSQLiteJournalSizeLimit)
|
||||
}
|
||||
|
||||
var journalMode string
|
||||
if err := db.QueryRow("PRAGMA journal_mode").Scan(&journalMode); err != nil {
|
||||
t.Fatalf("read journal_mode: %v", err)
|
||||
}
|
||||
if !strings.EqualFold(journalMode, "wal") {
|
||||
t.Fatalf("journal_mode = %q, want WAL", journalMode)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMainSQLiteAutoCheckpointKeepsWALBounded(t *testing.T) {
|
||||
databasePath := filepath.Join(t.TempDir(), "komari.db")
|
||||
db, err := sqlitetune.Open(buildSQLiteDSN(databasePath), mainSQLiteOptions())
|
||||
if err != nil {
|
||||
t.Fatalf("open SQLite connector: %v", err)
|
||||
}
|
||||
defer db.Close()
|
||||
|
||||
db.SetMaxOpenConns(1)
|
||||
db.SetMaxIdleConns(1)
|
||||
if _, err := db.Exec("CREATE TABLE checkpoint_test (payload BLOB NOT NULL)"); err != nil {
|
||||
t.Fatalf("create checkpoint test table: %v", err)
|
||||
}
|
||||
|
||||
// This produces well beyond one auto-checkpoint interval. The checkpoint
|
||||
// below reports the WAL frame count observed before its own work begins.
|
||||
for range 1024 {
|
||||
if _, err := db.Exec("INSERT INTO checkpoint_test (payload) VALUES (zeroblob(4096))"); err != nil {
|
||||
t.Fatalf("insert checkpoint test row: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
var busy, logFrames, checkpointedFrames int
|
||||
if err := db.QueryRow("PRAGMA wal_checkpoint(PASSIVE)").Scan(&busy, &logFrames, &checkpointedFrames); err != nil {
|
||||
t.Fatalf("inspect automatic checkpoint: %v", err)
|
||||
}
|
||||
if busy != 0 {
|
||||
t.Fatalf("automatic checkpoint left a busy WAL: %d", busy)
|
||||
}
|
||||
if logFrames >= 2*mainSQLiteWALAutoCheckpoint {
|
||||
t.Fatalf("WAL retained %d frames after %d-frame auto-checkpoint interval", logFrames, mainSQLiteWALAutoCheckpoint)
|
||||
}
|
||||
}
|
||||
|
||||
func sqlitePragmaInt(t *testing.T, db *sql.DB, pragma string) int64 {
|
||||
t.Helper()
|
||||
|
||||
var value int64
|
||||
if err := db.QueryRow("PRAGMA " + pragma).Scan(&value); err != nil {
|
||||
t.Fatalf("read %s: %v", pragma, err)
|
||||
}
|
||||
return value
|
||||
}
|
||||
@@ -5,15 +5,6 @@ import (
|
||||
"github.com/komari-monitor/komari/database/models"
|
||||
)
|
||||
|
||||
func GetAllMessageSenderConfigs() []models.MessageSenderProvider {
|
||||
db := dbcore.GetDBInstance()
|
||||
var result []models.MessageSenderProvider
|
||||
if err := db.Find(&result).Error; err != nil {
|
||||
return nil
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
func GetMessageSenderConfigByName(name string) (*models.MessageSenderProvider, error) {
|
||||
db := dbcore.GetDBInstance()
|
||||
var config models.MessageSenderProvider
|
||||
|
||||
@@ -1,999 +0,0 @@
|
||||
package metricstore
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"log"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/komari-monitor/komari/database/models"
|
||||
"github.com/komari-monitor/komari/pkg/config"
|
||||
"github.com/komari-monitor/komari/pkg/metric"
|
||||
)
|
||||
|
||||
var (
|
||||
store *metric.Store
|
||||
storeFingerprint string
|
||||
storeMu sync.RWMutex
|
||||
storeOnce sync.Once
|
||||
storeOperations = newStoreOperationGate()
|
||||
compactAt int
|
||||
)
|
||||
|
||||
var ErrCompactInProgress = errors.New("metric store compact already in progress")
|
||||
|
||||
const (
|
||||
// DefaultRollupRawRetention keeps a short hot raw window; older samples are
|
||||
// served from rollups after compaction.
|
||||
DefaultRollupRawRetention = 15 * time.Minute
|
||||
DefaultRollupFinestTier = time.Minute
|
||||
)
|
||||
|
||||
// MetricStoreConfig 保存 metric store 配置。
|
||||
//
|
||||
// 注意:metric store 现在始终启用(旧的 metric_store_enabled 开关已废弃)。
|
||||
// 未显式配置时默认使用 SQLite(./data/metrics.db)。
|
||||
type MetricStoreConfig struct {
|
||||
Driver string `json:"metric_db_driver" default:"sqlite"` // 数据库类型: sqlite, mysql, postgresql
|
||||
DSN string `json:"metric_db_dsn" default:"./data/metrics.db"` // 数据库连接串
|
||||
DownsamplingEnabled bool `json:"metric_downsampling_enabled" default:"false"` // 是否启用分层降采样
|
||||
TablePrefix string `json:"metric_table_prefix" default:"metric_"` // 表名前缀
|
||||
MaxOpenConns int `json:"metric_max_open_conns" default:"25"` // 最大连接数
|
||||
MaxIdleConns int `json:"metric_max_idle_conns" default:"5"` // 最大空闲连接数
|
||||
}
|
||||
|
||||
// MetricStoreConfigKeys 配置键
|
||||
//
|
||||
// MetricStoreEnabledKey 已废弃:metric store 始终启用,保留常量仅用于清理旧配置。
|
||||
const (
|
||||
MetricStoreEnabledKey = "metric_store_enabled" // Deprecated: metric store 始终启用
|
||||
MetricDBDriverKey = "metric_db_driver"
|
||||
MetricDBDSNKey = "metric_db_dsn"
|
||||
MetricDownsamplingEnabledKey = "metric_downsampling_enabled"
|
||||
MetricTablePrefixKey = "metric_table_prefix"
|
||||
MetricMaxOpenConnsKey = "metric_max_open_conns"
|
||||
MetricMaxIdleConnsKey = "metric_max_idle_conns"
|
||||
// MigrationTargetKey 记录上一次成功完成启动迁移的目标指纹(driver+dsn)。
|
||||
// 当目标发生变化(例如从 SQLite 切换到 MySQL/PostgreSQL)时,启动迁移会
|
||||
// 重新执行,把上一个目标库的数据搬运到新的目标 metrics 库。
|
||||
MigrationTargetKey = "metric_migration_target"
|
||||
)
|
||||
|
||||
// buildMetricConfig 根据 MetricStoreConfig 构造底层 metric.Config。
|
||||
// autoMigrate 控制是否在 Open 时自动建表:正式初始化/热加载时为 true,
|
||||
// 仅做连接测试时为 false(不写入 schema,避免对目标库产生副作用)。
|
||||
func buildMetricConfig(cfg *MetricStoreConfig, autoMigrate bool) (metric.Config, error) {
|
||||
driver := ResolveDriverFromConfig(cfg.Driver, cfg.DSN)
|
||||
|
||||
tablePrefix := cfg.TablePrefix
|
||||
if tablePrefix == "" {
|
||||
tablePrefix = "metric_"
|
||||
}
|
||||
opts := []metric.Option{
|
||||
metric.WithTablePrefix(tablePrefix),
|
||||
metric.WithAutoMigrate(autoMigrate),
|
||||
}
|
||||
if cfg.DownsamplingEnabled {
|
||||
opts = append(opts, metric.WithRollupPolicy(defaultRollupPolicy()))
|
||||
}
|
||||
|
||||
switch driver {
|
||||
case metric.DriverSQLite:
|
||||
dsn := cfg.DSN
|
||||
if dsn == "" || dsn == "./data/metrics.db" {
|
||||
// 注意:刻意不使用 cache=shared。SQLite 共享缓存模式使用表级锁,
|
||||
// 当一个连接持有读锁、另一个连接尝试写入时会立即返回
|
||||
// SQLITE_LOCKED("database table is locked"),且 busy_timeout
|
||||
// 对共享缓存的表级锁无效,迁移期间与前台查询/实时写入并发时必然报错。
|
||||
// _txlock=immediate 让写事务开始即获取写锁,避免锁升级死锁。
|
||||
dsn = "file:./data/metrics.db?mode=rwc&_txlock=immediate"
|
||||
} else {
|
||||
// 用户自定义 DSN 时,剥离 cache=shared,避免上述表级锁问题。
|
||||
dsn = stripSharedCache(dsn)
|
||||
}
|
||||
// SQLite 串行化写入:固定单写连接以避免 "database is locked" 竞争,
|
||||
// 同时启用独立的 WAL 只读连接池提升前台查询并发(写仍走单主连接)。
|
||||
// 这里刻意忽略 cfg.MaxOpenConns/MaxIdleConns —— 对 SQLite 而言多写连接
|
||||
// 只会引入锁竞争而非提升吞吐。
|
||||
opts = append(opts,
|
||||
metric.WithMaxOpenConns(1),
|
||||
metric.WithMaxIdleConns(1),
|
||||
metric.WithSQLiteReadPool(4),
|
||||
)
|
||||
return metric.SQLite(dsn, opts...), nil
|
||||
case metric.DriverMySQL:
|
||||
opts = append(opts,
|
||||
metric.WithMaxOpenConns(cfg.MaxOpenConns),
|
||||
metric.WithMaxIdleConns(cfg.MaxIdleConns),
|
||||
)
|
||||
return metric.MySQL(cfg.DSN, opts...), nil
|
||||
case metric.DriverPostgreSQL:
|
||||
opts = append(opts,
|
||||
metric.WithMaxOpenConns(cfg.MaxOpenConns),
|
||||
metric.WithMaxIdleConns(cfg.MaxIdleConns),
|
||||
)
|
||||
return metric.PostgreSQL(cfg.DSN, opts...), nil
|
||||
default:
|
||||
return metric.Config{}, fmt.Errorf("unsupported metric database driver: %s", cfg.Driver)
|
||||
}
|
||||
}
|
||||
|
||||
func defaultRollupPolicy() metric.RollupPolicy {
|
||||
return metric.RollupPolicy{
|
||||
RawRetention: DefaultRollupRawRetention,
|
||||
Tiers: []metric.RollupTier{
|
||||
{Interval: DefaultRollupFinestTier, Retention: 48 * time.Hour},
|
||||
{Interval: 5 * time.Minute, Retention: 14 * 24 * time.Hour},
|
||||
{Interval: time.Hour, Retention: 14 * 24 * time.Hour},
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
// ResolveDriverFromConfig 根据 DSN 自动推断 metrics 数据库类型;当 DSN 不能可靠
|
||||
// 识别时回退到旧配置中的 driver,以兼容已有配置和非常规 DSN。
|
||||
func ResolveDriverFromConfig(configuredDriver, dsn string) metric.Driver {
|
||||
if driver, ok := InferDriverFromDSN(dsn); ok {
|
||||
return driver
|
||||
}
|
||||
|
||||
switch driver := metric.Driver(strings.ToLower(strings.TrimSpace(configuredDriver))); driver {
|
||||
case metric.DriverSQLite, metric.DriverMySQL, metric.DriverPostgreSQL:
|
||||
return driver
|
||||
default:
|
||||
return metric.DriverSQLite
|
||||
}
|
||||
}
|
||||
|
||||
// InferDriverFromDSN 尽量根据常见 DSN 格式推断数据库类型。
|
||||
// 返回 ok=false 表示格式不够明确,调用方应使用已有配置作为兜底。
|
||||
func InferDriverFromDSN(dsn string) (metric.Driver, bool) {
|
||||
raw := strings.TrimSpace(dsn)
|
||||
if raw == "" {
|
||||
return metric.DriverSQLite, true
|
||||
}
|
||||
lower := strings.ToLower(raw)
|
||||
|
||||
// PostgreSQL URL DSN: postgres://... 或 postgresql://...
|
||||
if strings.HasPrefix(lower, "postgres://") || strings.HasPrefix(lower, "postgresql://") {
|
||||
return metric.DriverPostgreSQL, true
|
||||
}
|
||||
|
||||
// SQLite 常见文件/内存 DSN。
|
||||
if raw == ":memory:" || strings.HasPrefix(lower, "file:") || strings.HasPrefix(lower, "sqlite://") || strings.HasPrefix(lower, "sqlite3://") {
|
||||
return metric.DriverSQLite, true
|
||||
}
|
||||
|
||||
// MySQL URL(虽然 go-sql-driver/mysql 原生 DSN 通常不是 URL,但这里用于给出
|
||||
// 类型推断;连接测试仍会校验 DSN 是否可被驱动接受)。
|
||||
if strings.HasPrefix(lower, "mysql://") {
|
||||
return metric.DriverMySQL, true
|
||||
}
|
||||
|
||||
// PostgreSQL 关键字/值 DSN: host=... user=... dbname=...
|
||||
if looksLikePostgreSQLKeyValueDSN(lower) {
|
||||
return metric.DriverPostgreSQL, true
|
||||
}
|
||||
|
||||
// go-sql-driver/mysql DSN: user:pass@tcp(host:3306)/db、user@unix(...)/db、user:pass@/db 等。
|
||||
if looksLikeMySQLDSN(lower) {
|
||||
return metric.DriverMySQL, true
|
||||
}
|
||||
|
||||
// SQLite 路径:./data/metrics.db、/var/lib/metrics.sqlite3、metrics.sqlite 等。
|
||||
if looksLikeSQLitePath(lower) {
|
||||
return metric.DriverSQLite, true
|
||||
}
|
||||
|
||||
return "", false
|
||||
}
|
||||
|
||||
func looksLikePostgreSQLKeyValueDSN(lower string) bool {
|
||||
if !strings.Contains(lower, "=") || strings.Contains(lower, "://") {
|
||||
return false
|
||||
}
|
||||
keys := []string{"host=", "user=", "password=", "dbname=", "port=", "sslmode="}
|
||||
matched := 0
|
||||
for _, key := range keys {
|
||||
if strings.Contains(lower, key) {
|
||||
matched++
|
||||
}
|
||||
}
|
||||
// dbname= 基本是 PostgreSQL libpq DSN 的强特征;否则至少匹配两个常见键。
|
||||
return strings.Contains(lower, "dbname=") || matched >= 2
|
||||
}
|
||||
|
||||
func looksLikeMySQLDSN(lower string) bool {
|
||||
if strings.Contains(lower, "://") || strings.Contains(lower, " ") {
|
||||
return false
|
||||
}
|
||||
if strings.Contains(lower, "@tcp(") || strings.Contains(lower, "@unix(") || strings.Contains(lower, "@/") {
|
||||
return true
|
||||
}
|
||||
// user:pass@host/db、user@host/db 这类虽然不是推荐格式,但也明显偏 MySQL。
|
||||
return strings.Contains(lower, "@") && strings.Contains(lower, "/")
|
||||
}
|
||||
|
||||
func looksLikeSQLitePath(lower string) bool {
|
||||
path := lower
|
||||
if idx := strings.IndexAny(path, "?"); idx >= 0 {
|
||||
path = path[:idx]
|
||||
}
|
||||
return strings.HasSuffix(path, ".db") || strings.HasSuffix(path, ".sqlite") || strings.HasSuffix(path, ".sqlite3")
|
||||
}
|
||||
|
||||
// stripSharedCache 从 SQLite DSN 中移除 cache=shared 参数,避免共享缓存模式下的
|
||||
// 表级锁(SQLITE_LOCKED "database table is locked")。其它参数保持不变。
|
||||
func stripSharedCache(dsn string) string {
|
||||
if !strings.Contains(dsn, "cache=shared") {
|
||||
return dsn
|
||||
}
|
||||
idx := strings.Index(dsn, "?")
|
||||
if idx < 0 {
|
||||
return dsn
|
||||
}
|
||||
base := dsn[:idx]
|
||||
query := dsn[idx+1:]
|
||||
parts := strings.Split(query, "&")
|
||||
kept := parts[:0]
|
||||
for _, p := range parts {
|
||||
if p == "cache=shared" {
|
||||
continue
|
||||
}
|
||||
kept = append(kept, p)
|
||||
}
|
||||
if len(kept) == 0 {
|
||||
return base
|
||||
}
|
||||
return base + "?" + strings.Join(kept, "&")
|
||||
}
|
||||
|
||||
// openStore 按配置打开 metric store 并创建指标定义。
|
||||
func openStore(ctx context.Context, cfg *MetricStoreConfig) (*metric.Store, error) {
|
||||
return openStoreWithDefaultRetention(ctx, cfg, defaultBuiltinMetricRetentionDays)
|
||||
}
|
||||
|
||||
func openStoreWithDefaultRetention(ctx context.Context, cfg *MetricStoreConfig, defaultRetentionDays int) (*metric.Store, error) {
|
||||
metricCfg, err := buildMetricConfig(cfg, true)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
s, err := metric.Open(ctx, metricCfg)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to open metric store: %w", err)
|
||||
}
|
||||
|
||||
if err := createMetricDefinitionsWithDefaultRetention(ctx, s, defaultRetentionDays); err != nil {
|
||||
s.Close()
|
||||
return nil, fmt.Errorf("failed to create metric definitions: %w", err)
|
||||
}
|
||||
|
||||
return s, nil
|
||||
}
|
||||
|
||||
// OpenStore opens an isolated metric store using the supplied configuration.
|
||||
// It is used by the pre-start upgrade flow before the process-wide store is
|
||||
// initialized. The caller owns the returned store and must close it.
|
||||
func OpenStore(ctx context.Context, cfg *MetricStoreConfig) (*metric.Store, error) {
|
||||
return openStore(ctx, cfg)
|
||||
}
|
||||
|
||||
// OpenStoreForMigration opens an isolated target and uses the legacy data span
|
||||
// as the initial retention for definitions that do not exist yet. Existing
|
||||
// definitions keep their configured retention, including an explicit zero.
|
||||
func OpenStoreForMigration(ctx context.Context, cfg *MetricStoreConfig, legacyRetentionDays int) (*metric.Store, error) {
|
||||
if legacyRetentionDays < defaultBuiltinMetricRetentionDays {
|
||||
legacyRetentionDays = defaultBuiltinMetricRetentionDays
|
||||
}
|
||||
return openStoreWithDefaultRetention(ctx, cfg, legacyRetentionDays)
|
||||
}
|
||||
|
||||
// TestConnection 使用给定配置尝试连接 metrics 数据库(不影响当前运行的 store)。
|
||||
// 仅打开连接并 Ping,不执行自动建表,连接成功后立即关闭。失败时返回可读错误。
|
||||
func TestConnection(ctx context.Context, cfg *MetricStoreConfig) error {
|
||||
metricCfg, err := buildMetricConfig(cfg, false)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
s, err := metric.Open(ctx, metricCfg)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer s.Close()
|
||||
|
||||
return s.Ping(ctx)
|
||||
}
|
||||
|
||||
// InitializeStore 初始化 metric store(启动时调用,仅执行一次)。
|
||||
func InitializeStore() error {
|
||||
var initErr error
|
||||
storeOnce.Do(func() {
|
||||
cfg, err := config.GetManyAs[MetricStoreConfig]()
|
||||
if err != nil {
|
||||
initErr = fmt.Errorf("failed to load metric store config: %w", err)
|
||||
return
|
||||
}
|
||||
|
||||
// metric store 始终启用;未配置时默认 SQLite(./data/metrics.db)。
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||||
defer cancel()
|
||||
|
||||
s, err := openStore(ctx, cfg)
|
||||
if err != nil {
|
||||
initErr = err
|
||||
return
|
||||
}
|
||||
|
||||
storeMu.Lock()
|
||||
store = s
|
||||
storeFingerprint = targetFingerprint(cfg)
|
||||
storeMu.Unlock()
|
||||
clearStoreClosing()
|
||||
|
||||
log.Printf("Metric store initialized successfully (driver=%s)", ResolveDriverFromConfig(cfg.Driver, cfg.DSN))
|
||||
})
|
||||
|
||||
return initErr
|
||||
}
|
||||
|
||||
// Reload 根据最新配置热重载 metric store,无需重启进程。
|
||||
// metric store 始终启用:用新配置打开并建表(内部已 Ping 校验连接),
|
||||
// 成功后再替换运行中的 store,最后关闭旧实例。任何失败都会保留旧 store 不变。
|
||||
//
|
||||
// 注意:Reload 只切换运行中的连接,不会把旧目标(如 SQLite)中的历史数据
|
||||
// 搬运到新目标(如 MySQL/PostgreSQL)。跨库数据迁移由启动迁移
|
||||
// (RunStartupMigration)在下次启动时按目标指纹自动完成。
|
||||
func Reload(ctx context.Context) error {
|
||||
if err := storeOperations.Acquire(ctx); err != nil {
|
||||
return fmt.Errorf("wait for metric store operations before reload: %w", err)
|
||||
}
|
||||
defer storeOperations.Release()
|
||||
if isStoreClosing() {
|
||||
return ErrStoreBusy
|
||||
}
|
||||
|
||||
cfg, err := config.GetManyAs[MetricStoreConfig]()
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to load metric store config: %w", err)
|
||||
}
|
||||
|
||||
// 用新配置打开并建表(内部已 Ping 校验连接)。
|
||||
s, err := openStore(ctx, cfg)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
storeMu.Lock()
|
||||
old := store
|
||||
store = s
|
||||
storeFingerprint = targetFingerprint(cfg)
|
||||
storeMu.Unlock()
|
||||
|
||||
if old != nil {
|
||||
if cerr := old.Close(); cerr != nil {
|
||||
log.Printf("Failed to close previous metric store on reload: %v", cerr)
|
||||
}
|
||||
}
|
||||
|
||||
log.Printf("Metric store reloaded successfully (driver=%s)", ResolveDriverFromConfig(cfg.Driver, cfg.DSN))
|
||||
return nil
|
||||
}
|
||||
|
||||
// GetStore 获取 metric store 实例(如果未启用返回 nil)
|
||||
func GetStore() *metric.Store {
|
||||
storeMu.RLock()
|
||||
defer storeMu.RUnlock()
|
||||
return store
|
||||
}
|
||||
|
||||
// RetentionSummary is the compatibility view of all persisted metric policies.
|
||||
type RetentionSummary struct {
|
||||
AllPositive bool
|
||||
MaxDays int
|
||||
}
|
||||
|
||||
// GetRetentionSummary aggregates the active store's metric definitions. An
|
||||
// empty definition set is not considered record-enabled.
|
||||
func GetRetentionSummary(ctx context.Context) (RetentionSummary, error) {
|
||||
s := GetStore()
|
||||
if s == nil {
|
||||
return RetentionSummary{}, fmt.Errorf("metric store not initialized")
|
||||
}
|
||||
defs, err := s.ListMetrics(ctx)
|
||||
if err != nil {
|
||||
return RetentionSummary{}, err
|
||||
}
|
||||
return summarizeRetentionDefinitions(defs), nil
|
||||
}
|
||||
|
||||
func summarizeRetentionDefinitions(defs []metric.Definition) RetentionSummary {
|
||||
if len(defs) == 0 {
|
||||
return RetentionSummary{}
|
||||
}
|
||||
summary := RetentionSummary{AllPositive: true}
|
||||
for _, def := range defs {
|
||||
if def.RetentionDays <= 0 {
|
||||
summary.AllPositive = false
|
||||
}
|
||||
if def.RetentionDays > summary.MaxDays {
|
||||
summary.MaxDays = def.RetentionDays
|
||||
}
|
||||
}
|
||||
return summary
|
||||
}
|
||||
|
||||
func Compact(ctx context.Context, now time.Time) (int, error) {
|
||||
if !storeOperations.TryAcquire() {
|
||||
return 0, ErrCompactInProgress
|
||||
}
|
||||
defer storeOperations.Release()
|
||||
|
||||
storeMu.RLock()
|
||||
activeStore := store
|
||||
storeMu.RUnlock()
|
||||
if activeStore == nil {
|
||||
return 0, fmt.Errorf("metric store not initialized")
|
||||
}
|
||||
|
||||
defs, err := activeStore.ListMetrics(ctx)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
if len(defs) == 0 {
|
||||
compactAt = 0
|
||||
return 0, nil
|
||||
}
|
||||
if compactAt >= len(defs) {
|
||||
compactAt = 0
|
||||
}
|
||||
|
||||
total := 0
|
||||
start := compactAt
|
||||
for i := 0; i < len(defs); i++ {
|
||||
idx := (start + i) % len(defs)
|
||||
n, err := activeStore.CompactMetric(ctx, defs[idx].Name, now)
|
||||
if err != nil {
|
||||
compactAt = idx
|
||||
return total, fmt.Errorf("compact metric %q: %w", defs[idx].Name, err)
|
||||
}
|
||||
total += n
|
||||
compactAt = (idx + 1) % len(defs)
|
||||
}
|
||||
if _, err := activeStore.CleanupExpired(ctx, now); err != nil {
|
||||
return total, fmt.Errorf("clean up expired raw metrics: %w", err)
|
||||
}
|
||||
return total, nil
|
||||
}
|
||||
|
||||
// CloseStoreContext stops the asynchronous store migration before taking the
|
||||
// store write lock, so shutdown cannot wait forever on the migration's lease.
|
||||
func CloseStoreContext(ctx context.Context) error {
|
||||
if err := stopStoreMigrationForClose(ctx); err != nil {
|
||||
clearStoreClosing()
|
||||
return err
|
||||
}
|
||||
if err := storeOperations.Acquire(ctx); err != nil {
|
||||
clearStoreClosing()
|
||||
return fmt.Errorf("wait for metric store operations before close: %w", err)
|
||||
}
|
||||
defer storeOperations.Release()
|
||||
|
||||
storeMu.Lock()
|
||||
defer storeMu.Unlock()
|
||||
|
||||
if store != nil {
|
||||
err := store.Close()
|
||||
store = nil
|
||||
storeFingerprint = ""
|
||||
return err
|
||||
}
|
||||
storeFingerprint = ""
|
||||
return nil
|
||||
}
|
||||
|
||||
const defaultBuiltinMetricRetentionDays = 1
|
||||
|
||||
// createMetricDefinitions creates built-in definitions with explicit policies.
|
||||
func createMetricDefinitions(ctx context.Context, s *metric.Store) error {
|
||||
return createMetricDefinitionsWithDefaultRetention(ctx, s, defaultBuiltinMetricRetentionDays)
|
||||
}
|
||||
|
||||
func createMetricDefinitionsWithDefaultRetention(ctx context.Context, s *metric.Store, defaultRetentionDays int) error {
|
||||
if defaultRetentionDays < defaultBuiltinMetricRetentionDays {
|
||||
defaultRetentionDays = defaultBuiltinMetricRetentionDays
|
||||
}
|
||||
definitions := []metric.Definition{
|
||||
{Name: MetricCPU, Type: metric.TypeGauge, Unit: "%", Description: "CPU usage percentage", RetentionDays: defaultRetentionDays},
|
||||
{Name: MetricGPU, Type: metric.TypeGauge, Unit: "%", Description: "GPU usage percentage", RetentionDays: defaultRetentionDays},
|
||||
{Name: MetricGPUDeviceUsage, Type: metric.TypeGauge, Unit: "%", Description: "Per-device GPU utilization", RetentionDays: defaultRetentionDays},
|
||||
{Name: MetricGPUMem, Type: metric.TypeGauge, Unit: "bytes", Description: "GPU memory used", RetentionDays: defaultRetentionDays},
|
||||
{Name: MetricGPUMemTotal, Type: metric.TypeGauge, Unit: "bytes", Description: "GPU memory total", RetentionDays: defaultRetentionDays},
|
||||
{Name: MetricGPUTemp, Type: metric.TypeGauge, Unit: "°C", Description: "GPU temperature", RetentionDays: defaultRetentionDays},
|
||||
{Name: MetricRAM, Type: metric.TypeGauge, Unit: "bytes", Description: "RAM used", RetentionDays: defaultRetentionDays},
|
||||
{Name: MetricRAMTotal, Type: metric.TypeGauge, Unit: "bytes", Description: "RAM total", RetentionDays: defaultRetentionDays},
|
||||
{Name: MetricSwap, Type: metric.TypeGauge, Unit: "bytes", Description: "Swap used", RetentionDays: defaultRetentionDays},
|
||||
{Name: MetricSwapTotal, Type: metric.TypeGauge, Unit: "bytes", Description: "Swap total", RetentionDays: defaultRetentionDays},
|
||||
{Name: MetricLoad, Type: metric.TypeGauge, Unit: "", Description: "System load average", RetentionDays: defaultRetentionDays},
|
||||
{Name: MetricTemp, Type: metric.TypeGauge, Unit: "°C", Description: "System temperature", RetentionDays: defaultRetentionDays},
|
||||
{Name: MetricDisk, Type: metric.TypeGauge, Unit: "bytes", Description: "Disk used", RetentionDays: defaultRetentionDays},
|
||||
{Name: MetricDiskTotal, Type: metric.TypeGauge, Unit: "bytes", Description: "Disk total", RetentionDays: defaultRetentionDays},
|
||||
{Name: MetricNetIn, Type: metric.TypeGauge, Unit: "bytes/s", Description: "Network in rate", RetentionDays: defaultRetentionDays},
|
||||
{Name: MetricNetOut, Type: metric.TypeGauge, Unit: "bytes/s", Description: "Network out rate", RetentionDays: defaultRetentionDays},
|
||||
{Name: MetricNetTotalUp, Type: metric.TypeCounter, Unit: "bytes", Description: "Network total upload", RetentionDays: defaultRetentionDays},
|
||||
{Name: MetricNetTotalDown, Type: metric.TypeCounter, Unit: "bytes", Description: "Network total download", RetentionDays: defaultRetentionDays},
|
||||
{Name: MetricTrafficUp, Type: metric.TypeGauge, Unit: "bytes", Description: "Traffic upload delta", RetentionDays: defaultRetentionDays},
|
||||
{Name: MetricTrafficDown, Type: metric.TypeGauge, Unit: "bytes", Description: "Traffic download delta", RetentionDays: defaultRetentionDays},
|
||||
{Name: MetricProcess, Type: metric.TypeGauge, Unit: "count", Description: "Process count", RetentionDays: defaultRetentionDays},
|
||||
{Name: MetricConnections, Type: metric.TypeGauge, Unit: "count", Description: "TCP connections", RetentionDays: defaultRetentionDays},
|
||||
{Name: MetricConnectionsUDP, Type: metric.TypeGauge, Unit: "count", Description: "UDP connections", RetentionDays: defaultRetentionDays},
|
||||
{Name: MetricPingLatency, Type: metric.TypeGauge, Unit: "ms", Description: "Ping latency", RetentionDays: defaultRetentionDays},
|
||||
{Name: MetricPingLoss, Type: metric.TypeGauge, Unit: "ratio", Description: "Ping packet loss indicator", RetentionDays: defaultRetentionDays},
|
||||
}
|
||||
|
||||
for _, def := range definitions {
|
||||
existing, err := s.GetMetric(ctx, def.Name)
|
||||
if err != nil && !errors.Is(err, metric.ErrNotFound) {
|
||||
return fmt.Errorf("failed to get metric %s: %w", def.Name, err)
|
||||
}
|
||||
if err == nil {
|
||||
if existing.RetentionDays == 0 {
|
||||
if _, err := s.SetMetricRetention(ctx, def.Name, 0); err != nil {
|
||||
return fmt.Errorf("failed to preserve disabled metric %s: %w", def.Name, err)
|
||||
}
|
||||
continue
|
||||
}
|
||||
def.RetentionDays = existing.RetentionDays
|
||||
}
|
||||
if err := s.UpsertMetric(ctx, def); err != nil {
|
||||
return fmt.Errorf("failed to create metric %s: %w", def.Name, err)
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// WritePingRecord 将 ping 记录写入 metric store
|
||||
func WritePingRecord(ctx context.Context, rec models.PingRecord) error {
|
||||
s := GetStore()
|
||||
if s == nil {
|
||||
return fmt.Errorf("metric store not enabled")
|
||||
}
|
||||
|
||||
ts := rec.Time
|
||||
entityID := rec.Client
|
||||
tags := map[string]string{
|
||||
"task_id": fmt.Sprintf("%d", rec.TaskId),
|
||||
}
|
||||
|
||||
loss := 0.0
|
||||
if rec.Value < 0 {
|
||||
loss = 1
|
||||
}
|
||||
points := []metric.Point{
|
||||
{
|
||||
MetricName: MetricPingLatency,
|
||||
EntityID: entityID,
|
||||
Timestamp: ts,
|
||||
Value: float64(rec.Value),
|
||||
Tags: tags,
|
||||
},
|
||||
{
|
||||
MetricName: MetricPingLoss,
|
||||
EntityID: entityID,
|
||||
Timestamp: ts,
|
||||
Value: loss,
|
||||
Tags: tags,
|
||||
},
|
||||
}
|
||||
|
||||
return s.WriteBatch(ctx, points)
|
||||
}
|
||||
|
||||
// GetRecordsByClientAndTime 从 metric store 查询记录并重构为 models.Record
|
||||
func GetRecordsByClientAndTime(ctx context.Context, clientUUID string, start, end time.Time) ([]models.Record, error) {
|
||||
s := GetStore()
|
||||
if s == nil {
|
||||
return nil, fmt.Errorf("metric store not enabled")
|
||||
}
|
||||
|
||||
return getRecordsByClientAndTimeFromSeries(ctx, s, clientUUID, start, end)
|
||||
}
|
||||
|
||||
// GetRecordsByTime 从 metric store 查询所有客户端在时间范围内的记录
|
||||
func GetRecordsByTime(ctx context.Context, start, end time.Time) ([]models.Record, error) {
|
||||
s := GetStore()
|
||||
if s == nil {
|
||||
return nil, fmt.Errorf("metric store not enabled")
|
||||
}
|
||||
|
||||
interval := recordSeriesInterval(s, start, end, time.Now().UTC())
|
||||
entityIDs, err := listRecordEntityIDs(ctx, s, start, end, interval)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var records []models.Record
|
||||
for _, entityID := range entityIDs {
|
||||
items, err := getRecordsByClientAndTimeFromSeries(ctx, s, entityID, start, end)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
records = append(records, items...)
|
||||
}
|
||||
sortRecords(records)
|
||||
return records, nil
|
||||
}
|
||||
|
||||
type recordSeriesKey struct {
|
||||
client string
|
||||
ts int64
|
||||
}
|
||||
|
||||
func getRecordsByClientAndTimeFromSeries(ctx context.Context, s *metric.Store, clientUUID string, start, end time.Time) ([]models.Record, error) {
|
||||
now := time.Now().UTC()
|
||||
interval := recordSeriesInterval(s, start, end, now)
|
||||
recordMap := make(map[recordSeriesKey]*models.Record)
|
||||
|
||||
for _, metricName := range loadRecordMetricNames {
|
||||
points, err := s.Series(ctx, metric.AggregateQuery{
|
||||
Query: metric.Query{
|
||||
MetricName: metricName,
|
||||
EntityID: clientUUID,
|
||||
Start: start,
|
||||
End: end,
|
||||
Order: metric.OrderAsc,
|
||||
},
|
||||
Aggregation: recordMetricAggregation(metricName),
|
||||
Interval: interval,
|
||||
}, now)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to query metric %s: %w", metricName, err)
|
||||
}
|
||||
for _, point := range points {
|
||||
entityID := point.EntityID
|
||||
if entityID == "" {
|
||||
entityID = clientUUID
|
||||
}
|
||||
key := recordSeriesKey{client: entityID, ts: point.Bucket.Unix()}
|
||||
if recordMap[key] == nil {
|
||||
recordMap[key] = &models.Record{
|
||||
Client: entityID,
|
||||
Time: point.Bucket.UTC(),
|
||||
}
|
||||
}
|
||||
applyRecordMetricValue(recordMap[key], metricName, point.Value)
|
||||
}
|
||||
}
|
||||
|
||||
records := make([]models.Record, 0, len(recordMap))
|
||||
for _, rec := range recordMap {
|
||||
records = append(records, *rec)
|
||||
}
|
||||
sortRecords(records)
|
||||
return records, nil
|
||||
}
|
||||
|
||||
func recordMetricAggregation(metricName string) metric.Aggregation {
|
||||
switch metricName {
|
||||
case MetricTrafficUp, MetricTrafficDown:
|
||||
return metric.AggSum
|
||||
case MetricNetTotalUp, MetricNetTotalDown:
|
||||
return metric.AggLast
|
||||
default:
|
||||
return metric.AggAvg
|
||||
}
|
||||
}
|
||||
|
||||
func recordSeriesInterval(s *metric.Store, start, end, now time.Time) time.Duration {
|
||||
interval := recordDownsampleInterval(end.Sub(start), 500)
|
||||
return s.CompatibleSeriesInterval(start, now, interval)
|
||||
}
|
||||
|
||||
func recordDownsampleInterval(rangeDuration time.Duration, maxPoints int) time.Duration {
|
||||
if maxPoints <= 0 {
|
||||
maxPoints = 500
|
||||
}
|
||||
nanos := rangeDuration.Nanoseconds()
|
||||
if nanos <= 0 {
|
||||
return time.Second
|
||||
}
|
||||
interval := time.Duration((nanos + int64(maxPoints) - 1) / int64(maxPoints))
|
||||
if interval < time.Second {
|
||||
return time.Second
|
||||
}
|
||||
return metric.FloorStandardInterval(interval)
|
||||
}
|
||||
|
||||
func listRecordEntityIDs(ctx context.Context, s *metric.Store, start, end time.Time, interval time.Duration) ([]string, error) {
|
||||
seen := make(map[string]struct{})
|
||||
for _, metricName := range loadRecordMetricNames {
|
||||
ids, err := s.EntityIDs(ctx, metric.Query{
|
||||
MetricName: metricName,
|
||||
Start: start.Add(-interval),
|
||||
End: end,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for _, id := range ids {
|
||||
seen[id] = struct{}{}
|
||||
}
|
||||
}
|
||||
out := make([]string, 0, len(seen))
|
||||
for id := range seen {
|
||||
out = append(out, id)
|
||||
}
|
||||
sort.Strings(out)
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func applyRecordMetricValue(rec *models.Record, metricName string, value float64) {
|
||||
switch metricName {
|
||||
case MetricCPU:
|
||||
rec.Cpu = float32(value)
|
||||
case MetricGPU:
|
||||
rec.Gpu = float32(value)
|
||||
case MetricRAM:
|
||||
rec.Ram = int64(value)
|
||||
case MetricRAMTotal:
|
||||
rec.RamTotal = int64(value)
|
||||
case MetricSwap:
|
||||
rec.Swap = int64(value)
|
||||
case MetricSwapTotal:
|
||||
rec.SwapTotal = int64(value)
|
||||
case MetricLoad:
|
||||
rec.Load = float32(value)
|
||||
case MetricTemp:
|
||||
rec.Temp = float32(value)
|
||||
case MetricDisk:
|
||||
rec.Disk = int64(value)
|
||||
case MetricDiskTotal:
|
||||
rec.DiskTotal = int64(value)
|
||||
case MetricNetIn:
|
||||
rec.NetIn = int64(value)
|
||||
case MetricNetOut:
|
||||
rec.NetOut = int64(value)
|
||||
case MetricNetTotalUp:
|
||||
rec.NetTotalUp = int64(value)
|
||||
case MetricNetTotalDown:
|
||||
rec.NetTotalDown = int64(value)
|
||||
case MetricTrafficUp:
|
||||
rec.TrafficUp = int64(value)
|
||||
case MetricTrafficDown:
|
||||
rec.TrafficDown = int64(value)
|
||||
case MetricProcess:
|
||||
rec.Process = int(value)
|
||||
case MetricConnections:
|
||||
rec.Connections = int(value)
|
||||
case MetricConnectionsUDP:
|
||||
rec.ConnectionsUdp = int(value)
|
||||
}
|
||||
}
|
||||
|
||||
func sortRecords(records []models.Record) {
|
||||
sort.Slice(records, func(i, j int) bool {
|
||||
if records[i].Client != records[j].Client {
|
||||
return records[i].Client < records[j].Client
|
||||
}
|
||||
return records[i].Time.Before(records[j].Time)
|
||||
})
|
||||
}
|
||||
|
||||
// GetGPURecordsByClientAndTime 从 metric store 查询 GPU 记录
|
||||
func GetGPURecordsByClientAndTime(ctx context.Context, clientUUID string, start, end time.Time) ([]models.GPURecord, error) {
|
||||
s := GetStore()
|
||||
if s == nil {
|
||||
return nil, fmt.Errorf("metric store not enabled")
|
||||
}
|
||||
|
||||
// 查询 GPU 相关指标(每设备利用率使用独立指标 gpu.device.usage)
|
||||
gpuMetrics := []string{MetricGPUDeviceUsage, MetricGPUMem, MetricGPUMemTotal, MetricGPUTemp}
|
||||
|
||||
// 按设备索引和时间组织数据
|
||||
type gpuKey struct {
|
||||
deviceIndex int
|
||||
timestamp int64
|
||||
}
|
||||
recordMap := make(map[gpuKey]*models.GPURecord)
|
||||
|
||||
for _, metricName := range gpuMetrics {
|
||||
points, err := s.Query(ctx, metric.Query{
|
||||
MetricName: metricName,
|
||||
EntityID: clientUUID,
|
||||
Start: start,
|
||||
End: end,
|
||||
Order: metric.OrderAsc,
|
||||
})
|
||||
if err != nil {
|
||||
continue // GPU 数据可能不存在
|
||||
}
|
||||
|
||||
for _, p := range points {
|
||||
deviceIndex := 0
|
||||
deviceName := ""
|
||||
if idx, ok := p.Tags["device_index"]; ok {
|
||||
fmt.Sscanf(idx, "%d", &deviceIndex)
|
||||
}
|
||||
if name, ok := p.Tags["device_name"]; ok {
|
||||
deviceName = name
|
||||
}
|
||||
|
||||
key := gpuKey{deviceIndex: deviceIndex, timestamp: p.Timestamp.Unix()}
|
||||
if recordMap[key] == nil {
|
||||
recordMap[key] = &models.GPURecord{
|
||||
Client: clientUUID,
|
||||
Time: p.Timestamp.UTC(),
|
||||
DeviceIndex: deviceIndex,
|
||||
DeviceName: deviceName,
|
||||
}
|
||||
}
|
||||
rec := recordMap[key]
|
||||
|
||||
switch metricName {
|
||||
case MetricGPUDeviceUsage:
|
||||
rec.Utilization = float32(p.Value)
|
||||
case MetricGPUMem:
|
||||
rec.MemUsed = int64(p.Value)
|
||||
case MetricGPUMemTotal:
|
||||
rec.MemTotal = int64(p.Value)
|
||||
case MetricGPUTemp:
|
||||
rec.Temperature = int(p.Value)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 转换为切片
|
||||
records := make([]models.GPURecord, 0, len(recordMap))
|
||||
for _, rec := range recordMap {
|
||||
records = append(records, *rec)
|
||||
}
|
||||
|
||||
return records, nil
|
||||
}
|
||||
|
||||
// GetPingRecords 从 metric store 查询兼容旧接口的 ping 记录。
|
||||
//
|
||||
// 旧接口过去直接读取 ping_records 的原始点。启用 metric rollup 后,较旧
|
||||
// 的 raw 点会被压入 rollup 并删除,因此这里使用 Series 走与 queryMetrics
|
||||
// 相同的 raw/rollup 混合读取路径,并保留 task_id 标签。
|
||||
func GetPingRecords(ctx context.Context, clientUUID string, taskID int, start, end time.Time) ([]models.PingRecord, error) {
|
||||
s := GetStore()
|
||||
if s == nil {
|
||||
return nil, fmt.Errorf("metric store not enabled")
|
||||
}
|
||||
|
||||
query := metric.Query{
|
||||
MetricName: MetricPingLatency,
|
||||
Start: start,
|
||||
End: end,
|
||||
Order: metric.OrderAsc,
|
||||
}
|
||||
|
||||
if clientUUID != "" {
|
||||
query.EntityID = clientUUID
|
||||
}
|
||||
|
||||
if taskID >= 0 {
|
||||
query.Tags = map[string]string{"task_id": fmt.Sprintf("%d", taskID)}
|
||||
}
|
||||
|
||||
interval := pingQueryInterval(end.Sub(start), 4000)
|
||||
interval = s.CompatibleSeriesInterval(start, time.Now().UTC(), interval)
|
||||
points, err := s.Series(ctx, metric.AggregateQuery{
|
||||
Query: query,
|
||||
Aggregation: metric.AggLast,
|
||||
Interval: interval,
|
||||
PreserveSeries: true,
|
||||
}, time.Now().UTC())
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
records := make([]models.PingRecord, 0, len(points))
|
||||
for _, p := range points {
|
||||
taskIDVal := uint(0)
|
||||
if tid, ok := p.Tags["task_id"]; ok {
|
||||
var t uint64
|
||||
fmt.Sscanf(tid, "%d", &t)
|
||||
taskIDVal = uint(t)
|
||||
}
|
||||
|
||||
records = append(records, models.PingRecord{
|
||||
Client: p.EntityID,
|
||||
TaskId: taskIDVal,
|
||||
Time: p.Bucket.UTC(),
|
||||
Value: int(p.Value),
|
||||
})
|
||||
}
|
||||
sort.Slice(records, func(i, j int) bool {
|
||||
return records[i].Time.After(records[j].Time)
|
||||
})
|
||||
|
||||
return records, nil
|
||||
}
|
||||
|
||||
func pingQueryInterval(rangeDuration time.Duration, maxPoints int) time.Duration {
|
||||
if maxPoints <= 0 {
|
||||
maxPoints = 4000
|
||||
}
|
||||
if rangeDuration <= 0 {
|
||||
return time.Second
|
||||
}
|
||||
interval := time.Duration((rangeDuration.Nanoseconds() + int64(maxPoints) - 1) / int64(maxPoints))
|
||||
if interval < time.Second {
|
||||
return time.Second
|
||||
}
|
||||
return metric.FloorStandardInterval(interval)
|
||||
}
|
||||
|
||||
// farFuture 返回一个足够远的未来时间,用于以 DeleteBefore 语义清空某指标的全部数据。
|
||||
func farFuture() time.Time {
|
||||
return time.Now().UTC().Add(24 * 365 * time.Hour)
|
||||
}
|
||||
|
||||
// DeleteAllRecords 删除所有负载/系统类记录(保留指标定义,不含 ping)。
|
||||
func DeleteAllRecords(ctx context.Context) error {
|
||||
s := GetStore()
|
||||
if s == nil {
|
||||
return fmt.Errorf("metric store not enabled")
|
||||
}
|
||||
|
||||
for _, metricName := range recordMetricNames {
|
||||
if _, err := s.DeleteBefore(ctx, metricName, farFuture()); err != nil {
|
||||
log.Printf("Failed to delete metric %s: %v", metricName, err)
|
||||
}
|
||||
}
|
||||
clearReportTrafficStates()
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// DeleteAllPingRecords 删除全部 ping 记录(保留指标定义)。
|
||||
func DeleteAllPingRecords(ctx context.Context) error {
|
||||
s := GetStore()
|
||||
if s == nil {
|
||||
return fmt.Errorf("metric store not enabled")
|
||||
}
|
||||
for _, metricName := range pingMetricNames {
|
||||
if _, err := s.DeleteBefore(ctx, metricName, farFuture()); err != nil {
|
||||
return fmt.Errorf("failed to delete ping records: %w", err)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// DeletePingRecordsByTask 删除指定任务(task_id)的全部 ping 记录。
|
||||
func DeletePingRecordsByTask(ctx context.Context, taskIDs []uint) error {
|
||||
s := GetStore()
|
||||
if s == nil {
|
||||
return fmt.Errorf("metric store not enabled")
|
||||
}
|
||||
for _, id := range taskIDs {
|
||||
for _, metricName := range pingMetricNames {
|
||||
if _, err := s.DeleteSeries(ctx, metric.Query{
|
||||
MetricName: metricName,
|
||||
Tags: map[string]string{"task_id": fmt.Sprintf("%d", id)},
|
||||
}); err != nil {
|
||||
return fmt.Errorf("failed to delete ping records for task %d: %w", id, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// DeleteEntity 删除指定 agent 在所有指标下的历史数据。
|
||||
func DeleteEntity(ctx context.Context, entityID string) error {
|
||||
s := GetStore()
|
||||
if s == nil {
|
||||
return fmt.Errorf("metric store not enabled")
|
||||
}
|
||||
if _, err := s.DeleteEntity(ctx, entityID); err != nil {
|
||||
return fmt.Errorf("failed to delete metric records for entity %s: %w", entityID, err)
|
||||
}
|
||||
deleteReportTrafficState(entityID)
|
||||
return nil
|
||||
}
|
||||
@@ -1,72 +0,0 @@
|
||||
package metricstore
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"log"
|
||||
"strings"
|
||||
|
||||
"github.com/komari-monitor/komari/pkg/config"
|
||||
)
|
||||
|
||||
// migration.go
|
||||
//
|
||||
// 启动阶段的 metrics 数据迁移。这里只处理 metrics 存储后端切换时的数据搬运
|
||||
// (例如默认 SQLite ./data/metrics.db 切换到 MySQL/PostgreSQL)。旧 komari.db
|
||||
// 监控表导入属于一次性迁移,见 pkg/migrations.RunMetricStoreMigrations。
|
||||
|
||||
// targetFingerprint 返回当前 metrics 目标库的指纹(driver + 归一化 DSN),
|
||||
// 用于判断 metrics 存储后端是否发生变化(例如从 SQLite 切换到 MySQL/PostgreSQL)。
|
||||
func targetFingerprint(cfg *MetricStoreConfig) string {
|
||||
driver := ResolveDriverFromConfig(cfg.Driver, cfg.DSN)
|
||||
dsn := strings.TrimSpace(cfg.DSN)
|
||||
return fmt.Sprintf("%s|%s", driver, dsn)
|
||||
}
|
||||
|
||||
// RunStartupMigration 在服务启动时检查 metrics 存储后端是否发生变化,
|
||||
// 若变化则把上一个 metrics 目标库的全部数据搬运到当前目标。
|
||||
//
|
||||
// 判定依据 MigrationTargetKey(记录“数据当前完整所在的 metrics 目标指纹”):
|
||||
// - saved == current:数据已在当前目标,无需处理。
|
||||
// - saved == "":无历史目标记录。老快照的 metrics 数据固定落在默认 SQLite
|
||||
// (./data/metrics.db);若当前目标不是默认 SQLite(例如已切到 MySQL/PostgreSQL),
|
||||
// 把默认 SQLite 中可能存在的历史数据搬运过来,否则仅登记指纹。
|
||||
// - saved != current:后端已切换,把上一个目标(saved)的数据搬运到当前目标。
|
||||
//
|
||||
// 搬运以 upsert 写入,幂等,中断后重启可安全重跑。任一失败都返回错误,
|
||||
// 调用方应让启动失败并打印明确错误。
|
||||
func RunStartupMigration() error {
|
||||
s := GetStore()
|
||||
if s == nil {
|
||||
return fmt.Errorf("metric store not initialized")
|
||||
}
|
||||
|
||||
cfg, err := config.GetManyAs[MetricStoreConfig]()
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to load metric store config: %w", err)
|
||||
}
|
||||
|
||||
current := targetFingerprint(cfg)
|
||||
saved, _ := config.GetAs[string](MigrationTargetKey, "")
|
||||
|
||||
// 数据已完整位于当前目标:无需搬运。
|
||||
if saved == current {
|
||||
return nil
|
||||
}
|
||||
|
||||
// 上一个目标:优先使用已保存指纹;无记录时按老快照默认 SQLite 推断。
|
||||
prev := saved
|
||||
if prev == "" {
|
||||
prev = defaultSQLiteFingerprint()
|
||||
}
|
||||
|
||||
if prev != current {
|
||||
if err := migrateFromPreviousStore(prev, cfg, s); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
if err := config.Set(MigrationTargetKey, current); err != nil {
|
||||
log.Printf("Failed to persist migration target fingerprint: %v", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -1,211 +0,0 @@
|
||||
package metricstore
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"log"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/komari-monitor/komari/pkg/metric"
|
||||
)
|
||||
|
||||
// storeMigrationWindow 是 store-to-store 迁移时按时间分窗查询的窗口大小。
|
||||
// 采样点按时间窗口分批拉取,避免一次性把整段序列读入内存。
|
||||
const storeMigrationWindow = 6 * time.Hour
|
||||
|
||||
// configFromFingerprint 从目标指纹(driver|dsn)重建一个 MetricStoreConfig,
|
||||
// 用于以只读方式打开“上一次的 metrics 目标库”。表前缀、保留天数、连接数等
|
||||
// 沿用当前配置(切换后端时这些通常不变)。
|
||||
func configFromFingerprint(fingerprint string, base *MetricStoreConfig) (*MetricStoreConfig, error) {
|
||||
idx := strings.Index(fingerprint, "|")
|
||||
if idx < 0 {
|
||||
return nil, fmt.Errorf("invalid target fingerprint: %q", fingerprint)
|
||||
}
|
||||
driver := fingerprint[:idx]
|
||||
dsn := fingerprint[idx+1:]
|
||||
if strings.TrimSpace(driver) == "" {
|
||||
return nil, fmt.Errorf("empty driver in target fingerprint: %q", fingerprint)
|
||||
}
|
||||
return &MetricStoreConfig{
|
||||
Driver: driver,
|
||||
DSN: dsn,
|
||||
DownsamplingEnabled: base.DownsamplingEnabled,
|
||||
TablePrefix: base.TablePrefix,
|
||||
MaxOpenConns: base.MaxOpenConns,
|
||||
MaxIdleConns: base.MaxIdleConns,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// openSourceStore 打开一个已存在的 metrics 目标库作为数据搬运的源库读取。
|
||||
//
|
||||
// 使用 autoMigrate=true:GORM 的 AutoMigrate 只新增表/列/索引、从不删除数据,
|
||||
// 因此对真实旧库幂等无害;而当源库文件/表不存在(例如老快照记录了 completed
|
||||
// 但 metrics.db 缺失)时,可创建空表让后续 ListMetrics 返回空集而非报
|
||||
// "no such table",从而把“无历史可迁移”识别为正常情况而非错误。
|
||||
func openSourceStore(ctx context.Context, cfg *MetricStoreConfig) (*metric.Store, error) {
|
||||
metricCfg, err := buildMetricConfig(cfg, true)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return metric.Open(ctx, metricCfg)
|
||||
}
|
||||
|
||||
// defaultSQLiteFingerprint 返回默认 SQLite metrics 库(./data/metrics.db)的目标指纹。
|
||||
// 老快照的 metrics 数据固定落在该 SQLite 文件,用于在缺失指纹时推断上一个源库。
|
||||
func defaultSQLiteFingerprint() string {
|
||||
return targetFingerprint(&MetricStoreConfig{Driver: "sqlite", DSN: "./data/metrics.db"})
|
||||
}
|
||||
|
||||
// migrateFromPreviousStore 打开由 prevFingerprint 指定的上一个 metrics 目标库作为源,
|
||||
// 把其中的全部指标搬运到当前目标 dst(例如 SQLite metrics.db → MySQL/PostgreSQL)。
|
||||
//
|
||||
// 该过程幂等:dst 以 (metric, entity, tags, ts) upsert 写入,中断后重启可安全重跑。
|
||||
func migrateFromPreviousStore(prevFingerprint string, cfg *MetricStoreConfig, dst *metric.Store) error {
|
||||
prevCfg, err := configFromFingerprint(prevFingerprint, cfg)
|
||||
if err != nil {
|
||||
return fmt.Errorf("parse previous metrics target %q: %w", prevFingerprint, err)
|
||||
}
|
||||
|
||||
ctx := context.Background()
|
||||
src, err := openSourceStore(ctx, prevCfg)
|
||||
if err != nil {
|
||||
return fmt.Errorf("open previous metrics store (%s): %w", prevFingerprint, err)
|
||||
}
|
||||
defer src.Close()
|
||||
|
||||
log.Printf("Migrating metrics from previous store %s to current target...", prevFingerprint)
|
||||
total, err := MigrateBetweenStores(ctx, src, dst)
|
||||
if err != nil {
|
||||
return fmt.Errorf("store-to-store metrics migration failed: %w", err)
|
||||
}
|
||||
log.Printf("Store-to-store metrics migration completed (%d points)", total)
|
||||
return nil
|
||||
}
|
||||
|
||||
// storeMigrationObserver 在 store-to-store 迁移过程中接收进度回调。
|
||||
// - currentMetric:当前正在搬运的指标名。
|
||||
// - metricIndex:该指标在全部指标中的序号(0 起),即已完成的指标数。
|
||||
// - totalMetrics:指标定义总数。
|
||||
// - addedPoints:本次新写入目标库的采样点数(用于外部累计)。
|
||||
type storeMigrationObserver func(currentMetric string, metricIndex, totalMetrics int, addedPoints int64)
|
||||
|
||||
// MigrateBetweenStores 把 src metric store 的全部指标定义与采样点搬运到 dst。
|
||||
//
|
||||
// 用于 metrics 后端切换(例如默认 SQLite metrics.db → MySQL/PostgreSQL):
|
||||
// - 先在 dst 建立/更新全部指标定义;
|
||||
// - 再按指标、按时间窗口分批读取源库采样点并写入 dst。
|
||||
//
|
||||
// 采样点在 dst 以 (metric_name, entity_id, tags, ts) upsert 写入,窗口边界重叠也
|
||||
// 幂等,因此整个过程可安全重试。返回搬运的采样点总数。
|
||||
func MigrateBetweenStores(ctx context.Context, src, dst *metric.Store) (int64, error) {
|
||||
return migrateBetweenStores(ctx, src, dst, nil)
|
||||
}
|
||||
|
||||
// migrateBetweenStores 是 MigrateBetweenStores 的内部实现,额外接受一个可选的进度
|
||||
// 观察者 observe(为 nil 时行为与旧版本完全一致)。启动迁移走 nil,WebUI/API 触发
|
||||
// 的迁移传入回调以实时更新进度。
|
||||
func migrateBetweenStores(ctx context.Context, src, dst *metric.Store, observe storeMigrationObserver) (int64, error) {
|
||||
if src == nil || dst == nil {
|
||||
return 0, fmt.Errorf("source or destination metric store is nil")
|
||||
}
|
||||
|
||||
defs, err := src.ListMetrics(ctx)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("list source metrics: %w", err)
|
||||
}
|
||||
|
||||
var total int64
|
||||
for i, def := range defs {
|
||||
if observe != nil {
|
||||
// 进入下一个指标:已完成 i 个指标。
|
||||
observe(def.Name, i, len(defs), 0)
|
||||
}
|
||||
// 目标库先建立指标定义,保证后续写入的指标存在。
|
||||
if err := dst.UpsertMetric(ctx, def); err != nil {
|
||||
return total, fmt.Errorf("upsert metric %q on target: %w", def.Name, err)
|
||||
}
|
||||
if def.RetentionDays == 0 {
|
||||
continue
|
||||
}
|
||||
|
||||
earliest, latest, ok, err := metricTimeBounds(ctx, src, def.Name)
|
||||
if err != nil {
|
||||
return total, fmt.Errorf("resolve time bounds for metric %q: %w", def.Name, err)
|
||||
}
|
||||
if !ok {
|
||||
// 该指标只有定义没有数据,跳过。
|
||||
continue
|
||||
}
|
||||
|
||||
var migrated int64
|
||||
for windowStart := earliest; !windowStart.After(latest); windowStart = windowStart.Add(storeMigrationWindow) {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return total, ctx.Err()
|
||||
default:
|
||||
}
|
||||
|
||||
windowEnd := windowStart.Add(storeMigrationWindow)
|
||||
points, err := src.Query(ctx, metric.Query{
|
||||
MetricName: def.Name,
|
||||
Start: windowStart,
|
||||
End: windowEnd,
|
||||
Order: metric.OrderAsc,
|
||||
})
|
||||
if err != nil {
|
||||
return total, fmt.Errorf("query metric %q window [%s,%s): %w", def.Name, windowStart, windowEnd, err)
|
||||
}
|
||||
if len(points) == 0 {
|
||||
continue
|
||||
}
|
||||
if err := dst.WriteBatch(ctx, points); err != nil {
|
||||
return total, fmt.Errorf("write metric %q batch to target: %w", def.Name, err)
|
||||
}
|
||||
migrated += int64(len(points))
|
||||
total += int64(len(points))
|
||||
if observe != nil {
|
||||
observe(def.Name, i, len(defs), int64(len(points)))
|
||||
}
|
||||
}
|
||||
if migrated > 0 {
|
||||
log.Printf("[store-migration] metric %q: migrated %d points", def.Name, migrated)
|
||||
}
|
||||
}
|
||||
|
||||
return total, nil
|
||||
}
|
||||
|
||||
// metricTimeBounds 返回某指标在 src 中最早/最晚采样时间。ok=false 表示无数据。
|
||||
// 通过升/降序各取一条采样点定位边界,避免把整段序列读入内存。
|
||||
func metricTimeBounds(ctx context.Context, src *metric.Store, name string) (time.Time, time.Time, bool, error) {
|
||||
wide := metric.Query{
|
||||
MetricName: name,
|
||||
Start: time.Unix(0, 0),
|
||||
End: time.Now().UTC().Add(24 * time.Hour),
|
||||
}
|
||||
|
||||
asc := wide
|
||||
asc.Order = metric.OrderAsc
|
||||
asc.Limit = 1
|
||||
first, err := src.Query(ctx, asc)
|
||||
if err != nil {
|
||||
return time.Time{}, time.Time{}, false, err
|
||||
}
|
||||
if len(first) == 0 {
|
||||
return time.Time{}, time.Time{}, false, nil
|
||||
}
|
||||
|
||||
desc := wide
|
||||
desc.Order = metric.OrderDesc
|
||||
desc.Limit = 1
|
||||
last, err := src.Query(ctx, desc)
|
||||
if err != nil {
|
||||
return time.Time{}, time.Time{}, false, err
|
||||
}
|
||||
if len(last) == 0 {
|
||||
return time.Time{}, time.Time{}, false, nil
|
||||
}
|
||||
|
||||
return first[0].Timestamp, last[0].Timestamp, true, nil
|
||||
}
|
||||
@@ -1,38 +0,0 @@
|
||||
package metricstore
|
||||
|
||||
import "context"
|
||||
|
||||
// storeOperationGate serializes operations that need a stable active Store.
|
||||
// A channel-backed gate supports both non-blocking scheduled work and
|
||||
// context-aware admin/shutdown waits.
|
||||
type storeOperationGate struct {
|
||||
token chan struct{}
|
||||
}
|
||||
|
||||
func newStoreOperationGate() *storeOperationGate {
|
||||
gate := &storeOperationGate{token: make(chan struct{}, 1)}
|
||||
gate.token <- struct{}{}
|
||||
return gate
|
||||
}
|
||||
|
||||
func (g *storeOperationGate) Acquire(ctx context.Context) error {
|
||||
select {
|
||||
case <-g.token:
|
||||
return nil
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
}
|
||||
}
|
||||
|
||||
func (g *storeOperationGate) TryAcquire() bool {
|
||||
select {
|
||||
case <-g.token:
|
||||
return true
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
func (g *storeOperationGate) Release() {
|
||||
g.token <- struct{}{}
|
||||
}
|
||||
@@ -1,467 +0,0 @@
|
||||
package metricstore
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"log"
|
||||
"strconv"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/komari-monitor/komari/database/models"
|
||||
"github.com/komari-monitor/komari/pkg/metric"
|
||||
v1 "github.com/komari-monitor/komari/protocol/v1"
|
||||
)
|
||||
|
||||
type reportTrafficState struct {
|
||||
mu sync.Mutex
|
||||
|
||||
reportTrafficValues
|
||||
}
|
||||
|
||||
type reportTrafficValues struct {
|
||||
initialized bool
|
||||
timestamp time.Time
|
||||
hasUp bool
|
||||
totalUp int64
|
||||
hasDown bool
|
||||
totalDown int64
|
||||
}
|
||||
|
||||
var reportTrafficStates sync.Map
|
||||
|
||||
const (
|
||||
reportBatchInterval = 3 * time.Second
|
||||
reportBatchQueueSize = 4096
|
||||
reportBatchMaxReports = 256 * 21
|
||||
reportBatchWriteTimeout = 10 * time.Second
|
||||
)
|
||||
|
||||
var (
|
||||
reportBatcherMu sync.Mutex
|
||||
reportBatcher *reportBatchWorker
|
||||
)
|
||||
|
||||
var (
|
||||
ErrReportBatchQueueFull = errors.New("metric report batch queue is full")
|
||||
ErrReportBatchStopped = errors.New("metric report batcher is stopped")
|
||||
)
|
||||
|
||||
type reportBatchRequest struct {
|
||||
ctx context.Context
|
||||
done chan error
|
||||
stop bool
|
||||
}
|
||||
|
||||
type reportBatchWorker struct {
|
||||
mu sync.Mutex
|
||||
queue chan v1.Report
|
||||
requests chan reportBatchRequest
|
||||
done chan struct{}
|
||||
stopping bool
|
||||
}
|
||||
|
||||
// StartReportBatcher starts the shared report writer. Reports are collected for
|
||||
// a short window so many agents can be persisted in one database transaction.
|
||||
func StartReportBatcher() {
|
||||
reportBatcherMu.Lock()
|
||||
defer reportBatcherMu.Unlock()
|
||||
if reportBatcher != nil {
|
||||
return
|
||||
}
|
||||
worker := &reportBatchWorker{
|
||||
queue: make(chan v1.Report, reportBatchQueueSize),
|
||||
requests: make(chan reportBatchRequest, 1),
|
||||
done: make(chan struct{}),
|
||||
}
|
||||
reportBatcher = worker
|
||||
go worker.run()
|
||||
}
|
||||
|
||||
// StopReportBatcher stops the report writer after flushing all queued reports.
|
||||
func StopReportBatcher(ctx context.Context) error {
|
||||
reportBatcherMu.Lock()
|
||||
worker := reportBatcher
|
||||
if worker == nil {
|
||||
reportBatcherMu.Unlock()
|
||||
return nil
|
||||
}
|
||||
worker.mu.Lock()
|
||||
worker.stopping = true
|
||||
worker.mu.Unlock()
|
||||
reportBatcherMu.Unlock()
|
||||
|
||||
request := reportBatchRequest{ctx: ctx, done: make(chan error, 1), stop: true}
|
||||
select {
|
||||
case worker.requests <- request:
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
}
|
||||
select {
|
||||
case err := <-request.done:
|
||||
<-worker.done
|
||||
reportBatcherMu.Lock()
|
||||
if reportBatcher == worker {
|
||||
reportBatcher = nil
|
||||
}
|
||||
reportBatcherMu.Unlock()
|
||||
return err
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
}
|
||||
}
|
||||
|
||||
// FlushReportBatch synchronously flushes the current queue. It is useful for
|
||||
// controlled handoff points and deterministic tests; normal operation relies
|
||||
// on the three-second ticker.
|
||||
func FlushReportBatch(ctx context.Context) error {
|
||||
reportBatcherMu.Lock()
|
||||
worker := reportBatcher
|
||||
reportBatcherMu.Unlock()
|
||||
if worker == nil {
|
||||
return nil
|
||||
}
|
||||
request := reportBatchRequest{ctx: ctx, done: make(chan error, 1)}
|
||||
worker.mu.Lock()
|
||||
stopping := worker.stopping
|
||||
worker.mu.Unlock()
|
||||
if stopping {
|
||||
return ErrReportBatchStopped
|
||||
}
|
||||
select {
|
||||
case worker.requests <- request:
|
||||
case <-worker.done:
|
||||
return ErrReportBatchStopped
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
}
|
||||
select {
|
||||
case err := <-request.done:
|
||||
return err
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
}
|
||||
}
|
||||
|
||||
// WriteReport persists one agent report as raw metric points sharing the same
|
||||
// server receive time. Traffic deltas are derived from the previous counters
|
||||
// and stored alongside the raw counters so they remain summable after rollup.
|
||||
func WriteReport(ctx context.Context, report v1.Report) (v1.Report, error) {
|
||||
if report.UUID == "" {
|
||||
return v1.Report{}, fmt.Errorf("report UUID is required")
|
||||
}
|
||||
if report.UpdatedAt.IsZero() {
|
||||
return v1.Report{}, fmt.Errorf("report receive time is required")
|
||||
}
|
||||
report.UpdatedAt = report.UpdatedAt.UTC()
|
||||
if GetStore() == nil {
|
||||
return v1.Report{}, fmt.Errorf("metric store not enabled")
|
||||
}
|
||||
|
||||
reportBatcherMu.Lock()
|
||||
worker := reportBatcher
|
||||
reportBatcherMu.Unlock()
|
||||
if worker != nil {
|
||||
if err := worker.enqueue(ctx, report); err != nil {
|
||||
return v1.Report{}, err
|
||||
}
|
||||
return report, nil
|
||||
}
|
||||
|
||||
saved, err := writeReportBatch(ctx, []v1.Report{report})
|
||||
if err != nil {
|
||||
return v1.Report{}, err
|
||||
}
|
||||
return saved[0], nil
|
||||
}
|
||||
|
||||
func (w *reportBatchWorker) enqueue(ctx context.Context, report v1.Report) error {
|
||||
w.mu.Lock()
|
||||
defer w.mu.Unlock()
|
||||
if w.stopping {
|
||||
return ErrReportBatchStopped
|
||||
}
|
||||
if err := ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
select {
|
||||
case w.queue <- report:
|
||||
return nil
|
||||
default:
|
||||
return ErrReportBatchQueueFull
|
||||
}
|
||||
}
|
||||
|
||||
func (w *reportBatchWorker) run() {
|
||||
ticker := time.NewTicker(reportBatchInterval)
|
||||
defer ticker.Stop()
|
||||
|
||||
var pending []v1.Report
|
||||
for {
|
||||
select {
|
||||
case report := <-w.queue:
|
||||
pending = append(pending, report)
|
||||
case request := <-w.requests:
|
||||
pending = append(pending, drainReportQueue(w.queue, reportBatchMaxReports-len(pending))...)
|
||||
err := writePendingReports(request.ctx, &pending)
|
||||
if request.stop {
|
||||
if err != nil {
|
||||
log.Printf("failed to flush metric report batch during shutdown: %v", err)
|
||||
}
|
||||
close(w.done)
|
||||
request.done <- err
|
||||
return
|
||||
}
|
||||
request.done <- err
|
||||
case <-ticker.C:
|
||||
pending = append(pending, drainReportQueue(w.queue, reportBatchMaxReports-len(pending))...)
|
||||
if err := writePendingReports(context.Background(), &pending); err != nil {
|
||||
log.Printf("failed to flush metric report batch: %v", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func drainReportQueue(queue <-chan v1.Report, limit int) []v1.Report {
|
||||
if limit <= 0 {
|
||||
return nil
|
||||
}
|
||||
reports := make([]v1.Report, 0, limit)
|
||||
for len(reports) < limit {
|
||||
select {
|
||||
case report := <-queue:
|
||||
reports = append(reports, report)
|
||||
default:
|
||||
return reports
|
||||
}
|
||||
}
|
||||
return reports
|
||||
}
|
||||
|
||||
func writePendingReports(ctx context.Context, pending *[]v1.Report) error {
|
||||
if len(*pending) == 0 {
|
||||
return nil
|
||||
}
|
||||
writeCtx, cancel := context.WithTimeout(ctx, reportBatchWriteTimeout)
|
||||
defer cancel()
|
||||
if err := func() error {
|
||||
reports := *pending
|
||||
_, err := writeReportBatch(writeCtx, reports)
|
||||
return err
|
||||
}(); err != nil {
|
||||
return err
|
||||
}
|
||||
*pending = (*pending)[:0]
|
||||
return nil
|
||||
}
|
||||
|
||||
func writeReportBatch(ctx context.Context, reports []v1.Report) ([]v1.Report, error) {
|
||||
if len(reports) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
if err := storeOperations.Acquire(ctx); err != nil {
|
||||
return nil, fmt.Errorf("wait for metric store operation before writing reports: %w", err)
|
||||
}
|
||||
defer storeOperations.Release()
|
||||
|
||||
s := GetStore()
|
||||
if s == nil {
|
||||
return nil, fmt.Errorf("metric store not enabled")
|
||||
}
|
||||
|
||||
prepared := make([]v1.Report, len(reports))
|
||||
copy(prepared, reports)
|
||||
points := make([]metric.Point, 0, len(reports)*20)
|
||||
pendingStates := make(map[*reportTrafficState]reportTrafficValues)
|
||||
for i, report := range prepared {
|
||||
stateValue, _ := reportTrafficStates.LoadOrStore(report.UUID, &reportTrafficState{})
|
||||
state := stateValue.(*reportTrafficState)
|
||||
values, ok := pendingStates[state]
|
||||
if !ok {
|
||||
state.mu.Lock()
|
||||
values = state.reportTrafficValues
|
||||
state.mu.Unlock()
|
||||
}
|
||||
if !values.initialized {
|
||||
var err error
|
||||
values.totalUp, values.hasUp, err = latestReportCounter(ctx, s, MetricNetTotalUp, report.UUID, report.UpdatedAt)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("load previous upload counter: %w", err)
|
||||
}
|
||||
values.totalDown, values.hasDown, err = latestReportCounter(ctx, s, MetricNetTotalDown, report.UUID, report.UpdatedAt)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("load previous download counter: %w", err)
|
||||
}
|
||||
values.initialized = true
|
||||
}
|
||||
|
||||
if !values.timestamp.IsZero() && !report.UpdatedAt.After(values.timestamp) {
|
||||
report.UpdatedAt = values.timestamp.Add(time.Nanosecond)
|
||||
}
|
||||
trafficUp := int64(0)
|
||||
if values.hasUp {
|
||||
trafficUp = TrafficCounterDelta(report.Network.TotalUp, values.totalUp)
|
||||
}
|
||||
trafficDown := int64(0)
|
||||
if values.hasDown {
|
||||
trafficDown = TrafficCounterDelta(report.Network.TotalDown, values.totalDown)
|
||||
}
|
||||
points = append(points, reportMetricPoints(report, trafficUp, trafficDown)...)
|
||||
values.timestamp = report.UpdatedAt
|
||||
values.hasUp = true
|
||||
values.totalUp = report.Network.TotalUp
|
||||
values.hasDown = true
|
||||
values.totalDown = report.Network.TotalDown
|
||||
pendingStates[state] = values
|
||||
prepared[i] = report
|
||||
}
|
||||
|
||||
if err := s.WriteBatch(ctx, points); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for state, values := range pendingStates {
|
||||
state.mu.Lock()
|
||||
state.reportTrafficValues = values
|
||||
state.mu.Unlock()
|
||||
}
|
||||
return prepared, nil
|
||||
}
|
||||
|
||||
func reportMetricPoints(report v1.Report, trafficUp, trafficDown int64) []metric.Point {
|
||||
entityID := report.UUID
|
||||
ts := report.UpdatedAt
|
||||
gpuUsage := 0.0
|
||||
if report.GPU != nil {
|
||||
gpuUsage = report.GPU.AverageUsage
|
||||
}
|
||||
points := []metric.Point{
|
||||
{MetricName: MetricCPU, EntityID: entityID, Timestamp: ts, Value: report.CPU.Usage},
|
||||
{MetricName: MetricGPU, EntityID: entityID, Timestamp: ts, Value: gpuUsage},
|
||||
{MetricName: MetricRAM, EntityID: entityID, Timestamp: ts, Value: float64(report.Ram.Used)},
|
||||
{MetricName: MetricRAMTotal, EntityID: entityID, Timestamp: ts, Value: float64(report.Ram.Total)},
|
||||
{MetricName: MetricSwap, EntityID: entityID, Timestamp: ts, Value: float64(report.Swap.Used)},
|
||||
{MetricName: MetricSwapTotal, EntityID: entityID, Timestamp: ts, Value: float64(report.Swap.Total)},
|
||||
{MetricName: MetricLoad, EntityID: entityID, Timestamp: ts, Value: report.Load.Load1},
|
||||
{MetricName: MetricTemp, EntityID: entityID, Timestamp: ts, Value: 0},
|
||||
{MetricName: MetricDisk, EntityID: entityID, Timestamp: ts, Value: float64(report.Disk.Used)},
|
||||
{MetricName: MetricDiskTotal, EntityID: entityID, Timestamp: ts, Value: float64(report.Disk.Total)},
|
||||
{MetricName: MetricNetIn, EntityID: entityID, Timestamp: ts, Value: float64(report.Network.Down)},
|
||||
{MetricName: MetricNetOut, EntityID: entityID, Timestamp: ts, Value: float64(report.Network.Up)},
|
||||
{MetricName: MetricNetTotalUp, EntityID: entityID, Timestamp: ts, Value: float64(report.Network.TotalUp)},
|
||||
{MetricName: MetricNetTotalDown, EntityID: entityID, Timestamp: ts, Value: float64(report.Network.TotalDown)},
|
||||
{MetricName: MetricTrafficUp, EntityID: entityID, Timestamp: ts, Value: float64(trafficUp)},
|
||||
{MetricName: MetricTrafficDown, EntityID: entityID, Timestamp: ts, Value: float64(trafficDown)},
|
||||
{MetricName: MetricProcess, EntityID: entityID, Timestamp: ts, Value: float64(report.Process)},
|
||||
{MetricName: MetricConnections, EntityID: entityID, Timestamp: ts, Value: float64(report.Connections.TCP)},
|
||||
{MetricName: MetricConnectionsUDP, EntityID: entityID, Timestamp: ts, Value: float64(report.Connections.UDP)},
|
||||
}
|
||||
if report.GPU == nil {
|
||||
return points
|
||||
}
|
||||
for deviceIndex, gpu := range report.GPU.DetailedInfo {
|
||||
tags := map[string]string{
|
||||
"device_index": strconv.Itoa(deviceIndex),
|
||||
"device_name": gpu.Name,
|
||||
}
|
||||
points = append(points,
|
||||
metric.Point{MetricName: MetricGPUMem, EntityID: entityID, Timestamp: ts, Value: float64(gpu.MemoryUsed), Tags: tags},
|
||||
metric.Point{MetricName: MetricGPUMemTotal, EntityID: entityID, Timestamp: ts, Value: float64(gpu.MemoryTotal), Tags: tags},
|
||||
metric.Point{MetricName: MetricGPUDeviceUsage, EntityID: entityID, Timestamp: ts, Value: gpu.Utilization, Tags: tags},
|
||||
metric.Point{MetricName: MetricGPUTemp, EntityID: entityID, Timestamp: ts, Value: float64(gpu.Temperature), Tags: tags},
|
||||
)
|
||||
}
|
||||
return points
|
||||
}
|
||||
|
||||
func latestReportCounter(ctx context.Context, s *metric.Store, metricName, entityID string, before time.Time) (int64, bool, error) {
|
||||
def, err := s.GetMetric(ctx, metricName)
|
||||
if errors.Is(err, metric.ErrNotFound) {
|
||||
return 0, false, nil
|
||||
}
|
||||
if err != nil {
|
||||
return 0, false, err
|
||||
}
|
||||
if def.RetentionDays <= 0 {
|
||||
return 0, false, nil
|
||||
}
|
||||
|
||||
end := before.UTC().Add(-time.Nanosecond)
|
||||
start := end.Add(-time.Duration(def.RetentionDays) * 24 * time.Hour)
|
||||
interval := s.CompatibleSeriesInterval(start, before, 24*time.Hour)
|
||||
points, err := s.Series(ctx, metric.AggregateQuery{
|
||||
Query: metric.Query{
|
||||
MetricName: metricName,
|
||||
EntityID: entityID,
|
||||
Start: start,
|
||||
End: end,
|
||||
Order: metric.OrderAsc,
|
||||
},
|
||||
Aggregation: metric.AggLast,
|
||||
Interval: interval,
|
||||
PreserveSeries: true,
|
||||
}, before)
|
||||
if err != nil {
|
||||
return 0, false, err
|
||||
}
|
||||
if len(points) == 0 {
|
||||
return 0, false, nil
|
||||
}
|
||||
return int64(points[len(points)-1].Value), true, nil
|
||||
}
|
||||
|
||||
// GetLatestTrafficBefore returns the latest retained upload/download counters
|
||||
// before a boundary, transparently reading raw points or rollups.
|
||||
func GetLatestTrafficBefore(ctx context.Context, entityIDs []string, before time.Time) (map[string]models.Record, error) {
|
||||
s := GetStore()
|
||||
if s == nil {
|
||||
return nil, fmt.Errorf("metric store not enabled")
|
||||
}
|
||||
result := make(map[string]models.Record, len(entityIDs))
|
||||
for _, entityID := range entityIDs {
|
||||
if entityID == "" {
|
||||
continue
|
||||
}
|
||||
up, hasUp, err := latestReportCounter(ctx, s, MetricNetTotalUp, entityID, before)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
down, hasDown, err := latestReportCounter(ctx, s, MetricNetTotalDown, entityID, before)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if !hasUp && !hasDown {
|
||||
continue
|
||||
}
|
||||
result[entityID] = models.Record{
|
||||
Client: entityID,
|
||||
Time: before.UTC().Add(-time.Nanosecond),
|
||||
NetTotalUp: up,
|
||||
NetTotalDown: down,
|
||||
}
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
// TrafficCounterDelta returns a reset-aware increase between two cumulative
|
||||
// traffic counters. After a reset, the current counter is the new increase.
|
||||
func TrafficCounterDelta(current, previous int64) int64 {
|
||||
if current < 0 || previous < 0 {
|
||||
return 0
|
||||
}
|
||||
if current >= previous {
|
||||
return current - previous
|
||||
}
|
||||
return current
|
||||
}
|
||||
|
||||
func deleteReportTrafficState(entityID string) {
|
||||
reportTrafficStates.Delete(entityID)
|
||||
}
|
||||
|
||||
func clearReportTrafficStates() {
|
||||
reportTrafficStates.Range(func(key, _ any) bool {
|
||||
reportTrafficStates.Delete(key)
|
||||
return true
|
||||
})
|
||||
}
|
||||
@@ -8,9 +8,9 @@ type MessageSenderProvider struct {
|
||||
}
|
||||
|
||||
type EventMessage struct {
|
||||
Event string `json:"event"`
|
||||
Event any `json:"event"`
|
||||
Clients []Client `json:"clients"`
|
||||
Time time.Time `json:"time"`
|
||||
Message string `json:"message"`
|
||||
Emoji string `json:"emoji"`
|
||||
Message any `json:"message"`
|
||||
Emoji any `json:"emoji"`
|
||||
}
|
||||
@@ -0,0 +1,86 @@
|
||||
package models
|
||||
|
||||
// Plugin is the manifest of an installed plugin (komari-plugin.json).
|
||||
//
|
||||
// Name, Description and Author accept either a plain string or an i18n map
|
||||
// like {"zh_CN": "...", "en": "..."}, mirroring the theme manifest. The
|
||||
// backend passes the value through and the frontend resolves it against the
|
||||
// current locale.
|
||||
type Plugin struct {
|
||||
Name any `json:"name"`
|
||||
Short string `json:"short"` // directory name under data/plugin
|
||||
Description any `json:"description"`
|
||||
Author any `json:"author"`
|
||||
Version string `json:"version"`
|
||||
URL string `json:"url"`
|
||||
Icon string `json:"icon"`
|
||||
Komari string `json:"komari"` // supported server version constraint, e.g. ">=0.0.1"
|
||||
Entry string `json:"entry"` // entry script, defaults to "script.js"
|
||||
Permissions PluginPermissions `json:"permissions"`
|
||||
Configuration Configuration `json:"configuration"` // declared config items, same shape as themes
|
||||
Pages []PluginPage `json:"pages,omitempty"` // injected admin pages
|
||||
}
|
||||
|
||||
// PluginPermissions declares the plugin capabilities that require admin
|
||||
// approval. Every field defaults to its zero value: nothing is granted
|
||||
// unless declared.
|
||||
//
|
||||
// The following capabilities are always granted and never require approval:
|
||||
// reading the plugin's own configuration (server.getConfig), registering
|
||||
// plugin-owned RPC methods (server.registerRPC), and reading/writing files
|
||||
// inside the plugin directory.
|
||||
type PluginPermissions struct {
|
||||
Node bool `json:"node"` // Node.js compatibility modules (runtime setting, not approval-relevant)
|
||||
AllowSystemRPC bool `json:"allowSystemRPC"` // server.call: call system RPC methods with admin authority
|
||||
AllowRoutes bool `json:"allowRoutes"` // server.route: register HTTP routes on the host engine
|
||||
AllowHooks bool `json:"allowHooks"` // server.hook: modify HTTP requests/responses and intercept WebSocket connections/frames
|
||||
AllowHTMLInject bool `json:"allowHTMLInject"` // server.injectHTML: embed CSS/JS into every HTML response
|
||||
AllowExec bool `json:"allowExec"` // child_process: execute child processes
|
||||
AllowListen bool `json:"allowListen"` // net/http servers: listen on local ports
|
||||
AllowAllFileAccess bool `json:"allowAllFileAccess"` // access files outside the plugin directory
|
||||
MaxHTTPBodyBytes int64 `json:"maxHTTPBodyBytes"` // runtime limit, not approval-relevant
|
||||
MaxChildOutputBytes int `json:"maxChildOutputBytes"` // runtime limit, not approval-relevant
|
||||
TimeoutSeconds int `json:"timeout"` // per-turn execution timeout in seconds, not approval-relevant
|
||||
}
|
||||
|
||||
// PageVisibility controls who can reach a plugin page.
|
||||
type PageVisibility string
|
||||
|
||||
const (
|
||||
// PageVisibilityAdmin limits the page to the admin sidebar.
|
||||
PageVisibilityAdmin PageVisibility = "admin"
|
||||
// PageVisibilityPublic serves the page through the public route without
|
||||
// navigation, mirroring the theme public behavior.
|
||||
PageVisibilityPublic PageVisibility = "public"
|
||||
)
|
||||
|
||||
// PageType controls how a plugin page is presented.
|
||||
type PageType string
|
||||
|
||||
const (
|
||||
// PageTypeIframe embeds a static file from the plugin directory.
|
||||
PageTypeIframe PageType = "iframe"
|
||||
// PageTypeRedirect navigates to an internal site path, mirroring the
|
||||
// theme redirect configuration. The URL must stay on the same origin.
|
||||
PageTypeRedirect PageType = "redirect"
|
||||
)
|
||||
|
||||
// PluginPage declares a page provided by the plugin. An iframe page is served
|
||||
// from the plugin directory (File); a redirect page navigates to an internal
|
||||
// site path (URL). Visibility defaults to admin so existing packages keep
|
||||
// their current behavior.
|
||||
type PluginPage struct {
|
||||
File string `json:"file"`
|
||||
Title any `json:"title"`
|
||||
Icon string `json:"icon"`
|
||||
Type PageType `json:"type,omitempty"`
|
||||
URL string `json:"url,omitempty"`
|
||||
Visibility PageVisibility `json:"visibility,omitempty"`
|
||||
}
|
||||
|
||||
// PluginConfiguration stores the saved configuration values of one plugin,
|
||||
// mirroring ThemeConfiguration.
|
||||
type PluginConfiguration struct {
|
||||
Short string `json:"short" gorm:"primaryKey;unique;not null"`
|
||||
Data string `json:"data" gorm:"type:longtext" default:"{}"`
|
||||
}
|
||||
@@ -53,11 +53,6 @@ func GetAllLoadNotifications() ([]models.LoadNotification, error) {
|
||||
return notifications, nil
|
||||
}
|
||||
|
||||
func SaveLoadNotification(record models.LoadNotification) error {
|
||||
db := dbcore.GetDBInstance()
|
||||
return db.Create(&record).Error
|
||||
}
|
||||
|
||||
func ReloadLoadNotificationSchedule() error {
|
||||
db := dbcore.GetDBInstance()
|
||||
var loadNotifications []models.LoadNotification
|
||||
|
||||
@@ -126,19 +126,3 @@ func DisableTrafficReportNotifications(uuids []string) error {
|
||||
Select("client", "enable").
|
||||
Create(notifications).Error
|
||||
}
|
||||
|
||||
// GetEnabledTrafficReportByType 获取启用了指定类型报告的客户端配置
|
||||
func GetEnabledTrafficReportByType(daily, weekly, monthly bool) ([]models.TrafficReportNotification, error) {
|
||||
db := dbcore.GetDBInstance()
|
||||
var notifications []models.TrafficReportNotification
|
||||
query := db.Model(&models.TrafficReportNotification{}).Where("enable = ?", true)
|
||||
if daily {
|
||||
query = query.Where("daily = ?", true)
|
||||
} else if weekly {
|
||||
query = query.Where("weekly = ?", true)
|
||||
} else if monthly {
|
||||
query = query.Where("monthly = ?", true)
|
||||
}
|
||||
err := query.Find(¬ifications).Error
|
||||
return notifications, err
|
||||
}
|
||||
@@ -5,15 +5,6 @@ import (
|
||||
"github.com/komari-monitor/komari/database/models"
|
||||
)
|
||||
|
||||
func GetAllOidcConfigs() []models.OidcProvider {
|
||||
db := dbcore.GetDBInstance()
|
||||
var result []models.OidcProvider
|
||||
if err := db.Find(&result).Error; err != nil {
|
||||
return nil
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
func GetOidcConfigByName(name string) (*models.OidcProvider, error) {
|
||||
db := dbcore.GetDBInstance()
|
||||
var config models.OidcProvider
|
||||
|
||||
@@ -4,7 +4,7 @@ import (
|
||||
"context"
|
||||
"time"
|
||||
|
||||
"github.com/komari-monitor/komari/database/metricstore"
|
||||
"github.com/komari-monitor/komari/internal/metricstore"
|
||||
"github.com/komari-monitor/komari/database/models"
|
||||
)
|
||||
|
||||
@@ -28,6 +28,11 @@ func GetRecordsByClientAndTime(uuid string, start, end time.Time) ([]models.Reco
|
||||
return metricstore.GetRecordsByClientAndTime(context.Background(), uuid, start, end)
|
||||
}
|
||||
|
||||
// GetRecordMetricMaxByClientAndTime 获取单项监控指标在各时间桶内的最大值。
|
||||
func GetRecordMetricMaxByClientAndTime(uuid, metricName string, start, end time.Time) ([]models.Record, error) {
|
||||
return metricstore.GetRecordMetricMaxByClientAndTime(context.Background(), uuid, metricName, start, end)
|
||||
}
|
||||
|
||||
// GetRecordsByTime 获取所有客户端在时间范围内的记录。
|
||||
func GetRecordsByTime(start, end time.Time) ([]models.Record, error) {
|
||||
return metricstore.GetRecordsByTime(context.Background(), start, end)
|
||||
|
||||
+28
-9
@@ -2,11 +2,12 @@ package tasks
|
||||
|
||||
import (
|
||||
"context"
|
||||
"sort"
|
||||
"time"
|
||||
|
||||
"github.com/komari-monitor/komari/database/dbcore"
|
||||
"github.com/komari-monitor/komari/database/metricstore"
|
||||
"github.com/komari-monitor/komari/database/models"
|
||||
"github.com/komari-monitor/komari/internal/metricstore"
|
||||
"github.com/komari-monitor/komari/utils"
|
||||
"gorm.io/gorm"
|
||||
)
|
||||
@@ -106,23 +107,42 @@ func GetAllPingTasks() ([]models.PingTask, error) {
|
||||
func GetPingTasksByClient(uuid string) []models.PingTask {
|
||||
db := dbcore.GetDBInstance()
|
||||
var tasks []models.PingTask
|
||||
if err := db.Where("clients LIKE ?", `%"`+uuid+`"%`).Find(&tasks).Error; err != nil {
|
||||
if err := db.Where("clients LIKE ?", `%"`+uuid+`"%`).Order("weight ASC").Order("id ASC").Find(&tasks).Error; err != nil {
|
||||
return nil
|
||||
}
|
||||
return tasks
|
||||
}
|
||||
|
||||
func UpdatePingTaskOrder(order map[uint]int) error {
|
||||
if len(order) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
db := dbcore.GetDBInstance()
|
||||
err := db.Transaction(func(tx *gorm.DB) error {
|
||||
for id, weight := range order {
|
||||
// Validate all ids before changing any weights. The update response is
|
||||
// not a reliable existence check because some drivers report zero rows
|
||||
// when the new value equals the current value.
|
||||
ids := make([]uint, 0, len(order))
|
||||
for id := range order {
|
||||
ids = append(ids, id)
|
||||
}
|
||||
sort.Slice(ids, func(i, j int) bool { return ids[i] < ids[j] })
|
||||
|
||||
var existing int64
|
||||
if err := tx.Model(&models.PingTask{}).Where("id IN ?", ids).Count(&existing).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
if existing != int64(len(ids)) {
|
||||
return gorm.ErrRecordNotFound
|
||||
}
|
||||
|
||||
for _, id := range ids {
|
||||
weight := order[id]
|
||||
result := tx.Model(&models.PingTask{}).Where("id = ?", id).Update("weight", weight)
|
||||
if result.Error != nil {
|
||||
return result.Error
|
||||
}
|
||||
if result.RowsAffected == 0 {
|
||||
return gorm.ErrRecordNotFound
|
||||
}
|
||||
}
|
||||
return nil
|
||||
})
|
||||
@@ -149,9 +169,8 @@ func DeleteAllPingRecords() error {
|
||||
}
|
||||
|
||||
func ReloadPingSchedule() error {
|
||||
db := dbcore.GetDBInstance()
|
||||
var pingTasks []models.PingTask
|
||||
if err := db.Find(&pingTasks).Error; err != nil {
|
||||
pingTasks, err := GetAllPingTasks()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return utils.ReloadPingSchedule(pingTasks)
|
||||
|
||||
@@ -40,3 +40,81 @@ func TestClearTaskResultsByTimeBeforeUsesUTCTimeValue(t *testing.T) {
|
||||
t.Fatalf("remaining results = %#v, want boundary and new", remaining)
|
||||
}
|
||||
}
|
||||
|
||||
func TestGetAllPingTasksOrdersByWeightThenID(t *testing.T) {
|
||||
flags.DatabaseType = flags.DatabaseTypeSQLite
|
||||
flags.DatabaseFile = "file:ping_task_order?mode=memory&cache=shared"
|
||||
db := dbcore.GetDBInstance()
|
||||
|
||||
tasks := []models.PingTask{
|
||||
{Name: "third", Weight: 2, Type: "icmp", Target: "third.example", Interval: 60},
|
||||
{Name: "first", Weight: 0, Type: "icmp", Target: "first.example", Interval: 60},
|
||||
{Name: "second", Weight: 0, Type: "icmp", Target: "second.example", Interval: 60},
|
||||
}
|
||||
if err := db.Create(&tasks).Error; err != nil {
|
||||
t.Fatalf("create ping tasks: %v", err)
|
||||
}
|
||||
|
||||
ordered, err := GetAllPingTasks()
|
||||
if err != nil {
|
||||
t.Fatalf("get ordered ping tasks: %v", err)
|
||||
}
|
||||
if len(ordered) != len(tasks) {
|
||||
t.Fatalf("ordered ping task count = %d, want %d", len(ordered), len(tasks))
|
||||
}
|
||||
if ordered[0].Id != tasks[1].Id || ordered[1].Id != tasks[2].Id || ordered[2].Id != tasks[0].Id {
|
||||
t.Fatalf("ping task order = %#v, want ids [%d %d %d]", ordered, tasks[1].Id, tasks[2].Id, tasks[0].Id)
|
||||
}
|
||||
}
|
||||
|
||||
func TestUpdatePingTaskOrderAllowsUnchangedWeights(t *testing.T) {
|
||||
flags.DatabaseType = flags.DatabaseTypeSQLite
|
||||
flags.DatabaseFile = "file:ping_task_update_order?mode=memory&cache=shared"
|
||||
db := dbcore.GetDBInstance()
|
||||
|
||||
items := []models.PingTask{
|
||||
{Name: "first", Weight: 0, Type: "icmp", Target: "first.example", Interval: 60},
|
||||
{Name: "second", Weight: 1, Type: "icmp", Target: "second.example", Interval: 60},
|
||||
}
|
||||
if err := db.Create(&items).Error; err != nil {
|
||||
t.Fatalf("create ping tasks: %v", err)
|
||||
}
|
||||
|
||||
if err := UpdatePingTaskOrder(map[uint]int{
|
||||
items[0].Id: 0,
|
||||
items[1].Id: 2,
|
||||
}); err != nil {
|
||||
t.Fatalf("update ping task order: %v", err)
|
||||
}
|
||||
|
||||
var first, second models.PingTask
|
||||
if err := db.First(&first, items[0].Id).Error; err != nil {
|
||||
t.Fatalf("load first ping task: %v", err)
|
||||
}
|
||||
if err := db.First(&second, items[1].Id).Error; err != nil {
|
||||
t.Fatalf("load second ping task: %v", err)
|
||||
}
|
||||
if first.Weight != 0 || second.Weight != 2 {
|
||||
t.Fatalf("weights = [%d, %d], want [0, 2]", first.Weight, second.Weight)
|
||||
}
|
||||
}
|
||||
|
||||
func TestGetPingTasksByClientOrdersByWeightThenID(t *testing.T) {
|
||||
flags.DatabaseType = flags.DatabaseTypeSQLite
|
||||
flags.DatabaseFile = "file:ping_task_client_order?mode=memory&cache=shared"
|
||||
db := dbcore.GetDBInstance()
|
||||
|
||||
items := []models.PingTask{
|
||||
{Name: "third", Weight: 2, Clients: models.StringArray{"node-a"}, Type: "icmp", Target: "third.example", Interval: 60},
|
||||
{Name: "first", Weight: 0, Clients: models.StringArray{"node-a"}, Type: "icmp", Target: "first.example", Interval: 60},
|
||||
{Name: "other", Weight: 1, Clients: models.StringArray{"node-b"}, Type: "icmp", Target: "other.example", Interval: 60},
|
||||
}
|
||||
if err := db.Create(&items).Error; err != nil {
|
||||
t.Fatalf("create ping tasks: %v", err)
|
||||
}
|
||||
|
||||
ordered := GetPingTasksByClient("node-a")
|
||||
if len(ordered) != 2 || ordered[0].Id != items[1].Id || ordered[1].Id != items[0].Id {
|
||||
t.Fatalf("ping task order = %#v, want ids [%d %d]", ordered, items[1].Id, items[0].Id)
|
||||
}
|
||||
}
|
||||
+4
-4
@@ -3,16 +3,16 @@ package database
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"log"
|
||||
logger "github.com/komari-monitor/komari/utils/log"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
|
||||
"github.com/gin-gonic/gin"
|
||||
"github.com/komari-monitor/komari/database/dbcore"
|
||||
"github.com/komari-monitor/komari/database/metricstore"
|
||||
"github.com/komari-monitor/komari/internal/metricstore"
|
||||
"github.com/komari-monitor/komari/database/models"
|
||||
"github.com/komari-monitor/komari/pkg/config"
|
||||
"github.com/komari-monitor/komari/internal/config"
|
||||
)
|
||||
|
||||
func GetPublicInfo() (map[string]interface{}, error) {
|
||||
@@ -67,7 +67,7 @@ func GetPublicInfo() (map[string]interface{}, error) {
|
||||
tc_data := gin.H{}
|
||||
err = json.Unmarshal([]byte(tc.Data), &tc_data)
|
||||
if err != nil {
|
||||
log.Printf("%v", err)
|
||||
logger.Infof("database", "%v", err)
|
||||
}
|
||||
// Try to load theme declaration file and merge defaults for managed configuration
|
||||
// Theme declarations live in ./data/theme/<short>/komari-theme.json
|
||||
|
||||
@@ -1,148 +0,0 @@
|
||||
# Komari
|
||||
|
||||

|
||||
|
||||

|
||||
|
||||
Komariは、サーバーのパフォーマンスを監視するためのシンプルで効率的なソリューションを提供することを目的とした、軽量の自己ホスト型サーバー監視ツールです。Webインターフェースを介してサーバーのステータスを表示し、軽量エージェントを介してデータを収集します。
|
||||
|
||||
> [!WARNING]
|
||||
> Komariは、自己ホスト型の監視/制御プログラムであり、所有しているシステム、または管理権限を得ているシステムでのみ使用してください。Komariを武器化したり、許可なく展開、アクセス、永続化、コマンド実行、その他の不正利用に使用したりしないでください。実際の悪用リスクについては、Huntressの分析をご参照ください:[Komari C2 agent abuse](https://www.huntress.com/blog/komari-c2-agent-abuse)。
|
||||
> Komariの展開および運用方法については、利用者自身が責任を負います。開発者は、無許可または不正な利用、およびその結果について責任を負いません。
|
||||
> Windows端末でリモートコントロールを有効にした場合、クライアントはユーザーがログインするたびに、KomariがリモートコントロールソフトウェアであることをWindows通知で知らせます。
|
||||
|
||||
[ドキュメント](https://komari-document.pages.dev/) | [Telegramグループ](https://t.me/komari_monitor)
|
||||
|
||||
## 特徴
|
||||
|
||||
- **軽量で効率的**: リソース消費が少なく、あらゆる規模のサーバーに適しています。
|
||||
- **自己ホスト型**: データプライバシーを完全に制御でき、展開も簡単です。
|
||||
- **Webインターフェース**: 直感的な監視ダッシュボードで、使いやすいです。
|
||||
|
||||
## クイックスタート
|
||||
|
||||
### 0. クラウドホスティングによるワンクリック展開
|
||||
|
||||
- Rainyun - CNY 4.5/月
|
||||
|
||||
[](https://app.rainyun.com/apps/rca/store/6780/NzYxNzAz_)
|
||||
|
||||
- 1Panel アプリストア
|
||||
|
||||
1Panel アプリストアで利用可能です。「アプリストア」>「ユーティリティ」>「Komari」からインストールしてください。
|
||||
|
||||
### 1. ワンクリックインストールスクリプトを使用する
|
||||
|
||||
systemdを使用するディストリビューション(Ubuntu、Debianなど)に適しています。
|
||||
|
||||
```bash
|
||||
curl -fsSL https://raw.githubusercontent.com/komari-monitor/komari/main/install-komari.sh -o install-komari.sh
|
||||
chmod +x install-komari.sh
|
||||
sudo ./install-komari.sh
|
||||
```
|
||||
|
||||
### 2. Docker展開
|
||||
|
||||
1. データディレクトリを作成します:
|
||||
```bash
|
||||
mkdir -p ./data
|
||||
```
|
||||
2. Dockerコンテナを実行します:
|
||||
```bash
|
||||
docker run -d \
|
||||
-p 25774:25774 \
|
||||
-v $(pwd)/data:/app/data \
|
||||
--name komari \
|
||||
ghcr.io/komari-monitor/komari:latest
|
||||
```
|
||||
3. デフォルトのユーザー名とパスワードを表示します:
|
||||
```bash
|
||||
docker logs komari
|
||||
```
|
||||
4. ブラウザで `http://<your_server_ip>:25774` にアクセスします。
|
||||
|
||||
> [!NOTE]
|
||||
> 環境変数 `ADMIN_USERNAME` と `ADMIN_PASSWORD` を使用して、初期のユーザー名とパスワードをカスタマイズすることもできます。
|
||||
|
||||
### 3. バイナリファイル展開
|
||||
|
||||
1. Komariの[GitHubリリース](https://github.com/komari-monitor/komari/releases)ページにアクセスして、お使いのオペレーティングシステム用の最新のバイナリをダウンロードします。
|
||||
2. Komariを実行します:
|
||||
```bash
|
||||
./komari server -l 0.0.0.0:25774
|
||||
```
|
||||
3. ブラウザで `http://<your_server_ip>:25774` にアクセスします。デフォルトのポートは `25774` です。
|
||||
4. デフォルトのユーザー名とパスワードは、起動ログで確認するか、環境変数 `ADMIN_USERNAME` と `ADMIN_PASSWORD` を介して設定できます。
|
||||
|
||||
> [!NOTE]
|
||||
> バイナリに実行権限があることを確認してください(`chmod +x komari`)。データは実行ディレクトリの `data` フォルダに保存されます。
|
||||
|
||||
### 手動ビルド
|
||||
|
||||
#### 依存関係
|
||||
|
||||
- Go 1.18+ および Node.js 20+(手動ビルド用)
|
||||
|
||||
1. フロントエンドの静的ファイルをビルドします:
|
||||
```bash
|
||||
git clone https://github.com/komari-monitor/komari-web
|
||||
cd komari-web
|
||||
npm install
|
||||
npm run build
|
||||
```
|
||||
2. バックエンドをビルドします:
|
||||
```bash
|
||||
git clone https://github.com/komari-monitor/komari
|
||||
cd komari
|
||||
```
|
||||
ステップ1で生成された静的ファイルを `komari` プロジェクトのルートにある `/web/public/defaultTheme/dist` フォルダにコピーし、`komari-theme.json` と `preview.png`/`perview.png` を `/web/public/defaultTheme` にコピーします。
|
||||
```bash
|
||||
go build -o komari
|
||||
```
|
||||
3. 実行:
|
||||
```bash
|
||||
./komari server -l 0.0.0.0:25774
|
||||
```
|
||||
デフォルトのリスニングポートは `25774` です。`http://localhost:25774` にアクセスします。
|
||||
|
||||
## フロントエンド開発ガイド
|
||||
|
||||
[Komariテーマ開発ガイド | Komari](https://komari-document.pages.dev/dev/theme.html)
|
||||
|
||||
[CrowdinでKomariを翻訳する](https://crowdin.com/project/komari/invite?h=cd051bf172c9a9f7f1360e87ffb521692507706)
|
||||
|
||||
## クライアントエージェント開発ガイド
|
||||
|
||||
[Komariエージェント情報レポートおよびイベント処理ドキュメント](https://komari-document.pages.dev/dev/agent.html)
|
||||
|
||||
## 貢献
|
||||
|
||||
IssueやPull Requestを歓迎します!
|
||||
|
||||
## 謝辞
|
||||
|
||||
### 破碎工坊云 (Crash Work)
|
||||
|
||||
[破碎工坊云 - 効率的で安定した、安全な高防御サーバーとCDNソリューションを提供する専門的なクラウドコンピューティングプラットフォーム](https://www.crash.work/)
|
||||
|
||||
### DreamCloud
|
||||
|
||||
[DreamCloud - コストパフォーマンスに優れたアジア太平洋向け高防御直通](https://as211392.com/)
|
||||
|
||||
### 🚀 SharonNetworks スポンサー
|
||||
|
||||
[](https://sharon.io)
|
||||
|
||||
SharonNetworks は、あなたのビジネスの離陸を力強くサポートします!
|
||||
|
||||
アジア太平洋のデータセンターから中国最適化ネットワーク接続を提供。低レイテンシー & 高帯域幅、Tbps 級のローカル洗浄 DDoS 防御で、ビジネスとお客様の体験を守ります。コミュニティ [Telegram グループ](https://t.me/SharonNetwork) に参加すると、チャリティまたはグループ内抽選で無料利用のチャンスがあります。
|
||||
|
||||
### オープンソースコミュニティ
|
||||
|
||||
PR を送ってくれた方、テーマを作成してくれた全ての開発者
|
||||
|
||||
—— そして:こんなに暇でいられる自分に感謝
|
||||
|
||||
## Star履歴
|
||||
|
||||
[](https://www.star-history.com/#komari-monitor/komari&Date)
|
||||
@@ -1,148 +0,0 @@
|
||||
# Komari
|
||||
|
||||

|
||||
|
||||

|
||||
|
||||
Komari 是一款輕量級的自託管伺服器監控工具,旨在提供簡單、高效的伺服器性能監控解決方案。它支援透過 Web 介面查看伺服器狀態,並透過輕量級 Agent 收集數據。
|
||||
|
||||
> [!WARNING]
|
||||
> Komari 是一款自託管的監控/控制程式,僅應部署在你擁有或已獲授權管理的系統上。請勿將 Komari 武器化,或在未獲授權的情況下部署、存取、持久化、執行命令及從事其他濫用行為。關於現實中的濫用風險,可參考 Huntress 的分析:[Komari C2 agent abuse](https://www.huntress.com/blog/komari-c2-agent-abuse)。
|
||||
> 使用者需要自行承擔部署和使用 Komari 的責任。開發者不對未經授權或濫用行為及其後果承擔責任。
|
||||
> 在 Windows 端啟用遠端控制後,客戶端會在每次使用者登入時透過 Windows 通知提醒使用者 Komari 是一款遠端控制軟體。
|
||||
|
||||
[文檔](https://komari-document.pages.dev/) | [Telegram 群組](https://t.me/komari_monitor)
|
||||
|
||||
## 特性
|
||||
|
||||
- **輕量高效**:低資源佔用,適合各種規模的伺服器。
|
||||
- **自託管**:完全掌控數據隱私,部署簡單。
|
||||
- **Web 介面**:直觀的監控儀表盤,易於使用。
|
||||
|
||||
## 快速開始
|
||||
|
||||
### 0. 容器雲一鍵部署
|
||||
|
||||
- 雨雲雲應用 - CNY 4.5/月
|
||||
|
||||
[](https://app.rainyun.com/apps/rca/store/6780/NzYxNzAz_)
|
||||
|
||||
- 1Panel 應用商店
|
||||
|
||||
已上架 1Panel 應用商店,應用商店-實用工具-Komari 即可安裝
|
||||
|
||||
### 1. 使用一鍵安裝腳本
|
||||
|
||||
適用於使用了 systemd 的發行版(Ubuntu、Debian...)。
|
||||
|
||||
```bash
|
||||
curl -fsSL https://raw.githubusercontent.com/komari-monitor/komari/main/install-komari.sh -o install-komari.sh
|
||||
chmod +x install-komari.sh
|
||||
sudo ./install-komari.sh
|
||||
```
|
||||
|
||||
### 2. Docker 部署
|
||||
|
||||
1. 建立資料目錄:
|
||||
```bash
|
||||
mkdir -p ./data
|
||||
```
|
||||
2. 執行 Docker 容器:
|
||||
```bash
|
||||
docker run -d \
|
||||
-p 25774:25774 \
|
||||
-v $(pwd)/data:/app/data \
|
||||
--name komari \
|
||||
ghcr.io/komari-monitor/komari:latest
|
||||
```
|
||||
3. 查看預設帳號和密碼:
|
||||
```bash
|
||||
docker logs komari
|
||||
```
|
||||
4. 在瀏覽器中存取 `http://<your_server_ip>:25774`。
|
||||
|
||||
> [!NOTE]
|
||||
> 你也可以透過環境變數 `ADMIN_USERNAME` 和 `ADMIN_PASSWORD` 自訂初始使用者名稱和密碼。
|
||||
|
||||
### 3. 二進位檔案部署
|
||||
|
||||
1. 存取 Komari 的 [GitHub Release 頁面](https://github.com/komari-monitor/komari/releases) 下載適用於你作業系統的最新二進位檔案。
|
||||
2. 執行 Komari:
|
||||
```bash
|
||||
./komari server -l 0.0.0.0:25774
|
||||
```
|
||||
3. 在瀏覽器中存取 `http://<your_server_ip>:25774`,預設監聽 `25774` 連接埠。
|
||||
4. 預設帳號和密碼可在啟動日誌中查看,或透過環境變數 `ADMIN_USERNAME` 和 `ADMIN_PASSWORD` 設定。
|
||||
|
||||
> [!NOTE]
|
||||
> 確保二進位檔案具有可執行權限(`chmod +x komari`)。資料將保存在執行目錄下的 `data` 資料夾中。
|
||||
|
||||
### 手工建置
|
||||
|
||||
#### 依賴
|
||||
|
||||
- Go 1.18+ 和 Node.js 20+(手工建置)
|
||||
|
||||
1. 建置前端靜態檔案:
|
||||
```bash
|
||||
git clone https://github.com/komari-monitor/komari-web
|
||||
cd komari-web
|
||||
npm install
|
||||
npm run build
|
||||
```
|
||||
2. 建置後端:
|
||||
```bash
|
||||
git clone https://github.com/komari-monitor/komari
|
||||
cd komari
|
||||
```
|
||||
將步驟1中產生的靜態檔案複製到 `komari` 專案根目錄下的 `/web/public/defaultTheme/dist` 資料夾,並將 `komari-theme.json` 與 `preview.png`/`perview.png` 複製到 `/web/public/defaultTheme`。
|
||||
```bash
|
||||
go build -o komari
|
||||
```
|
||||
3. 執行:
|
||||
```bash
|
||||
./komari server -l 0.0.0.0:25774
|
||||
```
|
||||
預設監聽 `25774` 連接埠,存取 `http://localhost:25774`。
|
||||
|
||||
## 前端開發指南
|
||||
|
||||
[Komari 主題開發指南 | Komari](https://komari-document.pages.dev/dev/theme.html)
|
||||
|
||||
[在 Crowdin 上翻譯 Komari](https://crowdin.com/project/komari/invite?h=cd051bf172c9a9f7f1360e87ffb521692507706)
|
||||
|
||||
## 客戶端 Agent 開發指南
|
||||
|
||||
[Komari Agent 資訊上報與事件處理文檔](https://komari-document.pages.dev/dev/agent.html)
|
||||
|
||||
## 貢獻
|
||||
|
||||
歡迎提交 Issue 或 Pull Request!
|
||||
|
||||
## 鳴謝
|
||||
|
||||
### 破碎工坊雲
|
||||
|
||||
[破碎工坊雲 - 專業雲計算服務平台,提供高效、穩定、安全的高防伺服器與CDN解決方案](https://www.crash.work/)
|
||||
|
||||
### DreamCloud
|
||||
|
||||
[DreamCloud - 極高性價比解鎖直連亞太高防](https://as211392.com/)
|
||||
|
||||
### 🚀 由 SharonNetworks 贊助
|
||||
|
||||
[](https://sharon.io)
|
||||
|
||||
SharonNetworks 為您的業務起飛保駕護航!
|
||||
|
||||
亞太資料中心提供頂級的中國優化網路接入 · 低延遲 & 高頻寬 & 提供 Tbps 級本地清洗高防服務,為您的業務保駕護航,為您的客戶提供極致體驗。加入社群 [Telegram 群組](https://t.me/SharonNetwork) 可參與公益募捐或群內抽獎免費使用。
|
||||
|
||||
### 開源社群
|
||||
|
||||
提交 PR、製作主題的各位開發者
|
||||
|
||||
—— 以及:感謝我自己能這麼閒
|
||||
|
||||
## Star History
|
||||
|
||||
[](https://www.star-history.com/#komari-monitor/komari&Date)
|
||||
@@ -1,148 +0,0 @@
|
||||
# Komari
|
||||
|
||||

|
||||
|
||||

|
||||
|
||||
Komari 是一款轻量级的自托管服务器监控工具,旨在提供简单、高效的服务器性能监控解决方案。它支持通过 Web 界面查看服务器状态,并通过轻量级 Agent 收集数据。
|
||||
|
||||
> [!WARNING]
|
||||
> Komari 是一款自托管的监控/控制程序,仅应部署在你拥有或已获得授权管理的系统上。请勿将 Komari 武器化,或在未获授权的情况下部署、访问、持久化、执行命令及从事其他滥用行为。关于现实中的滥用风险,可参考 Huntress 的分析:[Komari C2 agent abuse](https://www.huntress.com/blog/komari-c2-agent-abuse)。
|
||||
> 用户需要自行承担部署和使用 Komari 的责任。开发者不对未经授权或滥用行为及其后果承担责任。
|
||||
> 在 Windows 端开启远程控制后,客户端会在每次用户登录时通过 Windows 通知提醒用户 Komari 是一款远程控制软件。
|
||||
|
||||
[文档](https://komari-document.pages.dev/) | [文档(镜像站 By Geekertao)](https://www.komari.wiki) | [Telegram 群组](https://t.me/komari_monitor)
|
||||
|
||||
## 特性
|
||||
|
||||
- **轻量高效**:低资源占用,适合各种规模的服务器。
|
||||
- **自托管**:完全掌控数据隐私,部署简单。
|
||||
- **Web 界面**:直观的监控仪表盘,易于使用。
|
||||
|
||||
## 快速开始
|
||||
|
||||
### 0. 容器云一键部署
|
||||
|
||||
- 雨云云应用 - CNY 4.5/月
|
||||
|
||||
[](https://app.rainyun.com/apps/rca/store/6780/NzYxNzAz_)
|
||||
|
||||
- 1Panel 应用商店
|
||||
|
||||
已上架1Panel应用商店,应用商店-实用工具-Komari 即可安装
|
||||
|
||||
### 1. 使用一键安装脚本
|
||||
|
||||
适用于使用了 systemd 的发行版(Ubuntu、Debian...)。
|
||||
|
||||
```bash
|
||||
curl -fsSL https://raw.githubusercontent.com/komari-monitor/komari/main/install-komari.sh -o install-komari.sh
|
||||
chmod +x install-komari.sh
|
||||
sudo ./install-komari.sh
|
||||
```
|
||||
|
||||
### 2. Docker 部署
|
||||
|
||||
1. 创建数据目录:
|
||||
```bash
|
||||
mkdir -p ./data
|
||||
```
|
||||
2. 运行 Docker 容器:
|
||||
```bash
|
||||
docker run -d \
|
||||
-p 25774:25774 \
|
||||
-v $(pwd)/data:/app/data \
|
||||
--name komari \
|
||||
ghcr.io/komari-monitor/komari:latest
|
||||
```
|
||||
3. 查看默认账号和密码:
|
||||
```bash
|
||||
docker logs komari
|
||||
```
|
||||
4. 在浏览器中访问 `http://<your_server_ip>:25774`。
|
||||
|
||||
> [!NOTE]
|
||||
> 你也可以通过环境变量 `ADMIN_USERNAME` 和 `ADMIN_PASSWORD` 自定义初始用户名和密码。
|
||||
|
||||
### 3. 二进制文件部署
|
||||
|
||||
1. 访问 Komari 的 [GitHub Release 页面](https://github.com/komari-monitor/komari/releases) 下载适用于你操作系统的最新二进制文件。
|
||||
2. 运行 Komari:
|
||||
```bash
|
||||
./komari server -l 0.0.0.0:25774
|
||||
```
|
||||
3. 在浏览器中访问 `http://<your_server_ip>:25774`,默认监听 `25774` 端口。
|
||||
4. 默认账号和密码可在启动日志中查看,或通过环境变量 `ADMIN_USERNAME` 和 `ADMIN_PASSWORD` 设置。
|
||||
|
||||
> [!NOTE]
|
||||
> 确保二进制文件具有可执行权限(`chmod +x komari`)。数据将保存在运行目录下的 `data` 文件夹中。
|
||||
|
||||
### 手工构建
|
||||
|
||||
#### 依赖
|
||||
|
||||
- Go 1.18+ 和 Node.js 20+(手工构建)
|
||||
|
||||
1. 构建前端静态文件:
|
||||
```bash
|
||||
git clone https://github.com/komari-monitor/komari-web
|
||||
cd komari-web
|
||||
npm install
|
||||
npm run build
|
||||
```
|
||||
2. 构建后端:
|
||||
```bash
|
||||
git clone https://github.com/komari-monitor/komari
|
||||
cd komari
|
||||
```
|
||||
将步骤1中生成的静态文件复制到 `komari` 项目根目录下的 `/web/public/defaultTheme/dist` 文件夹,并将 `komari-theme.json` 与 `preview.png`/`perview.png` 复制到 `/web/public/defaultTheme`。
|
||||
```bash
|
||||
go build -o komari
|
||||
```
|
||||
3. 运行:
|
||||
```bash
|
||||
./komari server -l 0.0.0.0:25774
|
||||
```
|
||||
默认监听 `25774` 端口,访问 `http://localhost:25774`。
|
||||
|
||||
## 前端开发指南
|
||||
|
||||
[Komari 主题开发指南 | Komari](https://komari-document.pages.dev/dev/theme.html)
|
||||
|
||||
[在 Crowdin 上翻译 Komari](https://crowdin.com/project/komari/invite?h=cd051bf172c9a9f7f1360e87ffb521692507706)
|
||||
|
||||
## 客户端 Agent 开发指南
|
||||
|
||||
[Komari Agent 信息上报与事件处理文档](https://komari-document.pages.dev/dev/agent.html)
|
||||
|
||||
## 贡献
|
||||
|
||||
欢迎提交 Issue 或 Pull Request!
|
||||
|
||||
## 鸣谢
|
||||
|
||||
### 破碎工坊云
|
||||
|
||||
[破碎工坊云 - 专业云计算服务平台,提供高效、稳定、安全的高防服务器与CDN解决方案](https://www.crash.work/)
|
||||
|
||||
### DreamCloud
|
||||
|
||||
[DreamCloud - 极高性价比解锁直连亚太高防](https://as211392.com/)
|
||||
|
||||
### 🚀 由 SharonNetworks 赞助
|
||||
|
||||
[](https://sharon.io)
|
||||
|
||||
SharonNetworks 为您的业务起飞保驾护航!
|
||||
|
||||
亚太数据中心提供顶级的中国优化网络接入 · 低延时 & 高带宽 & 提供 Tbps 级本地清洗高防服务,为您的业务保驾护航,为您的客户提供极致体验。加入社区 [Telegram 群组](https://t.me/SharonNetwork) 可参与公益募捐或群内抽奖免费使用。
|
||||
|
||||
### 开源社区
|
||||
|
||||
提交 PR、制作主题的各位开发者
|
||||
|
||||
—— 以及:感谢我自己能这么闲
|
||||
|
||||
## Star History
|
||||
|
||||
[](https://www.star-history.com/#komari-monitor/komari&Date)
|
||||
@@ -1,327 +0,0 @@
|
||||
# Visitor Audit RPC 使用说明
|
||||
|
||||
本文档说明 `public:recordVisitorEvent` RPC2 接口的调用方式。该接口用于让主题前端把访客访问和前端操作记录写入 Komari 后端审计日志。
|
||||
|
||||
## 接口概览
|
||||
|
||||
- RPC 方法:`public:recordVisitorEvent`
|
||||
- RPC 路径:`/api/rpc2`
|
||||
- HTTP 方法:`POST`
|
||||
- 写入位置:现有审计日志表 `models.Log`
|
||||
- 管理端读取:复用现有 `admin:getLogs`
|
||||
- 日志类型:`msg_type = "visitor"`
|
||||
- 默认状态:关闭,需设置 `visitor_audit_enabled = true`
|
||||
|
||||
该接口不会信任前端传入的 IP。来源 IP 和 User-Agent 由后端从请求上下文记录。
|
||||
|
||||
## 权限
|
||||
|
||||
`public:recordVisitorEvent` 属于 `public` 命名空间,访客可调用。
|
||||
|
||||
私有站点模式下,该方法也在登录页白名单内,便于记录登录前/前台访问事件。
|
||||
|
||||
这是有意保留的行为;总开关关闭时,白名单中的方法仍可调用,但不会写库。
|
||||
|
||||
## 启用
|
||||
|
||||
存量实例和新实例默认都不会开放匿名日志写入。管理员可通过
|
||||
`admin:editSettings` 将 `visitor_audit_enabled` 设为 `true`。公开设置
|
||||
`public:getPublicSettings` 也会返回这个字段,主题可据此决定是否上报。
|
||||
|
||||
未启用时接口返回 `status = "disabled"`。启用后,每个来源 IP 使用内存令牌桶限流:
|
||||
平均每分钟 30 次,允许短时突发 10 次;超限时返回 `status = "rate_limited"`,不写库。
|
||||
|
||||
## 请求参数
|
||||
|
||||
| 参数 | 类型 | 必填 | 说明 |
|
||||
| --- | --- | --- | --- |
|
||||
| `event` | `string` | 是 | 事件名,例如 `page_view`、`node_open`、`search` |
|
||||
| `action` | `string` | 否 | `event` 的别名 |
|
||||
| `operation` | `string` | 否 | `event` 的别名 |
|
||||
| `path` | `string` | 否 | 前端路径,例如 `/`、`/instance/<uuid>` |
|
||||
| `route` | `string` | 否 | 前端路由名,例如 `home`、`instance-detail` |
|
||||
| `target` | `string` | 否 | 操作目标,例如节点 UUID、工具 key、按钮 key |
|
||||
| `detail` | `object` | 否 | 额外元数据,大小受限 |
|
||||
|
||||
`event`、`action`、`operation` 三者任选一个即可,优先级为:
|
||||
|
||||
```text
|
||||
event > action > operation
|
||||
```
|
||||
|
||||
## 后端自动记录的信息
|
||||
|
||||
| 字段 | 来源 |
|
||||
| --- | --- |
|
||||
| IP | `rpc.ContextMeta.RemoteIP`,来自服务端 `c.ClientIP()` |
|
||||
| User-Agent | `rpc.ContextMeta.UserAgent`,来自请求头 |
|
||||
| 用户 UUID | 登录状态下使用 `rpc.ContextMeta.UserUUID`;访客为空 |
|
||||
| 时间 | 后端写入审计日志时生成 |
|
||||
|
||||
## 字段限制
|
||||
|
||||
字符串字段按 Unicode 字符计数,`detail` 和最终 message 按序列化后的字节数计数。
|
||||
|
||||
| 字段 | 最大长度 |
|
||||
| --- | ---: |
|
||||
| `event` | 64 |
|
||||
| `path` | 512 |
|
||||
| `route` | 128 |
|
||||
| `target` | 128 |
|
||||
| User-Agent | 512 |
|
||||
| `detail` JSON | 2048 |
|
||||
| 最终日志 message | 4096 |
|
||||
|
||||
如果 `detail` 超过限制,不会完整保存,而会写入截断标记:
|
||||
|
||||
```json
|
||||
{
|
||||
"truncated": true,
|
||||
"size": 4096
|
||||
}
|
||||
```
|
||||
|
||||
## 事件名规范化
|
||||
|
||||
服务端会规范化事件名:
|
||||
|
||||
- 转小写
|
||||
- 空格转 `_`
|
||||
- 仅保留字母、数字、`_`、`-`、`:`、`.`
|
||||
- 最大长度 64
|
||||
|
||||
示例:
|
||||
|
||||
```text
|
||||
Page View -> page_view
|
||||
node:open.detail -> node:open.detail
|
||||
```
|
||||
|
||||
## 请求示例:页面访问
|
||||
|
||||
```bash
|
||||
curl -X POST http://localhost:25774/api/rpc2 \
|
||||
-H 'Content-Type: application/json' \
|
||||
--data '{
|
||||
"jsonrpc": "2.0",
|
||||
"method": "public:recordVisitorEvent",
|
||||
"params": {
|
||||
"event": "page_view",
|
||||
"path": "/",
|
||||
"route": "home",
|
||||
"detail": {
|
||||
"theme": "glassmorphism"
|
||||
}
|
||||
},
|
||||
"id": 1
|
||||
}'
|
||||
```
|
||||
|
||||
写入成功返回:
|
||||
|
||||
```json
|
||||
{
|
||||
"jsonrpc": "2.0",
|
||||
"result": {
|
||||
"status": "success"
|
||||
},
|
||||
"id": 1
|
||||
}
|
||||
```
|
||||
|
||||
`disabled` 和 `rate_limited` 也属于正常响应,不会返回 JSON-RPC 错误;主题上报不应影响主流程。
|
||||
|
||||
## 请求示例:打开节点详情
|
||||
|
||||
```json
|
||||
{
|
||||
"jsonrpc": "2.0",
|
||||
"method": "public:recordVisitorEvent",
|
||||
"params": {
|
||||
"event": "node_open",
|
||||
"path": "/instance/6f0b-example",
|
||||
"route": "instance-detail",
|
||||
"target": "6f0b-example",
|
||||
"detail": {
|
||||
"source": "node_card"
|
||||
}
|
||||
},
|
||||
"id": 2
|
||||
}
|
||||
```
|
||||
|
||||
## 请求示例:搜索
|
||||
|
||||
```json
|
||||
{
|
||||
"jsonrpc": "2.0",
|
||||
"method": "public:recordVisitorEvent",
|
||||
"params": {
|
||||
"event": "search",
|
||||
"path": "/",
|
||||
"route": "home",
|
||||
"detail": {
|
||||
"keyword_length": 6,
|
||||
"result_count": 3
|
||||
}
|
||||
},
|
||||
"id": 3
|
||||
}
|
||||
```
|
||||
|
||||
建议不要记录完整搜索关键词,避免把敏感内容写入审计日志。
|
||||
|
||||
## 前端封装示例
|
||||
|
||||
```ts
|
||||
interface VisitorAuditEvent {
|
||||
event?: string
|
||||
action?: string
|
||||
operation?: string
|
||||
path?: string
|
||||
route?: string
|
||||
target?: string
|
||||
detail?: Record<string, unknown>
|
||||
}
|
||||
|
||||
let auditRequestId = 0
|
||||
|
||||
export async function recordVisitorEvent(event: VisitorAuditEvent): Promise<void> {
|
||||
try {
|
||||
await fetch('/api/rpc2', {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
'Content-Type': 'application/json',
|
||||
},
|
||||
credentials: 'include',
|
||||
body: JSON.stringify({
|
||||
jsonrpc: '2.0',
|
||||
method: 'public:recordVisitorEvent',
|
||||
params: event,
|
||||
id: ++auditRequestId,
|
||||
}),
|
||||
})
|
||||
}
|
||||
catch {
|
||||
// 审计上报不应影响用户正常浏览
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
## Vue Router 访问记录示例
|
||||
|
||||
```ts
|
||||
router.afterEach((to) => {
|
||||
void recordVisitorEvent({
|
||||
event: 'page_view',
|
||||
path: to.fullPath,
|
||||
route: String(to.name ?? ''),
|
||||
detail: {
|
||||
params: Object.keys(to.params),
|
||||
query_keys: Object.keys(to.query),
|
||||
},
|
||||
})
|
||||
})
|
||||
```
|
||||
|
||||
建议只记录 query key,不记录完整 query value,避免泄露 token 或敏感参数。
|
||||
|
||||
## 推荐事件名
|
||||
|
||||
| 场景 | event |
|
||||
| --- | --- |
|
||||
| 页面访问 | `page_view` |
|
||||
| 打开节点详情 | `node_open` |
|
||||
| 搜索 | `search` |
|
||||
| 切换分组 | `group_change` |
|
||||
| 切换视图模式 | `view_mode_change` |
|
||||
| 点击管理入口 | `admin_entry_click` |
|
||||
| 打开高级工具 | `home_tool_open` |
|
||||
| 审计日志刷新 | `audit_refresh` |
|
||||
| 审计日志翻页 | `audit_page_change` |
|
||||
| 导出 JSON | `export_json` |
|
||||
| 导出 CSV | `export_csv` |
|
||||
| WebRTC 自检开始 | `webrtc_check_start` |
|
||||
| WebRTC 自检完成 | `webrtc_check_done` |
|
||||
|
||||
## 审计日志保存形态
|
||||
|
||||
写入现有 `models.Log`:
|
||||
|
||||
| 字段 | 值 |
|
||||
| --- | --- |
|
||||
| `ip` | 服务端看到的来源 IP |
|
||||
| `uuid` | 登录用户 UUID;访客为空 |
|
||||
| `msg_type` | `visitor` |
|
||||
| `message` | `visitor event: {...}` |
|
||||
| `time` | 后端写入时间 |
|
||||
|
||||
示例 `message`:
|
||||
|
||||
```text
|
||||
visitor event: {"event":"page_view","path":"/","route":"home","user_agent":"Mozilla/5.0 ...","detail":{"theme":"glassmorphism"}}
|
||||
```
|
||||
|
||||
## 管理端读取
|
||||
|
||||
复用现有接口:
|
||||
|
||||
```text
|
||||
admin:getLogs
|
||||
```
|
||||
|
||||
请求示例:
|
||||
|
||||
```json
|
||||
{
|
||||
"jsonrpc": "2.0",
|
||||
"method": "admin:getLogs",
|
||||
"params": {
|
||||
"limit": "100",
|
||||
"page": "1",
|
||||
"msg_type": "visitor"
|
||||
},
|
||||
"id": 10
|
||||
}
|
||||
```
|
||||
|
||||
`msg_type` 为可选的精确匹配过滤参数。传入 `visitor` 后,计数和分页都会在 SQL 查询层过滤,
|
||||
无需先拉取所有日志再由前端筛选。
|
||||
|
||||
## 安全注意事项
|
||||
|
||||
### 不要从前端传 IP
|
||||
|
||||
前端 IP 不可信。该接口会自动使用服务端看到的来源 IP。
|
||||
|
||||
### 不要记录敏感值
|
||||
|
||||
不建议记录:
|
||||
|
||||
- 密码
|
||||
- token
|
||||
- cookie
|
||||
- 完整 URL query
|
||||
- 完整搜索关键词
|
||||
- 导出内容
|
||||
- WebSSH 命令内容
|
||||
- 剪贴板内容
|
||||
|
||||
建议只记录:
|
||||
|
||||
- 操作类型
|
||||
- 路由
|
||||
- 目标 ID
|
||||
- 结果数量
|
||||
- 布尔状态
|
||||
- 非敏感摘要
|
||||
|
||||
### 上报不要阻塞主流程
|
||||
|
||||
前端调用建议使用:
|
||||
|
||||
```ts
|
||||
void recordVisitorEvent(...)
|
||||
```
|
||||
|
||||
即使上报失败,也不影响页面功能。
|
||||
@@ -10,20 +10,23 @@ require (
|
||||
github.com/google/uuid v1.6.0
|
||||
github.com/gorilla/websocket v1.5.3
|
||||
github.com/jackc/pgx/v5 v5.10.0
|
||||
github.com/klauspost/compress v1.17.11
|
||||
github.com/mattn/go-sqlite3 v1.14.27
|
||||
github.com/oschwald/maxminddb-golang v1.13.1
|
||||
github.com/patrickmn/go-cache v2.1.0+incompatible
|
||||
github.com/pquerna/otp v1.5.0
|
||||
github.com/spf13/cobra v1.9.1
|
||||
github.com/stretchr/testify v1.11.1
|
||||
golang.org/x/image v0.35.0
|
||||
golang.org/x/sys v0.33.0
|
||||
gorm.io/driver/sqlite v1.5.7
|
||||
gorm.io/gorm v1.26.1
|
||||
)
|
||||
|
||||
require github.com/dop251/base64dec v0.0.0-20231022112746-c6c9f9a96217 // indirect
|
||||
|
||||
// Misuse of ServerConfig.PublicKeyCallback may cause authorization bypass in golang.org/x/crypto #1
|
||||
// golang.org/x/crypto Vulnerable to Denial of Service (DoS) via Slow or Incomplete Key Exchange #3
|
||||
require golang.org/x/crypto v0.39.0 // indirect
|
||||
require golang.org/x/crypto v0.39.0
|
||||
|
||||
// HTTP Proxy bypass using IPv6 Zone IDs in golang.org/x/net #2
|
||||
// golang.org/x/net vulnerable to Cross-site Scripting #4
|
||||
@@ -66,9 +69,8 @@ require (
|
||||
github.com/twitchyliquid64/golang-asm v0.15.1 // indirect
|
||||
github.com/ugorji/go/codec v1.2.12 // indirect
|
||||
golang.org/x/arch v0.15.0 // indirect
|
||||
golang.org/x/sync v0.19.0 // indirect
|
||||
golang.org/x/sys v0.33.0 // indirect
|
||||
golang.org/x/text v0.33.0 // indirect
|
||||
golang.org/x/sync v0.19.0
|
||||
golang.org/x/text v0.33.0
|
||||
google.golang.org/protobuf v1.36.6 // indirect
|
||||
gopkg.in/yaml.v3 v3.0.1 // indirect
|
||||
)
|
||||
@@ -19,6 +19,8 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c
|
||||
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/dlclark/regexp2 v1.11.4 h1:rPYF9/LECdNymJufQKmri9gV604RvvABwgOA8un7yAo=
|
||||
github.com/dlclark/regexp2 v1.11.4/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8=
|
||||
github.com/dop251/base64dec v0.0.0-20231022112746-c6c9f9a96217 h1:16iT9CBDOniJwFGPI41MbUDfEk74hFaKTqudrX8kenY=
|
||||
github.com/dop251/base64dec v0.0.0-20231022112746-c6c9f9a96217/go.mod h1:eIb+f24U+eWQCIsj9D/ah+MD9UP+wdxuqzsdLD+mhGM=
|
||||
github.com/dop251/goja v0.0.0-20251008123653-cf18d89f3cf6 h1:6dE1TmjqkY6tehR4A67gDNhvDtuZ54ocu7ab4K9o540=
|
||||
github.com/dop251/goja v0.0.0-20251008123653-cf18d89f3cf6/go.mod h1:MxLav0peU43GgvwVgNbLAj1s/bSGboKkhuULvq/7hx4=
|
||||
github.com/dop251/goja_nodejs v0.0.0-20251015164255-5e94316bedaf h1:gbmvliZnCut4NjaPSNOQlfqBoZ9C5Dpf72mHMMYhgVE=
|
||||
@@ -68,6 +70,8 @@ github.com/jinzhu/now v1.1.5 h1:/o9tlHleP7gOFmsnYNz3RGnqzefHA47wQpKrrdTIwXQ=
|
||||
github.com/jinzhu/now v1.1.5/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8=
|
||||
github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM=
|
||||
github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo=
|
||||
github.com/klauspost/compress v1.17.11 h1:In6xLpyWOi1+C7tXUUWv2ot1QvBjxevKAaI6IXrJmUc=
|
||||
github.com/klauspost/compress v1.17.11/go.mod h1:pMDklpSncoRMuLFrf1W9Ss9KT+0rH90U12bZKk7uwG0=
|
||||
github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
|
||||
github.com/klauspost/cpuid/v2 v2.2.10 h1:tBs3QSyvjDyFTq3uoc/9xFpCuOsJQFNPiAhYdw2skhE=
|
||||
github.com/klauspost/cpuid/v2 v2.2.10/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0=
|
||||
@@ -125,12 +129,12 @@ github.com/twitchyliquid64/golang-asm v0.15.1 h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS
|
||||
github.com/twitchyliquid64/golang-asm v0.15.1/go.mod h1:a1lVb/DtPvCB8fslRZhAngC2+aY1QWCk3Cedj/Gdt08=
|
||||
github.com/ugorji/go/codec v1.2.12 h1:9LC83zGrHhuUA9l16C9AHXAqEV/2wBQ4nkvumAE65EE=
|
||||
github.com/ugorji/go/codec v1.2.12/go.mod h1:UNopzCgEMSXjBc6AOMqYvWC1ktqTAfzJZUZgYf6w6lg=
|
||||
go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto=
|
||||
go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE=
|
||||
golang.org/x/arch v0.15.0 h1:QtOrQd0bTUnhNVNndMpLHNWrDmYzZ2KDqSrEymqInZw=
|
||||
golang.org/x/arch v0.15.0/go.mod h1:JmwW7aLIoRUKgaTzhkiEFxvcEiQGyOg9BMonBJUS7EE=
|
||||
golang.org/x/crypto v0.39.0 h1:SHs+kF4LP+f+p14esP5jAoDpHU8Gu/v9lFRK6IT5imM=
|
||||
golang.org/x/crypto v0.39.0/go.mod h1:L+Xg3Wf6HoL4Bn4238Z6ft6KfEpN0tJGo53AAPC632U=
|
||||
golang.org/x/image v0.35.0 h1:LKjiHdgMtO8z7Fh18nGY6KDcoEtVfsgLDPeLyguqb7I=
|
||||
golang.org/x/image v0.35.0/go.mod h1:MwPLTVgvxSASsxdLzKrl8BRFuyqMyGhLwmC+TO1Sybk=
|
||||
golang.org/x/net v0.41.0 h1:vBTly1HeNPEn3wtREYfy4GZ/NECgw2Cnl+nK6Nz3uvw=
|
||||
golang.org/x/net v0.41.0/go.mod h1:B/K4NNqkfmg07DQYrbwvSluqCJOOXwUjeb/5lOisjbA=
|
||||
golang.org/x/sync v0.19.0 h1:vV+1eWNmZ5geRlYjzm2adRgW2/mcpevXNg50YZtPCE4=
|
||||
|
||||
+3
-12
@@ -368,13 +368,7 @@ install_binary() {
|
||||
if systemctl is-active --quiet ${SERVICE_NAME}.service; then
|
||||
log_success "Komari 服务启动成功"
|
||||
|
||||
log_step "正在获取初始密码..."
|
||||
sleep 5
|
||||
local password=$(journalctl -u ${SERVICE_NAME} --since "1 minute ago" | grep "admin account created." | tail -n 1 | sed -e 's/.*admin account created.//')
|
||||
if [ -z "$password" ]; then
|
||||
log_error "未能获取初始密码,请检查日志"
|
||||
fi
|
||||
show_access_info "$password" "$LISTEN_PORT"
|
||||
show_access_info "$LISTEN_PORT"
|
||||
else
|
||||
ui_msgbox "错误" "Komari 服务启动失败。\n\n查看日志: journalctl -u ${SERVICE_NAME} -f"
|
||||
return 1
|
||||
@@ -408,16 +402,13 @@ EOF
|
||||
|
||||
# Show access information
|
||||
show_access_info() {
|
||||
local password=$1
|
||||
local port=${2:-$DEFAULT_PORT}
|
||||
local port=${1:-$DEFAULT_PORT}
|
||||
local ip=$(hostname -I | awk '{print $1}')
|
||||
|
||||
local content="安装完成!\n\n"
|
||||
content+="访问信息:\n"
|
||||
content+=" URL: http://${ip}:${port}\n"
|
||||
if [ -n "$password" ]; then
|
||||
content+=" 初始登录信息(仅显示一次): ${password}\n"
|
||||
fi
|
||||
content+="\n首次使用请访问上述地址,按安装向导创建管理员账号。\n"
|
||||
content+="\n服务管理命令:\n"
|
||||
content+=" 状态: systemctl status $SERVICE_NAME\n"
|
||||
content+=" 启动: systemctl start $SERVICE_NAME\n"
|
||||
|
||||
@@ -3,11 +3,11 @@ package config
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"reflect"
|
||||
"strings"
|
||||
"sync"
|
||||
|
||||
logger "github.com/komari-monitor/komari/utils/log"
|
||||
"gorm.io/gorm"
|
||||
"gorm.io/gorm/clause"
|
||||
)
|
||||
@@ -105,7 +105,7 @@ func GetMany(keys map[string]any) (map[string]any, error) {
|
||||
// 序列化后加入待写入列表
|
||||
jsonBytes, err := json.Marshal(def)
|
||||
if err != nil {
|
||||
slog.Warn("marshal default value failed", "key", k, "error", err)
|
||||
logger.Warn("config", "marshal default value failed", "key", k, "error", err)
|
||||
continue
|
||||
}
|
||||
toInsert = append(toInsert, ConfigItem{
|
||||
@@ -122,7 +122,7 @@ func GetMany(keys map[string]any) (map[string]any, error) {
|
||||
Columns: []clause.Column{{Name: "key"}},
|
||||
DoUpdates: clause.AssignmentColumns([]string{"value"}),
|
||||
}).Create(&toInsert).Error; err != nil {
|
||||
slog.Warn("batch insert default config failed", "error", err)
|
||||
logger.Warn("config", "batch insert default config failed", "error", err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -199,18 +199,18 @@ func GetManyAs[T any]() (*T, error) {
|
||||
if dbValue, found := foundItems[fi.key]; found {
|
||||
// 数据库中存在,使用数据库值
|
||||
if err := unmarshalToField(dbValue, fieldVal); err != nil {
|
||||
slog.Warn("unmarshal config failed", "key", fi.key, "error", err)
|
||||
logger.Warn("config", "unmarshal config failed", "key", fi.key, "error", err)
|
||||
}
|
||||
} else if fi.hasDefault {
|
||||
// 数据库中不存在,但有 default tag,解析默认值并写入数据库
|
||||
if err := parseDefaultToField(fi.defaultVal, fieldVal); err != nil {
|
||||
slog.Warn("parse default value failed", "key", fi.key, "error", err)
|
||||
logger.Warn("config", "parse default value failed", "key", fi.key, "error", err)
|
||||
continue
|
||||
}
|
||||
// 序列化后写入数据库
|
||||
jsonBytes, err := json.Marshal(fieldVal.Interface())
|
||||
if err != nil {
|
||||
slog.Warn("marshal default value failed", "key", fi.key, "error", err)
|
||||
logger.Warn("config", "marshal default value failed", "key", fi.key, "error", err)
|
||||
continue
|
||||
}
|
||||
toInsert = append(toInsert, ConfigItem{
|
||||
@@ -227,7 +227,7 @@ func GetManyAs[T any]() (*T, error) {
|
||||
Columns: []clause.Column{{Name: "key"}},
|
||||
DoUpdates: clause.AssignmentColumns([]string{"value"}),
|
||||
}).Create(&toInsert).Error; err != nil {
|
||||
slog.Warn("batch insert default config failed", "error", err)
|
||||
logger.Warn("config", "batch insert default config failed", "error", err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -74,4 +74,6 @@ const (
|
||||
TrafficLimitPercentageKey = "traffic_limit_percentage"
|
||||
UpdatedAtKey = "updated_at"
|
||||
XtermjsSettingsKey = "xtermjs_settings"
|
||||
ThemeMarketSourcesKey = "theme_market_sources"
|
||||
PluginMarketSourcesKey = "plugin_market_sources"
|
||||
)
|
||||
@@ -0,0 +1,24 @@
|
||||
// Package lifecycle carries process-level requests without coupling web RPC
|
||||
// handlers to the server runtime.
|
||||
package lifecycle
|
||||
|
||||
// RestartReason identifies why the current process must restart.
|
||||
type RestartReason string
|
||||
|
||||
const RestartForMetricStoreStructureUpgrade RestartReason = "metric-store-structure-upgrade"
|
||||
|
||||
var restartRequests = make(chan RestartReason, 1)
|
||||
|
||||
// RequestRestart asks the running server to shut down cleanly and exit. A
|
||||
// pending request is sufficient because the process will be replaced.
|
||||
func RequestRestart(reason RestartReason) {
|
||||
select {
|
||||
case restartRequests <- reason:
|
||||
default:
|
||||
}
|
||||
}
|
||||
|
||||
// RestartRequests returns process restart requests for the server runtime.
|
||||
func RestartRequests() <-chan RestartReason {
|
||||
return restartRequests
|
||||
}
|
||||
@@ -0,0 +1,60 @@
|
||||
package metricstore
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"time"
|
||||
)
|
||||
|
||||
func Compact(ctx context.Context, now time.Time) (int, error) {
|
||||
if !compactOperations.TryAcquire() {
|
||||
return 0, ErrCompactInProgress
|
||||
}
|
||||
defer compactOperations.Release()
|
||||
if err := storeOperations.AcquireShared(ctx); err != nil {
|
||||
return 0, fmt.Errorf("wait for metric store operation before compaction: %w", err)
|
||||
}
|
||||
defer storeOperations.ReleaseShared()
|
||||
|
||||
storeMu.RLock()
|
||||
defer storeMu.RUnlock()
|
||||
activeStore := store
|
||||
if activeStore == nil {
|
||||
return 0, fmt.Errorf("metric store not initialized")
|
||||
}
|
||||
total, err := activeStore.Flush(ctx, now)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("flush closed metric minutes: %w", err)
|
||||
}
|
||||
coarse, err := activeStore.FlushCoarse(ctx, now)
|
||||
if err != nil {
|
||||
return total, fmt.Errorf("seal closed metric rollups: %w", err)
|
||||
}
|
||||
return total + coarse, nil
|
||||
}
|
||||
|
||||
// CleanupExpired applies retention in a separate, low-frequency operation.
|
||||
// It intentionally shares Compact's gate so delete-heavy maintenance never
|
||||
// overlaps minute or coarse-rollup persistence.
|
||||
func CleanupExpired(ctx context.Context, now time.Time) (int64, error) {
|
||||
if !compactOperations.TryAcquire() {
|
||||
return 0, ErrCompactInProgress
|
||||
}
|
||||
defer compactOperations.Release()
|
||||
if err := storeOperations.AcquireShared(ctx); err != nil {
|
||||
return 0, fmt.Errorf("wait for metric store operation before retention cleanup: %w", err)
|
||||
}
|
||||
defer storeOperations.ReleaseShared()
|
||||
|
||||
storeMu.RLock()
|
||||
defer storeMu.RUnlock()
|
||||
activeStore := store
|
||||
if activeStore == nil {
|
||||
return 0, fmt.Errorf("metric store not initialized")
|
||||
}
|
||||
deleted, err := activeStore.CleanupExpired(ctx, now)
|
||||
if err != nil {
|
||||
return deleted, fmt.Errorf("clean up expired metric data: %w", err)
|
||||
}
|
||||
return deleted, nil
|
||||
}
|
||||
@@ -0,0 +1,323 @@
|
||||
package metricstore
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/komari-monitor/komari/pkg/metric"
|
||||
)
|
||||
|
||||
const (
|
||||
// DefaultRollupRawRetention documents the fixed in-memory exact-sample
|
||||
// window. Samples older than one minute are losslessly byte-encoded;
|
||||
// older history is served by the persisted rollup ladder.
|
||||
DefaultRollupRawRetention = 10 * time.Minute
|
||||
DefaultRollupFinestTier = time.Minute
|
||||
defaultRollupPointLimit = 600
|
||||
|
||||
defaultRollupMinuteRetentionMinutes = defaultRollupPointLimit
|
||||
defaultRollupFiveMinuteRetentionMinutes = 5 * defaultRollupPointLimit
|
||||
defaultRollupHourRetentionHours = defaultRollupPointLimit
|
||||
defaultRollupDayRetentionDays = 100 * 365
|
||||
)
|
||||
|
||||
// MetricStoreConfig 保存 metric store 配置。
|
||||
//
|
||||
// 注意:metric store 现在始终启用(旧的 metric_store_enabled 开关已废弃)。
|
||||
// 未显式配置时默认使用 SQLite(./data/metrics.db)。
|
||||
type MetricStoreConfig struct {
|
||||
Driver string `json:"metric_db_driver" default:"sqlite"` // 数据库类型: sqlite, mysql, postgresql
|
||||
DSN string `json:"metric_db_dsn" default:"./data/metrics.db"` // 数据库连接串
|
||||
TablePrefix string `json:"metric_table_prefix" default:"metric_"` // 表名前缀
|
||||
MaxOpenConns int `json:"metric_max_open_conns" default:"25"` // 最大连接数
|
||||
MaxIdleConns int `json:"metric_max_idle_conns" default:"5"` // 最大空闲连接数
|
||||
// RollupMinuteRetentionMinutes controls the persisted 1-minute bucket window.
|
||||
RollupMinuteRetentionMinutes int `json:"metric_rollup_minute_retention_minutes" default:"600"`
|
||||
// RollupFiveMinuteRetentionMinutes controls the persisted 5-minute bucket window.
|
||||
RollupFiveMinuteRetentionMinutes int `json:"metric_rollup_five_minute_retention_minutes" default:"3000"`
|
||||
// RollupHourRetentionHours controls the persisted 1-hour bucket window.
|
||||
RollupHourRetentionHours int `json:"metric_rollup_hour_retention_hours" default:"600"`
|
||||
}
|
||||
|
||||
// MetricStoreConfigKeys 配置键
|
||||
const (
|
||||
MetricDBDriverKey = "metric_db_driver"
|
||||
MetricDBDSNKey = "metric_db_dsn"
|
||||
MetricTablePrefixKey = "metric_table_prefix"
|
||||
MetricMaxOpenConnsKey = "metric_max_open_conns"
|
||||
MetricMaxIdleConnsKey = "metric_max_idle_conns"
|
||||
MetricRollupMinuteRetentionMinutesKey = "metric_rollup_minute_retention_minutes"
|
||||
MetricRollupFiveMinuteRetentionMinutesKey = "metric_rollup_five_minute_retention_minutes"
|
||||
MetricRollupHourRetentionHoursKey = "metric_rollup_hour_retention_hours"
|
||||
// MigrationTargetKey 记录上一次成功完成手动迁移的目标指纹(driver+dsn),
|
||||
// 用于在下一次管理员手动迁移时推断默认源库。
|
||||
MigrationTargetKey = "metric_migration_target"
|
||||
)
|
||||
|
||||
func targetFingerprint(cfg *MetricStoreConfig) string {
|
||||
driver := ResolveDriverFromConfig(cfg.Driver, cfg.DSN)
|
||||
dsn := strings.TrimSpace(cfg.DSN)
|
||||
return fmt.Sprintf("%s|%s", driver, dsn)
|
||||
}
|
||||
|
||||
// buildMetricConfig 根据 MetricStoreConfig 构造底层 metric.Config。
|
||||
// autoMigrate 控制是否在 Open 时自动建表:正式初始化/热加载时为 true,
|
||||
// 仅做连接测试时为 false(不写入 schema,避免对目标库产生副作用)。
|
||||
func buildMetricConfig(cfg *MetricStoreConfig, autoMigrate bool) (metric.Config, error) {
|
||||
if cfg == nil {
|
||||
return metric.Config{}, fmt.Errorf("metric store config is nil")
|
||||
}
|
||||
driver := ResolveDriverFromConfig(cfg.Driver, cfg.DSN)
|
||||
|
||||
tablePrefix := cfg.TablePrefix
|
||||
if tablePrefix == "" {
|
||||
tablePrefix = "metric_"
|
||||
}
|
||||
opts := []metric.Option{
|
||||
metric.WithTablePrefix(tablePrefix),
|
||||
metric.WithAutoMigrate(autoMigrate),
|
||||
}
|
||||
policy, err := rollupPolicyFromConfig(cfg)
|
||||
if err != nil {
|
||||
return metric.Config{}, err
|
||||
}
|
||||
opts = append(opts, metric.WithRollupPolicy(policy))
|
||||
|
||||
switch driver {
|
||||
case metric.DriverSQLite:
|
||||
dsn := cfg.DSN
|
||||
if dsn == "" || dsn == "./data/metrics.db" {
|
||||
// 注意:刻意不使用 cache=shared。SQLite 共享缓存模式使用表级锁,
|
||||
// 当一个连接持有读锁、另一个连接尝试写入时会立即返回
|
||||
// SQLITE_LOCKED("database table is locked"),且 busy_timeout
|
||||
// 对共享缓存的表级锁无效,迁移期间与前台查询/实时写入并发时必然报错。
|
||||
// _txlock=immediate 让写事务开始即获取写锁,避免锁升级死锁。
|
||||
dsn = "file:./data/metrics.db?mode=rwc&_txlock=immediate"
|
||||
} else {
|
||||
// 用户自定义 DSN 时,剥离 cache=shared,避免上述表级锁问题。
|
||||
dsn = stripSharedCache(dsn)
|
||||
}
|
||||
// SQLite 串行化写入:固定单写连接以避免 "database is locked" 竞争,
|
||||
// 同时启用独立的 WAL 只读连接池提升前台查询并发(写仍走单主连接)。
|
||||
// 这里刻意忽略 cfg.MaxOpenConns/MaxIdleConns —— 对 SQLite 而言多写连接
|
||||
// 只会引入锁竞争而非提升吞吐。
|
||||
opts = append(opts, metric.WithMaxOpenConns(1), metric.WithMaxIdleConns(1))
|
||||
opts = append(opts, metric.WithSQLiteReadPool(2))
|
||||
return metric.SQLite(dsn, opts...), nil
|
||||
case metric.DriverMySQL:
|
||||
opts = append(opts,
|
||||
metric.WithMaxOpenConns(cfg.MaxOpenConns),
|
||||
metric.WithMaxIdleConns(cfg.MaxIdleConns),
|
||||
)
|
||||
return metric.MySQL(cfg.DSN, opts...), nil
|
||||
case metric.DriverPostgreSQL:
|
||||
opts = append(opts,
|
||||
metric.WithMaxOpenConns(cfg.MaxOpenConns),
|
||||
metric.WithMaxIdleConns(cfg.MaxIdleConns),
|
||||
)
|
||||
return metric.PostgreSQL(cfg.DSN, opts...), nil
|
||||
default:
|
||||
return metric.Config{}, fmt.Errorf("unsupported metric database driver: %s", cfg.Driver)
|
||||
}
|
||||
}
|
||||
|
||||
func defaultRollupPolicy() metric.RollupPolicy {
|
||||
return rollupPolicyFromValues(
|
||||
defaultRollupMinuteRetentionMinutes,
|
||||
defaultRollupFiveMinuteRetentionMinutes,
|
||||
defaultRollupHourRetentionHours,
|
||||
)
|
||||
}
|
||||
|
||||
func rollupPolicyFromConfig(cfg *MetricStoreConfig) (metric.RollupPolicy, error) {
|
||||
if cfg == nil {
|
||||
return metric.RollupPolicy{}, fmt.Errorf("metric store config is nil")
|
||||
}
|
||||
|
||||
minuteRetention := cfg.RollupMinuteRetentionMinutes
|
||||
fiveMinuteRetention := cfg.RollupFiveMinuteRetentionMinutes
|
||||
hourRetention := cfg.RollupHourRetentionHours
|
||||
// Configs constructed by older callers do not have the new fields. Treat
|
||||
// their zero values as omitted so recovery and migration remain compatible.
|
||||
if minuteRetention == 0 {
|
||||
minuteRetention = defaultRollupMinuteRetentionMinutes
|
||||
}
|
||||
if fiveMinuteRetention == 0 {
|
||||
fiveMinuteRetention = defaultRollupFiveMinuteRetentionMinutes
|
||||
}
|
||||
if hourRetention == 0 {
|
||||
hourRetention = defaultRollupHourRetentionHours
|
||||
}
|
||||
if minuteRetention < 0 || fiveMinuteRetention < 0 || hourRetention < 0 {
|
||||
return metric.RollupPolicy{}, fmt.Errorf("metric rollup retention values must be positive integers")
|
||||
}
|
||||
|
||||
minuteDuration, err := rollupDuration(minuteRetention, time.Minute)
|
||||
if err != nil {
|
||||
return metric.RollupPolicy{}, err
|
||||
}
|
||||
fiveMinuteDuration, err := rollupDuration(fiveMinuteRetention, time.Minute)
|
||||
if err != nil {
|
||||
return metric.RollupPolicy{}, err
|
||||
}
|
||||
hourDuration, err := rollupDuration(hourRetention, time.Hour)
|
||||
if err != nil {
|
||||
return metric.RollupPolicy{}, err
|
||||
}
|
||||
|
||||
policy := rollupPolicyFromDurations(minuteDuration, fiveMinuteDuration, hourDuration)
|
||||
if err := policy.Validate(); err != nil {
|
||||
return metric.RollupPolicy{}, fmt.Errorf("invalid metric rollup retention policy: %w", err)
|
||||
}
|
||||
return policy, nil
|
||||
}
|
||||
|
||||
func rollupPolicyFromValues(minuteRetentionMinutes, fiveMinuteRetentionMinutes, hourRetentionHours int) metric.RollupPolicy {
|
||||
return rollupPolicyFromDurations(
|
||||
time.Duration(minuteRetentionMinutes)*time.Minute,
|
||||
time.Duration(fiveMinuteRetentionMinutes)*time.Minute,
|
||||
time.Duration(hourRetentionHours)*time.Hour,
|
||||
)
|
||||
}
|
||||
|
||||
func rollupPolicyFromDurations(minuteRetention, fiveMinuteRetention, hourRetention time.Duration) metric.RollupPolicy {
|
||||
return metric.RollupPolicy{
|
||||
RawRetention: DefaultRollupRawRetention,
|
||||
Tiers: []metric.RollupTier{
|
||||
{Interval: time.Minute, Retention: minuteRetention},
|
||||
{Interval: 5 * time.Minute, Retention: fiveMinuteRetention},
|
||||
{Interval: time.Hour, Retention: hourRetention},
|
||||
// Daily buckets form the terminal tier. They remain available until
|
||||
// the metric's own retention policy removes them.
|
||||
{Interval: 24 * time.Hour, Retention: time.Duration(defaultRollupDayRetentionDays) * 24 * time.Hour},
|
||||
},
|
||||
Compression: 30,
|
||||
}
|
||||
}
|
||||
|
||||
func rollupDuration(value int, unit time.Duration) (time.Duration, error) {
|
||||
maxDurationValue := int64((time.Duration(1<<63 - 1)) / unit)
|
||||
if value <= 0 || int64(value) > maxDurationValue {
|
||||
return 0, fmt.Errorf("metric rollup retention value must be a positive duration")
|
||||
}
|
||||
return time.Duration(value) * unit, nil
|
||||
}
|
||||
|
||||
// ResolveDriverFromConfig 根据 DSN 自动推断 metrics 数据库类型;当 DSN 不能可靠
|
||||
// 识别时回退到旧配置中的 driver,以兼容已有配置和非常规 DSN。
|
||||
func ResolveDriverFromConfig(configuredDriver, dsn string) metric.Driver {
|
||||
if driver, ok := InferDriverFromDSN(dsn); ok {
|
||||
return driver
|
||||
}
|
||||
|
||||
switch driver := metric.Driver(strings.ToLower(strings.TrimSpace(configuredDriver))); driver {
|
||||
case metric.DriverSQLite, metric.DriverMySQL, metric.DriverPostgreSQL:
|
||||
return driver
|
||||
default:
|
||||
return metric.DriverSQLite
|
||||
}
|
||||
}
|
||||
|
||||
// InferDriverFromDSN 尽量根据常见 DSN 格式推断数据库类型。
|
||||
// 返回 ok=false 表示格式不够明确,调用方应使用已有配置作为兜底。
|
||||
func InferDriverFromDSN(dsn string) (metric.Driver, bool) {
|
||||
raw := strings.TrimSpace(dsn)
|
||||
if raw == "" {
|
||||
return metric.DriverSQLite, true
|
||||
}
|
||||
lower := strings.ToLower(raw)
|
||||
|
||||
// PostgreSQL URL DSN: postgres://... 或 postgresql://...
|
||||
if strings.HasPrefix(lower, "postgres://") || strings.HasPrefix(lower, "postgresql://") {
|
||||
return metric.DriverPostgreSQL, true
|
||||
}
|
||||
|
||||
// SQLite 常见文件/内存 DSN。
|
||||
if raw == ":memory:" || strings.HasPrefix(lower, "file:") || strings.HasPrefix(lower, "sqlite://") || strings.HasPrefix(lower, "sqlite3://") {
|
||||
return metric.DriverSQLite, true
|
||||
}
|
||||
|
||||
// MySQL URL(虽然 go-sql-driver/mysql 原生 DSN 通常不是 URL,但这里用于给出
|
||||
// 类型推断;连接测试仍会校验 DSN 是否可被驱动接受)。
|
||||
if strings.HasPrefix(lower, "mysql://") {
|
||||
return metric.DriverMySQL, true
|
||||
}
|
||||
|
||||
// PostgreSQL 关键字/值 DSN: host=... user=... dbname=...
|
||||
if looksLikePostgreSQLKeyValueDSN(lower) {
|
||||
return metric.DriverPostgreSQL, true
|
||||
}
|
||||
|
||||
// go-sql-driver/mysql DSN: user:pass@tcp(host:3306)/db、user@unix(...)/db、user:pass@/db 等。
|
||||
if looksLikeMySQLDSN(lower) {
|
||||
return metric.DriverMySQL, true
|
||||
}
|
||||
|
||||
// SQLite 路径:./data/metrics.db、/var/lib/metrics.sqlite3、metrics.sqlite 等。
|
||||
if looksLikeSQLitePath(lower) {
|
||||
return metric.DriverSQLite, true
|
||||
}
|
||||
|
||||
return "", false
|
||||
}
|
||||
|
||||
func looksLikePostgreSQLKeyValueDSN(lower string) bool {
|
||||
if !strings.Contains(lower, "=") || strings.Contains(lower, "://") {
|
||||
return false
|
||||
}
|
||||
keys := []string{"host=", "user=", "password=", "dbname=", "port=", "sslmode="}
|
||||
matched := 0
|
||||
for _, key := range keys {
|
||||
if strings.Contains(lower, key) {
|
||||
matched++
|
||||
}
|
||||
}
|
||||
// dbname= 基本是 PostgreSQL libpq DSN 的强特征;否则至少匹配两个常见键。
|
||||
return strings.Contains(lower, "dbname=") || matched >= 2
|
||||
}
|
||||
|
||||
func looksLikeMySQLDSN(lower string) bool {
|
||||
if strings.Contains(lower, "://") || strings.Contains(lower, " ") {
|
||||
return false
|
||||
}
|
||||
if strings.Contains(lower, "@tcp(") || strings.Contains(lower, "@unix(") || strings.Contains(lower, "@/") {
|
||||
return true
|
||||
}
|
||||
// user:pass@host/db、user@host/db 这类虽然不是推荐格式,但也明显偏 MySQL。
|
||||
return strings.Contains(lower, "@") && strings.Contains(lower, "/")
|
||||
}
|
||||
|
||||
func looksLikeSQLitePath(lower string) bool {
|
||||
path := lower
|
||||
if idx := strings.IndexAny(path, "?"); idx >= 0 {
|
||||
path = path[:idx]
|
||||
}
|
||||
return strings.HasSuffix(path, ".db") || strings.HasSuffix(path, ".sqlite") || strings.HasSuffix(path, ".sqlite3")
|
||||
}
|
||||
|
||||
// stripSharedCache 从 SQLite DSN 中移除 cache=shared 参数,避免共享缓存模式下的
|
||||
// 表级锁(SQLITE_LOCKED "database table is locked")。其它参数保持不变。
|
||||
func stripSharedCache(dsn string) string {
|
||||
if !strings.Contains(dsn, "cache=shared") {
|
||||
return dsn
|
||||
}
|
||||
idx := strings.Index(dsn, "?")
|
||||
if idx < 0 {
|
||||
return dsn
|
||||
}
|
||||
base := dsn[:idx]
|
||||
query := dsn[idx+1:]
|
||||
parts := strings.Split(query, "&")
|
||||
kept := parts[:0]
|
||||
for _, p := range parts {
|
||||
if p == "cache=shared" {
|
||||
continue
|
||||
}
|
||||
kept = append(kept, p)
|
||||
}
|
||||
if len(kept) == 0 {
|
||||
return base
|
||||
}
|
||||
return base + "?" + strings.Join(kept, "&")
|
||||
}
|
||||
@@ -0,0 +1,113 @@
|
||||
package metricstore
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
|
||||
"github.com/komari-monitor/komari/pkg/metric"
|
||||
)
|
||||
|
||||
const defaultBuiltinMetricRetentionDays = 1
|
||||
|
||||
// RetentionSummary is the compatibility view of all persisted metric policies.
|
||||
type RetentionSummary struct {
|
||||
AllPositive bool
|
||||
MaxDays int
|
||||
}
|
||||
|
||||
// GetRetentionSummary aggregates the active store's metric definitions. An
|
||||
// empty definition set is not considered record-enabled.
|
||||
func GetRetentionSummary(ctx context.Context) (RetentionSummary, error) {
|
||||
s := GetStore()
|
||||
if s == nil {
|
||||
return RetentionSummary{}, fmt.Errorf("metric store not initialized")
|
||||
}
|
||||
defs, err := s.ListMetrics(ctx)
|
||||
if err != nil {
|
||||
return RetentionSummary{}, err
|
||||
}
|
||||
return summarizeRetentionDefinitions(defs), nil
|
||||
}
|
||||
|
||||
func summarizeRetentionDefinitions(defs []metric.Definition) RetentionSummary {
|
||||
if len(defs) == 0 {
|
||||
return RetentionSummary{}
|
||||
}
|
||||
summary := RetentionSummary{AllPositive: true}
|
||||
for _, def := range defs {
|
||||
if def.RetentionDays <= 0 {
|
||||
summary.AllPositive = false
|
||||
}
|
||||
if def.RetentionDays > summary.MaxDays {
|
||||
summary.MaxDays = def.RetentionDays
|
||||
}
|
||||
}
|
||||
return summary
|
||||
}
|
||||
|
||||
// createMetricDefinitions creates built-in definitions with explicit policies.
|
||||
func createMetricDefinitions(ctx context.Context, s *metric.Store) error {
|
||||
return createMetricDefinitionsWithDefaultRetention(ctx, s, defaultBuiltinMetricRetentionDays)
|
||||
}
|
||||
|
||||
// EnsureBuiltinMetricDefinitions registers definitions for the server's
|
||||
// built-in report and ping writers before a standalone Store receives points.
|
||||
func EnsureBuiltinMetricDefinitions(ctx context.Context, s *metric.Store) error {
|
||||
return createMetricDefinitions(ctx, s)
|
||||
}
|
||||
|
||||
func createMetricDefinitionsWithDefaultRetention(ctx context.Context, s *metric.Store, defaultRetentionDays int) error {
|
||||
if defaultRetentionDays < defaultBuiltinMetricRetentionDays {
|
||||
defaultRetentionDays = defaultBuiltinMetricRetentionDays
|
||||
}
|
||||
definitions := []metric.Definition{
|
||||
{Name: MetricCPU, Type: metric.TypeGauge, Unit: "%", Description: "CPU usage percentage", RetentionDays: defaultRetentionDays},
|
||||
{Name: MetricGPU, Type: metric.TypeGauge, Unit: "%", Description: "GPU usage percentage", RetentionDays: defaultRetentionDays},
|
||||
{Name: MetricGPUDeviceUsage, Type: metric.TypeGauge, Unit: "%", Description: "Per-device GPU utilization", RetentionDays: defaultRetentionDays},
|
||||
{Name: MetricGPUMem, Type: metric.TypeGauge, Unit: "bytes", Description: "GPU memory used", RetentionDays: defaultRetentionDays},
|
||||
{Name: MetricGPUMemTotal, Type: metric.TypeGauge, Unit: "bytes", Description: "GPU memory total", RetentionDays: defaultRetentionDays},
|
||||
{Name: MetricGPUTemp, Type: metric.TypeGauge, Unit: "°C", Description: "GPU temperature", RetentionDays: defaultRetentionDays},
|
||||
{Name: MetricRAM, Type: metric.TypeGauge, Unit: "bytes", Description: "RAM used", RetentionDays: defaultRetentionDays},
|
||||
{Name: MetricSwap, Type: metric.TypeGauge, Unit: "bytes", Description: "Swap used", RetentionDays: defaultRetentionDays},
|
||||
{Name: MetricLoad, Type: metric.TypeGauge, Unit: "", Description: "System load average", RetentionDays: defaultRetentionDays},
|
||||
{Name: MetricDisk, Type: metric.TypeGauge, Unit: "bytes", Description: "Disk used", RetentionDays: defaultRetentionDays},
|
||||
{Name: MetricNetIn, Type: metric.TypeGauge, Unit: "bytes/s", Description: "Network in rate", RetentionDays: defaultRetentionDays},
|
||||
{Name: MetricNetOut, Type: metric.TypeGauge, Unit: "bytes/s", Description: "Network out rate", RetentionDays: defaultRetentionDays},
|
||||
{Name: MetricNetTotalUp, Type: metric.TypeCounter, Unit: "bytes", Description: "Network total upload", RetentionDays: defaultRetentionDays},
|
||||
{Name: MetricNetTotalDown, Type: metric.TypeCounter, Unit: "bytes", Description: "Network total download", RetentionDays: defaultRetentionDays},
|
||||
{Name: MetricTrafficUp, Type: metric.TypeGauge, Unit: "bytes", Description: "Traffic upload delta", RetentionDays: defaultRetentionDays},
|
||||
{Name: MetricTrafficDown, Type: metric.TypeGauge, Unit: "bytes", Description: "Traffic download delta", RetentionDays: defaultRetentionDays},
|
||||
{Name: MetricProcess, Type: metric.TypeGauge, Unit: "count", Description: "Process count", RetentionDays: defaultRetentionDays},
|
||||
{Name: MetricConnections, Type: metric.TypeGauge, Unit: "count", Description: "TCP connections", RetentionDays: defaultRetentionDays},
|
||||
{Name: MetricConnectionsUDP, Type: metric.TypeGauge, Unit: "count", Description: "UDP connections", RetentionDays: defaultRetentionDays},
|
||||
{Name: MetricPingLatency, Type: metric.TypeGauge, Unit: "ms", Description: "Ping latency", RetentionDays: defaultRetentionDays},
|
||||
{Name: MetricPingLoss, Type: metric.TypeGauge, Unit: "ratio", Description: "Ping packet loss indicator", RetentionDays: defaultRetentionDays},
|
||||
}
|
||||
|
||||
for _, def := range definitions {
|
||||
existing, err := s.GetMetric(ctx, def.Name)
|
||||
if err != nil && !errors.Is(err, metric.ErrNotFound) {
|
||||
return fmt.Errorf("failed to get metric %s: %w", def.Name, err)
|
||||
}
|
||||
if err == nil {
|
||||
if existing.RetentionDays == 0 {
|
||||
if _, err := s.SetMetricRetention(ctx, def.Name, 0); err != nil {
|
||||
return fmt.Errorf("failed to preserve disabled metric %s: %w", def.Name, err)
|
||||
}
|
||||
continue
|
||||
}
|
||||
def.RetentionDays = existing.RetentionDays
|
||||
}
|
||||
if err := s.UpsertMetric(ctx, def); err != nil {
|
||||
return fmt.Errorf("failed to create metric %s: %w", def.Name, err)
|
||||
}
|
||||
}
|
||||
for _, name := range obsoleteBuiltinMetricNames {
|
||||
if err := s.DeleteMetric(ctx, name); err != nil {
|
||||
return fmt.Errorf("failed to remove obsolete metric %s: %w", name, err)
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,104 @@
|
||||
package metricstore
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
logger "github.com/komari-monitor/komari/utils/log"
|
||||
|
||||
"github.com/komari-monitor/komari/pkg/metric"
|
||||
)
|
||||
|
||||
// farFuture 返回一个足够远的未来时间,用于以 DeleteBefore 语义清空某指标的全部数据。
|
||||
func farFuture() time.Time {
|
||||
return time.Now().UTC().Add(24 * 365 * time.Hour)
|
||||
}
|
||||
|
||||
// DeleteAllRecords 删除所有负载/系统类记录(保留指标定义,不含 ping)。
|
||||
func DeleteAllRecords(ctx context.Context) error {
|
||||
s := GetStore()
|
||||
if s == nil {
|
||||
return fmt.Errorf("metric store not enabled")
|
||||
}
|
||||
|
||||
for _, metricName := range recordMetricNames {
|
||||
if _, err := s.DeleteBefore(ctx, metricName, farFuture()); err != nil {
|
||||
logger.Errorf("metricstore", "Failed to delete metric %s: %v", metricName, err)
|
||||
}
|
||||
}
|
||||
clearReportTrafficStates()
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// DeleteAllPingRecords 删除全部 ping 记录(保留指标定义)。
|
||||
func DeleteAllPingRecords(ctx context.Context) error {
|
||||
s := GetStore()
|
||||
if s == nil {
|
||||
return fmt.Errorf("metric store not enabled")
|
||||
}
|
||||
for _, metricName := range pingMetricNames {
|
||||
if _, err := s.DeleteBefore(ctx, metricName, farFuture()); err != nil {
|
||||
return fmt.Errorf("failed to delete ping records: %w", err)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// DeletePingRecordsByTask 删除指定任务(task_id)的全部 ping 记录。
|
||||
func DeletePingRecordsByTask(ctx context.Context, taskIDs []uint) error {
|
||||
s := GetStore()
|
||||
if s == nil {
|
||||
return fmt.Errorf("metric store not enabled")
|
||||
}
|
||||
for _, id := range taskIDs {
|
||||
for _, metricName := range pingMetricNames {
|
||||
if _, err := s.DeleteSeries(ctx, metric.Query{
|
||||
MetricName: metricName,
|
||||
Tags: map[string]string{"task_id": fmt.Sprintf("%d", id)},
|
||||
}); err != nil {
|
||||
return fmt.Errorf("failed to delete ping records for task %d: %w", id, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// DeleteEntity 删除指定 agent 在所有指标下的历史数据。
|
||||
func DeleteEntity(ctx context.Context, entityID string) error {
|
||||
s := GetStore()
|
||||
if s == nil {
|
||||
return fmt.Errorf("metric store not enabled")
|
||||
}
|
||||
if _, err := s.DeleteEntity(ctx, entityID); err != nil {
|
||||
return fmt.Errorf("failed to delete metric records for entity %s: %w", entityID, err)
|
||||
}
|
||||
deleteReportTrafficState(entityID)
|
||||
return nil
|
||||
}
|
||||
|
||||
// DeleteEntityAsync clears one agent's metric history without delaying the
|
||||
// client deletion response.
|
||||
func DeleteEntityAsync(entityID string) {
|
||||
go func() {
|
||||
if err := DeleteEntity(context.Background(), entityID); err != nil {
|
||||
logger.Errorf("metricstore", "Failed to delete metric records for entity %s: %v", entityID, err)
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
// DeleteMetricDataAsync clears disabled metric history without delaying an
|
||||
// admin retention-policy update response.
|
||||
func DeleteMetricDataAsync(metricName string) {
|
||||
go func() {
|
||||
s := GetStore()
|
||||
if s == nil {
|
||||
logger.Errorf("metricstore", "Failed to delete disabled metric %s: metric store not enabled", metricName)
|
||||
return
|
||||
}
|
||||
if _, err := s.DeleteMetricDataIfDisabled(context.Background(), metricName); err != nil {
|
||||
logger.Errorf("metricstore", "Failed to delete disabled metric %s: %v", metricName, err)
|
||||
}
|
||||
}()
|
||||
}
|
||||
@@ -0,0 +1,313 @@
|
||||
package metricstore
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"sort"
|
||||
"time"
|
||||
|
||||
"github.com/komari-monitor/komari/database/models"
|
||||
"github.com/komari-monitor/komari/pkg/metric"
|
||||
)
|
||||
|
||||
// GetRecordsByClientAndTime 从 metric store 查询记录并重构为 models.Record
|
||||
func GetRecordsByClientAndTime(ctx context.Context, clientUUID string, start, end time.Time) ([]models.Record, error) {
|
||||
s := GetStore()
|
||||
if s == nil {
|
||||
return nil, fmt.Errorf("metric store not enabled")
|
||||
}
|
||||
|
||||
return getRecordsByClientAndTimeFromSeries(ctx, s, clientUUID, start, end)
|
||||
}
|
||||
|
||||
// GetRecordMetricMaxByClientAndTime 查询单项监控指标在各时间桶内的最大值。
|
||||
func GetRecordMetricMaxByClientAndTime(ctx context.Context, clientUUID, recordMetric string, start, end time.Time) ([]models.Record, error) {
|
||||
s := GetStore()
|
||||
if s == nil {
|
||||
return nil, fmt.Errorf("metric store not enabled")
|
||||
}
|
||||
|
||||
return getRecordMetricMaxByClientAndTimeFromSeries(ctx, s, clientUUID, recordMetric, start, end)
|
||||
}
|
||||
|
||||
// GetRecordsByTime 从 metric store 查询所有客户端在时间范围内的记录
|
||||
func GetRecordsByTime(ctx context.Context, start, end time.Time) ([]models.Record, error) {
|
||||
s := GetStore()
|
||||
if s == nil {
|
||||
return nil, fmt.Errorf("metric store not enabled")
|
||||
}
|
||||
|
||||
interval := recordSeriesInterval(s, start, end, time.Now().UTC())
|
||||
entityIDs, err := listRecordEntityIDs(ctx, s, start, end, interval)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var records []models.Record
|
||||
for _, entityID := range entityIDs {
|
||||
items, err := getRecordsByClientAndTimeFromSeries(ctx, s, entityID, start, end)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
records = append(records, items...)
|
||||
}
|
||||
sortRecords(records)
|
||||
return records, nil
|
||||
}
|
||||
|
||||
type recordSeriesKey struct {
|
||||
client string
|
||||
ts int64
|
||||
}
|
||||
|
||||
func getRecordsByClientAndTimeFromSeries(ctx context.Context, s *metric.Store, clientUUID string, start, end time.Time) ([]models.Record, error) {
|
||||
now := time.Now().UTC()
|
||||
interval := recordSeriesInterval(s, start, end, now)
|
||||
recordMap := make(map[recordSeriesKey]*models.Record)
|
||||
|
||||
for _, metricName := range loadRecordMetricNames {
|
||||
points, err := s.Series(ctx, metric.AggregateQuery{
|
||||
Query: metric.Query{
|
||||
MetricName: metricName,
|
||||
EntityID: clientUUID,
|
||||
Start: start,
|
||||
End: end,
|
||||
Order: metric.OrderAsc,
|
||||
},
|
||||
Aggregation: recordMetricAggregation(metricName),
|
||||
Interval: interval,
|
||||
}, now)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to query metric %s: %w", metricName, err)
|
||||
}
|
||||
for _, point := range points {
|
||||
entityID := point.EntityID
|
||||
if entityID == "" {
|
||||
entityID = clientUUID
|
||||
}
|
||||
key := recordSeriesKey{client: entityID, ts: point.Bucket.Unix()}
|
||||
if recordMap[key] == nil {
|
||||
recordMap[key] = &models.Record{
|
||||
Client: entityID,
|
||||
Time: point.Bucket.UTC(),
|
||||
}
|
||||
}
|
||||
applyRecordMetricValue(recordMap[key], metricName, point.Value)
|
||||
}
|
||||
}
|
||||
|
||||
records := make([]models.Record, 0, len(recordMap))
|
||||
for _, rec := range recordMap {
|
||||
records = append(records, *rec)
|
||||
}
|
||||
sortRecords(records)
|
||||
return records, nil
|
||||
}
|
||||
|
||||
func getRecordMetricMaxByClientAndTimeFromSeries(ctx context.Context, s *metric.Store, clientUUID, recordMetric string, start, end time.Time) ([]models.Record, error) {
|
||||
metricName, ok := metricNameForRecordField(recordMetric)
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("unsupported record metric %q", recordMetric)
|
||||
}
|
||||
|
||||
now := time.Now().UTC()
|
||||
points, err := s.Series(ctx, metric.AggregateQuery{
|
||||
Query: metric.Query{
|
||||
MetricName: metricName,
|
||||
EntityID: clientUUID,
|
||||
Start: start,
|
||||
End: end,
|
||||
Order: metric.OrderAsc,
|
||||
},
|
||||
Aggregation: metric.AggMax,
|
||||
Interval: recordSeriesInterval(s, start, end, now),
|
||||
}, now)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to query metric %s: %w", metricName, err)
|
||||
}
|
||||
|
||||
records := make([]models.Record, 0, len(points))
|
||||
for _, point := range points {
|
||||
entityID := point.EntityID
|
||||
if entityID == "" {
|
||||
entityID = clientUUID
|
||||
}
|
||||
record := models.Record{Client: entityID, Time: point.Bucket.UTC()}
|
||||
applyRecordMetricValue(&record, metricName, point.Value)
|
||||
records = append(records, record)
|
||||
}
|
||||
return records, nil
|
||||
}
|
||||
|
||||
func recordMetricAggregation(metricName string) metric.Aggregation {
|
||||
switch metricName {
|
||||
case MetricTrafficUp, MetricTrafficDown:
|
||||
return metric.AggSum
|
||||
case MetricNetTotalUp, MetricNetTotalDown:
|
||||
return metric.AggLast
|
||||
default:
|
||||
return metric.AggAvg
|
||||
}
|
||||
}
|
||||
|
||||
func recordSeriesInterval(s *metric.Store, start, end, now time.Time) time.Duration {
|
||||
interval := recordDownsampleInterval(end.Sub(start), 500)
|
||||
return s.CompatibleSeriesInterval(start, now, interval)
|
||||
}
|
||||
|
||||
func recordDownsampleInterval(rangeDuration time.Duration, maxPoints int) time.Duration {
|
||||
if maxPoints <= 0 {
|
||||
maxPoints = 500
|
||||
}
|
||||
nanos := rangeDuration.Nanoseconds()
|
||||
if nanos <= 0 {
|
||||
return time.Second
|
||||
}
|
||||
interval := time.Duration((nanos + int64(maxPoints) - 1) / int64(maxPoints))
|
||||
if interval < time.Second {
|
||||
return time.Second
|
||||
}
|
||||
return metric.FloorStandardInterval(interval)
|
||||
}
|
||||
|
||||
func listRecordEntityIDs(ctx context.Context, s *metric.Store, start, end time.Time, interval time.Duration) ([]string, error) {
|
||||
seen := make(map[string]struct{})
|
||||
for _, metricName := range loadRecordMetricNames {
|
||||
ids, err := s.EntityIDs(ctx, metric.Query{
|
||||
MetricName: metricName,
|
||||
Start: start.Add(-interval),
|
||||
End: end,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for _, id := range ids {
|
||||
seen[id] = struct{}{}
|
||||
}
|
||||
}
|
||||
out := make([]string, 0, len(seen))
|
||||
for id := range seen {
|
||||
out = append(out, id)
|
||||
}
|
||||
sort.Strings(out)
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func applyRecordMetricValue(rec *models.Record, metricName string, value float64) {
|
||||
switch metricName {
|
||||
case MetricCPU:
|
||||
rec.Cpu = float32(value)
|
||||
case MetricGPU:
|
||||
rec.Gpu = float32(value)
|
||||
case MetricRAM:
|
||||
rec.Ram = int64(value)
|
||||
case MetricSwap:
|
||||
rec.Swap = int64(value)
|
||||
case MetricLoad:
|
||||
rec.Load = float32(value)
|
||||
case MetricDisk:
|
||||
rec.Disk = int64(value)
|
||||
case MetricNetIn:
|
||||
rec.NetIn = int64(value)
|
||||
case MetricNetOut:
|
||||
rec.NetOut = int64(value)
|
||||
case MetricNetTotalUp:
|
||||
rec.NetTotalUp = int64(value)
|
||||
case MetricNetTotalDown:
|
||||
rec.NetTotalDown = int64(value)
|
||||
case MetricTrafficUp:
|
||||
rec.TrafficUp = int64(value)
|
||||
case MetricTrafficDown:
|
||||
rec.TrafficDown = int64(value)
|
||||
case MetricProcess:
|
||||
rec.Process = int(value)
|
||||
case MetricConnections:
|
||||
rec.Connections = int(value)
|
||||
case MetricConnectionsUDP:
|
||||
rec.ConnectionsUdp = int(value)
|
||||
}
|
||||
}
|
||||
|
||||
func sortRecords(records []models.Record) {
|
||||
sort.Slice(records, func(i, j int) bool {
|
||||
if records[i].Client != records[j].Client {
|
||||
return records[i].Client < records[j].Client
|
||||
}
|
||||
return records[i].Time.Before(records[j].Time)
|
||||
})
|
||||
}
|
||||
|
||||
// GetGPURecordsByClientAndTime 从 metric store 查询 GPU 记录
|
||||
func GetGPURecordsByClientAndTime(ctx context.Context, clientUUID string, start, end time.Time) ([]models.GPURecord, error) {
|
||||
s := GetStore()
|
||||
if s == nil {
|
||||
return nil, fmt.Errorf("metric store not enabled")
|
||||
}
|
||||
|
||||
// 查询 GPU 相关指标(每设备利用率使用独立指标 gpu.device.usage)
|
||||
gpuMetrics := []string{MetricGPUDeviceUsage, MetricGPUMem, MetricGPUMemTotal, MetricGPUTemp}
|
||||
|
||||
// 按设备索引和时间组织数据
|
||||
type gpuKey struct {
|
||||
deviceIndex int
|
||||
timestamp int64
|
||||
}
|
||||
recordMap := make(map[gpuKey]*models.GPURecord)
|
||||
|
||||
for _, metricName := range gpuMetrics {
|
||||
points, err := s.Query(ctx, metric.Query{
|
||||
MetricName: metricName,
|
||||
EntityID: clientUUID,
|
||||
Start: start,
|
||||
End: end,
|
||||
Order: metric.OrderAsc,
|
||||
})
|
||||
if err != nil {
|
||||
continue // GPU 数据可能不存在
|
||||
}
|
||||
|
||||
for _, p := range points {
|
||||
deviceIndex := 0
|
||||
deviceName := ""
|
||||
if idx, ok := p.Tags["device_index"]; ok {
|
||||
fmt.Sscanf(idx, "%d", &deviceIndex)
|
||||
}
|
||||
if name, ok := p.Tags["device_name"]; ok {
|
||||
deviceName = name
|
||||
}
|
||||
|
||||
key := gpuKey{deviceIndex: deviceIndex, timestamp: p.Timestamp.Unix()}
|
||||
if recordMap[key] == nil {
|
||||
recordMap[key] = &models.GPURecord{
|
||||
Client: clientUUID,
|
||||
Time: p.Timestamp.UTC(),
|
||||
DeviceIndex: deviceIndex,
|
||||
DeviceName: deviceName,
|
||||
}
|
||||
}
|
||||
rec := recordMap[key]
|
||||
|
||||
switch metricName {
|
||||
case MetricGPUDeviceUsage:
|
||||
rec.Utilization = float32(p.Value)
|
||||
case MetricGPUMem:
|
||||
rec.MemUsed = int64(p.Value)
|
||||
case MetricGPUMemTotal:
|
||||
rec.MemTotal = int64(p.Value)
|
||||
case MetricGPUTemp:
|
||||
rec.Temperature = int(p.Value)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 转换为切片
|
||||
records := make([]models.GPURecord, 0, len(recordMap))
|
||||
for _, rec := range recordMap {
|
||||
records = append(records, *rec)
|
||||
}
|
||||
|
||||
return records, nil
|
||||
}
|
||||
|
||||
// GetPingRecords 从 metric store 查询兼容旧接口的 ping 记录。
|
||||
//
|
||||
// 旧接口过去直接读取 ping_records。这里使用与 queryMetrics 相同的 Series
|
||||
@@ -37,10 +37,10 @@ type MaintenanceResult struct {
|
||||
// InspectStorage reads physical storage information while preventing a store
|
||||
// reload from closing the active connection underneath the query.
|
||||
func InspectStorage(ctx context.Context) (StorageInfo, error) {
|
||||
if err := storeOperations.Acquire(ctx); err != nil {
|
||||
if err := storeOperations.AcquireShared(ctx); err != nil {
|
||||
return StorageInfo{}, fmt.Errorf("wait for metric store operations before inspection: %w", err)
|
||||
}
|
||||
defer storeOperations.Release()
|
||||
defer storeOperations.ReleaseShared()
|
||||
|
||||
storeMu.RLock()
|
||||
activeStore := store
|
||||
@@ -59,21 +59,15 @@ func InspectStorage(ctx context.Context) (StorageInfo, error) {
|
||||
}
|
||||
|
||||
// ReclaimSpace performs the driver-specific physical maintenance operation.
|
||||
// It shares the exclusive operation lock with Compact so retention compaction
|
||||
// and a table/file rewrite can never run against the metric store concurrently.
|
||||
// It takes the exclusive operation lock, so a table/file rewrite cannot run
|
||||
// concurrently with report writes or compaction.
|
||||
func ReclaimSpace(ctx context.Context) (MaintenanceResult, error) {
|
||||
if !storeOperations.TryAcquire() {
|
||||
storeMu.RLock()
|
||||
defer storeMu.RUnlock()
|
||||
if store == nil {
|
||||
return MaintenanceResult{}, ErrStoreNotInitialized
|
||||
}
|
||||
return MaintenanceResult{
|
||||
Driver: store.Driver(),
|
||||
Action: store.MaintenanceAction(),
|
||||
BeforeSizeError: ErrStoreBusy,
|
||||
AfterSizeError: ErrStoreBusy,
|
||||
}, ErrStoreBusy
|
||||
// Reclamation is intentionally non-cancellable. It waits for all active
|
||||
// operations, then keeps the store gate until digest conversion, VACUUM,
|
||||
// and the final checkpoint have completed.
|
||||
_ = ctx
|
||||
if err := storeOperations.Acquire(context.Background()); err != nil {
|
||||
return MaintenanceResult{}, err
|
||||
}
|
||||
defer storeOperations.Release()
|
||||
|
||||
@@ -88,8 +82,9 @@ func ReclaimSpace(ctx context.Context) (MaintenanceResult, error) {
|
||||
Driver: activeStore.Driver(),
|
||||
Action: activeStore.MaintenanceAction(),
|
||||
}
|
||||
result.Before, result.BeforeSizeError = activeStore.StorageSize(ctx)
|
||||
maintenanceErr := activeStore.ReclaimSpace(ctx)
|
||||
result.After, result.AfterSizeError = activeStore.StorageSize(ctx)
|
||||
maintenanceCtx := context.Background()
|
||||
result.Before, result.BeforeSizeError = activeStore.StorageSize(maintenanceCtx)
|
||||
maintenanceErr := activeStore.ReclaimSpace(maintenanceCtx)
|
||||
result.After, result.AfterSizeError = activeStore.StorageSize(maintenanceCtx)
|
||||
return result, maintenanceErr
|
||||
}
|
||||
+41
-12
@@ -40,30 +40,59 @@ func TestInspectAndReclaimStorage(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestReclaimSpaceReportsBusyStore(t *testing.T) {
|
||||
func TestReclaimSpaceWaitsForStore(t *testing.T) {
|
||||
s, err := metric.Open(context.Background(), metric.SQLite(":memory:"))
|
||||
if err != nil {
|
||||
t.Fatalf("open metric store: %v", err)
|
||||
}
|
||||
installTestStore(t, s)
|
||||
|
||||
if !storeOperations.TryAcquire() {
|
||||
t.Fatal("acquire store operation gate")
|
||||
if err := storeOperations.AcquireShared(context.Background()); err != nil {
|
||||
t.Fatalf("acquire shared store operation gate: %v", err)
|
||||
}
|
||||
defer storeOperations.Release()
|
||||
|
||||
result, err := ReclaimSpace(context.Background())
|
||||
if !errors.Is(err, ErrStoreBusy) {
|
||||
t.Fatalf("reclaim error = %v, want %v", err, ErrStoreBusy)
|
||||
done := make(chan struct{})
|
||||
var result MaintenanceResult
|
||||
var reclaimErr error
|
||||
canceledCtx, cancel := context.WithCancel(context.Background())
|
||||
cancel()
|
||||
go func() {
|
||||
result, reclaimErr = ReclaimSpace(canceledCtx)
|
||||
close(done)
|
||||
}()
|
||||
select {
|
||||
case <-done:
|
||||
t.Fatal("reclaim returned while another operation held the gate")
|
||||
case <-time.After(25 * time.Millisecond):
|
||||
}
|
||||
storeOperations.ReleaseShared()
|
||||
select {
|
||||
case <-done:
|
||||
case <-time.After(time.Second):
|
||||
t.Fatal("reclaim did not resume after the gate was released")
|
||||
}
|
||||
if reclaimErr != nil {
|
||||
t.Fatalf("reclaim error: %v", reclaimErr)
|
||||
}
|
||||
if result.Driver != metric.DriverSQLite || result.Action != metric.MaintenanceVacuum {
|
||||
t.Fatalf("busy result lost store metadata: %#v", result)
|
||||
t.Fatalf("reclaim result lost store metadata: %#v", result)
|
||||
}
|
||||
if !errors.Is(result.BeforeSizeError, ErrStoreBusy) || !errors.Is(result.AfterSizeError, ErrStoreBusy) {
|
||||
t.Fatalf("busy result should mark both measurements unavailable: %#v", result)
|
||||
if result.BeforeSizeError != nil || result.AfterSizeError != nil {
|
||||
t.Fatalf("unexpected size errors: before=%v after=%v", result.BeforeSizeError, result.AfterSizeError)
|
||||
}
|
||||
compactCtx, cancel := context.WithTimeout(context.Background(), time.Second)
|
||||
defer cancel()
|
||||
if _, err := Compact(compactCtx, time.Now()); err != nil {
|
||||
t.Fatalf("compact should not wait for report writes: %v", err)
|
||||
}
|
||||
if !compactOperations.TryAcquire() {
|
||||
t.Fatal("acquire compact operation gate")
|
||||
}
|
||||
defer compactOperations.Release()
|
||||
if _, err := Compact(context.Background(), time.Now()); !errors.Is(err, ErrCompactInProgress) {
|
||||
t.Fatalf("compact error = %v, want %v", err, ErrCompactInProgress)
|
||||
t.Fatalf("overlapping compact error = %v, want %v", err, ErrCompactInProgress)
|
||||
}
|
||||
if _, err := CleanupExpired(context.Background(), time.Now()); !errors.Is(err, ErrCompactInProgress) {
|
||||
t.Fatalf("overlapping retention cleanup error = %v, want %v", err, ErrCompactInProgress)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -8,13 +8,9 @@ const (
|
||||
MetricGPUMemTotal = "gpu.memory.total"
|
||||
MetricGPUTemp = "gpu.temperature"
|
||||
MetricRAM = "memory.used"
|
||||
MetricRAMTotal = "memory.total"
|
||||
MetricSwap = "swap.used"
|
||||
MetricSwapTotal = "swap.total"
|
||||
MetricLoad = "load.average"
|
||||
MetricTemp = "temperature"
|
||||
MetricDisk = "disk.used"
|
||||
MetricDiskTotal = "disk.total"
|
||||
MetricNetIn = "net.in.rate"
|
||||
MetricNetOut = "net.out.rate"
|
||||
MetricNetTotalUp = "net.total.up"
|
||||
@@ -31,12 +27,18 @@ const (
|
||||
// loadRecordMetricNames are the entity-level metrics used to reconstruct the
|
||||
// legacy Record response shape.
|
||||
var loadRecordMetricNames = []string{
|
||||
MetricCPU, MetricGPU, MetricRAM, MetricRAMTotal, MetricSwap, MetricSwapTotal,
|
||||
MetricLoad, MetricTemp, MetricDisk, MetricDiskTotal, MetricNetIn, MetricNetOut,
|
||||
MetricCPU, MetricGPU, MetricRAM, MetricSwap, MetricLoad, MetricDisk, MetricNetIn, MetricNetOut,
|
||||
MetricNetTotalUp, MetricNetTotalDown, MetricTrafficUp, MetricTrafficDown,
|
||||
MetricProcess, MetricConnections, MetricConnectionsUDP,
|
||||
}
|
||||
|
||||
var obsoleteBuiltinMetricNames = []string{
|
||||
"memory.total",
|
||||
"swap.total",
|
||||
"temperature",
|
||||
"disk.total",
|
||||
}
|
||||
|
||||
// gpuDeviceRecordMetricNames are stored separately from the entity-level GPU
|
||||
// average and are included when deleting all system records.
|
||||
var gpuDeviceRecordMetricNames = []string{
|
||||
@@ -48,6 +50,43 @@ var recordMetricNames = joinMetricNames(loadRecordMetricNames, gpuDeviceRecordMe
|
||||
// Ping has an independent retention and cleanup boundary.
|
||||
var pingMetricNames = []string{MetricPingLatency, MetricPingLoss}
|
||||
|
||||
func metricNameForRecordField(name string) (string, bool) {
|
||||
switch name {
|
||||
case "cpu":
|
||||
return MetricCPU, true
|
||||
case "gpu":
|
||||
return MetricGPU, true
|
||||
case "ram":
|
||||
return MetricRAM, true
|
||||
case "swap":
|
||||
return MetricSwap, true
|
||||
case "load":
|
||||
return MetricLoad, true
|
||||
case "disk":
|
||||
return MetricDisk, true
|
||||
case "net_in", "netin":
|
||||
return MetricNetIn, true
|
||||
case "net_out", "netout":
|
||||
return MetricNetOut, true
|
||||
case "net_total_up":
|
||||
return MetricNetTotalUp, true
|
||||
case "net_total_down":
|
||||
return MetricNetTotalDown, true
|
||||
case "traffic_up":
|
||||
return MetricTrafficUp, true
|
||||
case "traffic_down":
|
||||
return MetricTrafficDown, true
|
||||
case "process":
|
||||
return MetricProcess, true
|
||||
case "connections":
|
||||
return MetricConnections, true
|
||||
case "connections_udp":
|
||||
return MetricConnectionsUDP, true
|
||||
default:
|
||||
return "", false
|
||||
}
|
||||
}
|
||||
|
||||
func joinMetricNames(groups ...[]string) []string {
|
||||
total := 0
|
||||
for _, group := range groups {
|
||||
+233
-68
@@ -2,6 +2,9 @@ package metricstore
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"errors"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
@@ -18,12 +21,12 @@ func TestDefaultRollupPolicy(t *testing.T) {
|
||||
if policy.RawRetention != DefaultRollupRawRetention {
|
||||
t.Fatalf("raw retention = %s, want %s", policy.RawRetention, DefaultRollupRawRetention)
|
||||
}
|
||||
if len(policy.Tiers) != 3 {
|
||||
t.Fatalf("expected 3 rollup tiers, got %d", len(policy.Tiers))
|
||||
if len(policy.Tiers) != 4 {
|
||||
t.Fatalf("expected 4 rollup tiers, got %d", len(policy.Tiers))
|
||||
}
|
||||
|
||||
wantIntervals := []time.Duration{time.Minute, 5 * time.Minute, time.Hour}
|
||||
wantRetentions := []time.Duration{48 * time.Hour, 14 * 24 * time.Hour, 14 * 24 * time.Hour}
|
||||
wantIntervals := []time.Duration{time.Minute, 5 * time.Minute, time.Hour, 24 * time.Hour}
|
||||
wantRetentions := []time.Duration{600 * time.Minute, 600 * 5 * time.Minute, 600 * time.Hour, 100 * 365 * 24 * time.Hour}
|
||||
for i := range wantIntervals {
|
||||
if policy.Tiers[i].Interval != wantIntervals[i] {
|
||||
t.Fatalf("tier %d interval = %s, want %s", i, policy.Tiers[i].Interval, wantIntervals[i])
|
||||
@@ -36,10 +39,9 @@ func TestDefaultRollupPolicy(t *testing.T) {
|
||||
|
||||
func TestBuildMetricConfigEnablesDefaultRollupPolicy(t *testing.T) {
|
||||
cfg, err := buildMetricConfig(&MetricStoreConfig{
|
||||
Driver: "sqlite",
|
||||
DSN: ":memory:",
|
||||
DownsamplingEnabled: true,
|
||||
TablePrefix: "metric_",
|
||||
Driver: "sqlite",
|
||||
DSN: ":memory:",
|
||||
TablePrefix: "metric_",
|
||||
}, false)
|
||||
if err != nil {
|
||||
t.Fatalf("build metric config: %v", err)
|
||||
@@ -50,35 +52,122 @@ func TestBuildMetricConfigEnablesDefaultRollupPolicy(t *testing.T) {
|
||||
if cfg.RollupPolicy.RawRetention != DefaultRollupRawRetention {
|
||||
t.Fatalf("raw retention = %s, want %s", cfg.RollupPolicy.RawRetention, DefaultRollupRawRetention)
|
||||
}
|
||||
if cfg.SQLite.ReadPoolSize != 2 {
|
||||
t.Fatalf("metric store read pool = %d, want fixed size 2", cfg.SQLite.ReadPoolSize)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildMetricConfigLeavesFinalRetentionToMetricDefinition(t *testing.T) {
|
||||
cfg, err := buildMetricConfig(&MetricStoreConfig{
|
||||
Driver: "sqlite",
|
||||
DSN: ":memory:",
|
||||
DownsamplingEnabled: true,
|
||||
Driver: "sqlite",
|
||||
DSN: ":memory:",
|
||||
}, false)
|
||||
if err != nil {
|
||||
t.Fatalf("build metric config: %v", err)
|
||||
}
|
||||
wantRollupRetention := 14 * 24 * time.Hour
|
||||
wantRollupRetention := 100 * 365 * 24 * time.Hour
|
||||
lastTier := cfg.RollupPolicy.Tiers[len(cfg.RollupPolicy.Tiers)-1]
|
||||
if lastTier.Retention != wantRollupRetention {
|
||||
t.Fatalf("rollup retention = %s, want %s", lastTier.Retention, wantRollupRetention)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildMetricConfigCanDisableDownsampling(t *testing.T) {
|
||||
func TestBuildMetricConfigUsesCustomRollupRetention(t *testing.T) {
|
||||
cfg, err := buildMetricConfig(&MetricStoreConfig{
|
||||
Driver: "sqlite",
|
||||
DSN: ":memory:",
|
||||
DownsamplingEnabled: false,
|
||||
Driver: "sqlite",
|
||||
DSN: ":memory:",
|
||||
RollupMinuteRetentionMinutes: 30,
|
||||
RollupFiveMinuteRetentionMinutes: 150,
|
||||
RollupHourRetentionHours: 300,
|
||||
}, false)
|
||||
if err != nil {
|
||||
t.Fatalf("build metric config: %v", err)
|
||||
}
|
||||
if cfg.RollupPolicy.Enabled() {
|
||||
t.Fatal("expected rollup policy to be disabled")
|
||||
|
||||
want := []time.Duration{30 * time.Minute, 150 * time.Minute, 300 * time.Hour}
|
||||
if len(cfg.RollupPolicy.Tiers) != 4 {
|
||||
t.Fatalf("tier count = %d, want 4", len(cfg.RollupPolicy.Tiers))
|
||||
}
|
||||
for i, retention := range want {
|
||||
if cfg.RollupPolicy.Tiers[i].Retention != retention {
|
||||
t.Fatalf("tier %d retention = %s, want %s", i, cfg.RollupPolicy.Tiers[i].Retention, retention)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildMetricConfigRejectsInvalidRollupRetention(t *testing.T) {
|
||||
tests := []MetricStoreConfig{
|
||||
{
|
||||
Driver: "sqlite",
|
||||
DSN: ":memory:",
|
||||
RollupMinuteRetentionMinutes: -1,
|
||||
},
|
||||
{
|
||||
Driver: "sqlite",
|
||||
DSN: ":memory:",
|
||||
RollupMinuteRetentionMinutes: 120,
|
||||
RollupFiveMinuteRetentionMinutes: 60,
|
||||
RollupHourRetentionHours: 600,
|
||||
},
|
||||
{
|
||||
Driver: "sqlite",
|
||||
DSN: ":memory:",
|
||||
RollupMinuteRetentionMinutes: 30,
|
||||
RollupFiveMinuteRetentionMinutes: 150,
|
||||
RollupHourRetentionHours: 1,
|
||||
},
|
||||
}
|
||||
for i, cfg := range tests {
|
||||
if _, err := buildMetricConfig(&cfg, false); err == nil {
|
||||
t.Fatalf("case %d: expected invalid rollup retention error", i)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildMetricConfigDefaultsOmittedRollupRetention(t *testing.T) {
|
||||
cfg, err := buildMetricConfig(&MetricStoreConfig{Driver: "sqlite", DSN: ":memory:"}, false)
|
||||
if err != nil {
|
||||
t.Fatalf("build metric config: %v", err)
|
||||
}
|
||||
if got, want := cfg.RollupPolicy.Tiers[0].Retention, 600*time.Minute; got != want {
|
||||
t.Fatalf("minute retention = %s, want %s", got, want)
|
||||
}
|
||||
if got, want := cfg.RollupPolicy.Tiers[1].Retention, 3000*time.Minute; got != want {
|
||||
t.Fatalf("five-minute retention = %s, want %s", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestConfigFromFingerprintPreservesRollupRetention(t *testing.T) {
|
||||
base := &MetricStoreConfig{
|
||||
TablePrefix: "metrics_",
|
||||
MaxOpenConns: 11,
|
||||
MaxIdleConns: 4,
|
||||
RollupMinuteRetentionMinutes: 30,
|
||||
RollupFiveMinuteRetentionMinutes: 150,
|
||||
RollupHourRetentionHours: 300,
|
||||
}
|
||||
|
||||
got, err := configFromFingerprint("mysql|user:password@tcp(host:3306)/metrics", base)
|
||||
if err != nil {
|
||||
t.Fatalf("config from fingerprint: %v", err)
|
||||
}
|
||||
if got.RollupMinuteRetentionMinutes != base.RollupMinuteRetentionMinutes ||
|
||||
got.RollupFiveMinuteRetentionMinutes != base.RollupFiveMinuteRetentionMinutes ||
|
||||
got.RollupHourRetentionHours != base.RollupHourRetentionHours {
|
||||
t.Fatalf("rollup retention was not preserved: %#v", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildMetricConfigAlwaysEnablesDownsampling(t *testing.T) {
|
||||
cfg, err := buildMetricConfig(&MetricStoreConfig{
|
||||
Driver: "sqlite",
|
||||
DSN: ":memory:",
|
||||
}, false)
|
||||
if err != nil {
|
||||
t.Fatalf("build metric config: %v", err)
|
||||
}
|
||||
if !cfg.RollupPolicy.Enabled() {
|
||||
t.Fatal("expected rollup policy to be enabled")
|
||||
}
|
||||
}
|
||||
|
||||
@@ -152,8 +241,8 @@ func TestCreateMetricDefinitionsUsesExplicitRetentionAndPreservesOverrides(t *te
|
||||
if err != nil {
|
||||
t.Fatalf("list definitions: %v", err)
|
||||
}
|
||||
if len(defs) != 25 {
|
||||
t.Fatalf("definition count = %d, want 25", len(defs))
|
||||
if len(defs) != 21 {
|
||||
t.Fatalf("definition count = %d, want 21", len(defs))
|
||||
}
|
||||
for _, def := range defs {
|
||||
if def.RetentionDays != defaultBuiltinMetricRetentionDays {
|
||||
@@ -194,6 +283,34 @@ func TestCreateMetricDefinitionsUsesExplicitRetentionAndPreservesOverrides(t *te
|
||||
}
|
||||
}
|
||||
|
||||
func TestCreateMetricDefinitionsRemovesObsoleteMetrics(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
s, err := metric.Open(ctx, metric.SQLite(":memory:", metric.WithMaxOpenConns(1)))
|
||||
if err != nil {
|
||||
t.Fatalf("open metric store: %v", err)
|
||||
}
|
||||
defer s.Close()
|
||||
if err := s.CreateMetric(ctx, metric.Definition{Name: "memory.total", Type: metric.TypeGauge, RetentionDays: 1}); err != nil {
|
||||
t.Fatalf("create obsolete definition: %v", err)
|
||||
}
|
||||
if err := s.Write(ctx, metric.Point{MetricName: "memory.total", EntityID: "node-a", Timestamp: time.Now().UTC(), Value: 1024}); err != nil {
|
||||
t.Fatalf("write obsolete point: %v", err)
|
||||
}
|
||||
if err := createMetricDefinitions(ctx, s); err != nil {
|
||||
t.Fatalf("refresh built-in definitions: %v", err)
|
||||
}
|
||||
if _, err := s.GetMetric(ctx, "memory.total"); !errors.Is(err, metric.ErrNotFound) {
|
||||
t.Fatalf("obsolete definition error = %v, want ErrNotFound", err)
|
||||
}
|
||||
points, err := s.Query(ctx, metric.Query{MetricName: "memory.total", EntityID: "node-a", Start: time.Now().UTC().Add(-time.Hour), End: time.Now().UTC().Add(time.Hour)})
|
||||
if err != nil {
|
||||
t.Fatalf("query obsolete points: %v", err)
|
||||
}
|
||||
if len(points) != 0 {
|
||||
t.Fatalf("obsolete points remain: %#v", points)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCreateMetricDefinitionsUsesLegacySpanOnlyForNewDefinitions(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
s, err := metric.Open(ctx, metric.SQLite(":memory:", metric.WithMaxOpenConns(1)))
|
||||
@@ -298,9 +415,9 @@ func TestSummarizeRetentionDefinitionsRequiresEveryMetricToBePositive(t *testing
|
||||
}
|
||||
}
|
||||
|
||||
func TestCompactCleansExpiredRawPointsWhenDownsamplingDisabled(t *testing.T) {
|
||||
func TestCompactCleansPointsOutsideFixedRawWindow(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
s, err := metric.Open(ctx, metric.SQLite(":memory:", metric.WithMaxOpenConns(1)))
|
||||
s, err := metric.Open(ctx, metric.SQLite(":memory:", metric.WithMaxOpenConns(1), metric.WithRollupPolicy(metric.RollupPolicy{})))
|
||||
if err != nil {
|
||||
t.Fatalf("open metric store: %v", err)
|
||||
}
|
||||
@@ -312,24 +429,21 @@ func TestCompactCleansExpiredRawPointsWhenDownsamplingDisabled(t *testing.T) {
|
||||
t.Fatalf("upsert metric: %v", err)
|
||||
}
|
||||
|
||||
now := time.Date(2026, 7, 11, 12, 0, 0, 0, time.UTC)
|
||||
now := time.Now().UTC()
|
||||
if err := s.WriteBatch(ctx, []metric.Point{
|
||||
{MetricName: "raw.metric", EntityID: "node", Timestamp: now.Add(-48 * time.Hour), Value: 1},
|
||||
{MetricName: "raw.metric", EntityID: "node", Timestamp: now.Add(-time.Hour), Value: 2},
|
||||
{MetricName: "raw.metric", EntityID: "node", Timestamp: now.Add(-11 * time.Minute), Value: 1},
|
||||
{MetricName: "raw.metric", EntityID: "node", Timestamp: now.Add(-30 * time.Second), Value: 2},
|
||||
}); err != nil {
|
||||
t.Fatalf("write points: %v", err)
|
||||
}
|
||||
|
||||
storeMu.Lock()
|
||||
oldStore := store
|
||||
oldCompactAt := compactAt
|
||||
store = s
|
||||
compactAt = 0
|
||||
storeMu.Unlock()
|
||||
defer func() {
|
||||
storeMu.Lock()
|
||||
store = oldStore
|
||||
compactAt = oldCompactAt
|
||||
storeMu.Unlock()
|
||||
_ = s.Close()
|
||||
}()
|
||||
@@ -340,7 +454,7 @@ func TestCompactCleansExpiredRawPointsWhenDownsamplingDisabled(t *testing.T) {
|
||||
points, err := s.Query(ctx, metric.Query{
|
||||
MetricName: "raw.metric",
|
||||
EntityID: "node",
|
||||
Start: now.Add(-72 * time.Hour),
|
||||
Start: now.Add(-time.Hour),
|
||||
End: now,
|
||||
})
|
||||
if err != nil {
|
||||
@@ -351,40 +465,69 @@ func TestCompactCleansExpiredRawPointsWhenDownsamplingDisabled(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestCompactKeepsRotatingCursorAfterFullCycle(t *testing.T) {
|
||||
func TestRetentionCleanupReportsDeleteFailure(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
s, err := metric.Open(ctx, metric.SQLite(":memory:",
|
||||
dsn := filepath.Join(t.TempDir(), "compact.db")
|
||||
s, err := metric.Open(ctx, metric.SQLite(dsn,
|
||||
metric.WithMaxOpenConns(1),
|
||||
metric.WithRollupPolicy(defaultRollupPolicy()),
|
||||
))
|
||||
if err != nil {
|
||||
t.Fatalf("open metric store: %v", err)
|
||||
}
|
||||
for _, name := range []string{"a.metric", "b.metric", "c.metric"} {
|
||||
if err := s.UpsertMetric(ctx, metric.Definition{Name: name, Type: metric.TypeGauge}); err != nil {
|
||||
t.Fatalf("upsert metric %s: %v", name, err)
|
||||
t.Cleanup(func() { _ = s.Close() })
|
||||
for _, name := range []string{"a.invalid", "b.healthy"} {
|
||||
if err := s.CreateMetric(ctx, metric.Definition{Name: name, Type: metric.TypeGauge, RetentionDays: 1}); err != nil {
|
||||
t.Fatalf("create metric %s: %v", name, err)
|
||||
}
|
||||
}
|
||||
|
||||
storeMu.Lock()
|
||||
oldStore := store
|
||||
oldCompactAt := compactAt
|
||||
store = s
|
||||
compactAt = 1
|
||||
storeMu.Unlock()
|
||||
defer func() {
|
||||
storeMu.Lock()
|
||||
store = oldStore
|
||||
compactAt = oldCompactAt
|
||||
storeMu.Unlock()
|
||||
_ = s.Close()
|
||||
}()
|
||||
|
||||
if _, err := Compact(ctx, time.Date(2026, 7, 11, 12, 0, 0, 0, time.UTC)); err != nil {
|
||||
t.Fatalf("compact: %v", err)
|
||||
now := time.Date(2026, 7, 20, 12, 0, 0, 0, time.UTC)
|
||||
old := now.Add(-48 * time.Hour)
|
||||
if err := s.WriteBatch(ctx, []metric.Point{
|
||||
{MetricName: "a.invalid", EntityID: "node", Timestamp: old, Value: 1},
|
||||
{MetricName: "b.healthy", EntityID: "node", Timestamp: old, Value: 2},
|
||||
}); err != nil {
|
||||
t.Fatalf("write compact fixtures: %v", err)
|
||||
}
|
||||
if compactAt != 1 {
|
||||
t.Fatalf("compact cursor = %d, want 1 after a complete rotated cycle", compactAt)
|
||||
|
||||
rawDB, err := sql.Open("sqlite3", dsn)
|
||||
if err != nil {
|
||||
t.Fatalf("open raw sqlite connection: %v", err)
|
||||
}
|
||||
_, err = rawDB.ExecContext(ctx, `CREATE TRIGGER fail_invalid_metric_rollup
|
||||
BEFORE DELETE ON metric_rollups
|
||||
WHEN OLD.series_id IN (SELECT id FROM metric_series WHERE metric_name = 'a.invalid')
|
||||
BEGIN SELECT RAISE(FAIL, 'forced compact failure'); END`)
|
||||
_ = rawDB.Close()
|
||||
if err != nil {
|
||||
t.Fatalf("create compact failure trigger: %v", err)
|
||||
}
|
||||
|
||||
storeMu.Lock()
|
||||
previousStore := store
|
||||
store = s
|
||||
storeMu.Unlock()
|
||||
t.Cleanup(func() {
|
||||
storeMu.Lock()
|
||||
store = previousStore
|
||||
storeMu.Unlock()
|
||||
})
|
||||
|
||||
if _, err := CleanupExpired(ctx, now); err == nil {
|
||||
t.Fatal("expected retention cleanup to report the forced delete failure")
|
||||
}
|
||||
points, err := s.Query(ctx, metric.Query{
|
||||
MetricName: "b.healthy",
|
||||
EntityID: "node",
|
||||
Start: old.Add(-time.Minute),
|
||||
End: now,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("query healthy rollups: %v", err)
|
||||
}
|
||||
if len(points) != 0 {
|
||||
t.Fatalf("retention cleanup unexpectedly changed healthy metric data: %#v", points)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -403,14 +546,11 @@ func TestGetRecordsByClientAndTimeReadsRollupsAfterRawCompaction(t *testing.T) {
|
||||
|
||||
storeMu.Lock()
|
||||
oldStore := store
|
||||
oldCompactAt := compactAt
|
||||
store = s
|
||||
compactAt = 0
|
||||
storeMu.Unlock()
|
||||
defer func() {
|
||||
storeMu.Lock()
|
||||
store = oldStore
|
||||
compactAt = oldCompactAt
|
||||
storeMu.Unlock()
|
||||
_ = s.Close()
|
||||
}()
|
||||
@@ -447,19 +587,6 @@ func TestGetRecordsByClientAndTimeReadsRollupsAfterRawCompaction(t *testing.T) {
|
||||
if _, err := s.Compact(ctx, now); err != nil {
|
||||
t.Fatalf("compact raw into rollup: %v", err)
|
||||
}
|
||||
raw, err := s.Query(ctx, metric.Query{
|
||||
MetricName: MetricCPU,
|
||||
EntityID: rec.Client,
|
||||
Start: ts.Add(-time.Minute),
|
||||
End: now,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("query raw cpu: %v", err)
|
||||
}
|
||||
if len(raw) != 0 {
|
||||
t.Fatalf("expected old raw cpu point to be deleted after compaction, got %d", len(raw))
|
||||
}
|
||||
|
||||
got, err := GetRecordsByClientAndTime(ctx, rec.Client, ts.Add(-time.Minute), now)
|
||||
if err != nil {
|
||||
t.Fatalf("get records: %v", err)
|
||||
@@ -479,3 +606,41 @@ func TestGetRecordsByClientAndTimeReadsRollupsAfterRawCompaction(t *testing.T) {
|
||||
t.Fatalf("all-client records were not reconstructed from rollup: %#v", all)
|
||||
}
|
||||
}
|
||||
|
||||
func TestGetRecordMetricMaxByClientAndTimeQueriesOnlySelectedMetric(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
s, err := metric.Open(ctx, metric.SQLite(":memory:",
|
||||
metric.WithMaxOpenConns(1),
|
||||
metric.WithRollupPolicy(defaultRollupPolicy()),
|
||||
))
|
||||
if err != nil {
|
||||
t.Fatalf("open metric store: %v", err)
|
||||
}
|
||||
defer s.Close()
|
||||
if err := createMetricDefinitions(ctx, s); err != nil {
|
||||
t.Fatalf("create metric definitions: %v", err)
|
||||
}
|
||||
|
||||
base := time.Now().UTC().Truncate(time.Minute).Add(-time.Minute)
|
||||
if err := s.WriteBatch(ctx, []metric.Point{
|
||||
{MetricName: MetricCPU, EntityID: "node-a", Timestamp: base.Add(10 * time.Second), Value: 10},
|
||||
{MetricName: MetricCPU, EntityID: "node-a", Timestamp: base.Add(20 * time.Second), Value: 90},
|
||||
{MetricName: MetricRAM, EntityID: "node-a", Timestamp: base.Add(20 * time.Second), Value: 123456},
|
||||
}); err != nil {
|
||||
t.Fatalf("write metric points: %v", err)
|
||||
}
|
||||
|
||||
got, err := getRecordMetricMaxByClientAndTimeFromSeries(ctx, s, "node-a", "cpu", base, base.Add(time.Minute))
|
||||
if err != nil {
|
||||
t.Fatalf("get CPU max records: %v", err)
|
||||
}
|
||||
if len(got) != 1 {
|
||||
t.Fatalf("record count = %d, want 1: %#v", len(got), got)
|
||||
}
|
||||
if got[0].Cpu != 90 {
|
||||
t.Fatalf("CPU max = %v, want 90", got[0].Cpu)
|
||||
}
|
||||
if got[0].Ram != 0 {
|
||||
t.Fatalf("unselected RAM value = %d, want 0", got[0].Ram)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,112 @@
|
||||
package metricstore
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
"github.com/komari-monitor/komari/pkg/metric"
|
||||
logger "github.com/komari-monitor/komari/utils/log"
|
||||
)
|
||||
|
||||
const storeMigrationBatchSize = 500
|
||||
|
||||
// configFromFingerprint 从目标指纹(driver|dsn)重建一个 MetricStoreConfig,
|
||||
// 用于以只读方式打开“上一次的 metrics 目标库”。表前缀、保留天数、连接数等
|
||||
// 沿用当前配置(切换后端时这些通常不变)。
|
||||
func configFromFingerprint(fingerprint string, base *MetricStoreConfig) (*MetricStoreConfig, error) {
|
||||
idx := strings.Index(fingerprint, "|")
|
||||
if idx < 0 {
|
||||
return nil, fmt.Errorf("invalid target fingerprint: %q", fingerprint)
|
||||
}
|
||||
driver := fingerprint[:idx]
|
||||
dsn := fingerprint[idx+1:]
|
||||
if strings.TrimSpace(driver) == "" {
|
||||
return nil, fmt.Errorf("empty driver in target fingerprint: %q", fingerprint)
|
||||
}
|
||||
return &MetricStoreConfig{
|
||||
Driver: driver,
|
||||
DSN: dsn,
|
||||
TablePrefix: base.TablePrefix,
|
||||
MaxOpenConns: base.MaxOpenConns,
|
||||
MaxIdleConns: base.MaxIdleConns,
|
||||
RollupMinuteRetentionMinutes: base.RollupMinuteRetentionMinutes,
|
||||
RollupFiveMinuteRetentionMinutes: base.RollupFiveMinuteRetentionMinutes,
|
||||
RollupHourRetentionHours: base.RollupHourRetentionHours,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// openSourceStore 打开一个已存在的 metrics 目标库作为数据搬运的源库读取。
|
||||
//
|
||||
// 使用 autoMigrate=true 确保规范化表和索引存在;创建语句不会删除已有数据。
|
||||
// 当源库文件/表不存在(例如老快照记录了 completed 但 metrics.db 缺失)时,
|
||||
// 创建空表可让后续 ListMetrics 返回空集,把“无历史可迁移”识别为正常情况。
|
||||
func openSourceStore(ctx context.Context, cfg *MetricStoreConfig) (*metric.Store, error) {
|
||||
metricCfg, err := buildMetricConfig(cfg, true)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return metric.Open(ctx, metricCfg)
|
||||
}
|
||||
|
||||
// defaultSQLiteFingerprint 返回默认 SQLite metrics 库(./data/metrics.db)的目标指纹。
|
||||
// 老快照的 metrics 数据固定落在该 SQLite 文件,用于在缺失指纹时推断上一个源库。
|
||||
func defaultSQLiteFingerprint() string {
|
||||
return targetFingerprint(&MetricStoreConfig{Driver: "sqlite", DSN: "./data/metrics.db"})
|
||||
}
|
||||
|
||||
// storeMigrationObserver 在 store-to-store 迁移过程中接收进度回调。
|
||||
// - currentMetric:当前正在搬运的指标名。
|
||||
// - metricIndex:该指标在全部指标中的序号(0 起),即已完成的指标数。
|
||||
// - totalMetrics:指标定义总数。
|
||||
// - addedPoints:本次新写入目标库的 rollup 行数(用于外部累计)。
|
||||
type storeMigrationObserver func(currentMetric string, metricIndex, totalMetrics int, addedPoints int64)
|
||||
|
||||
// 观察者 observe(为 nil 时行为与旧版本完全一致)。WebUI/API 手动触发的迁移传入
|
||||
// 回调以实时更新进度。
|
||||
func migrateBetweenStores(ctx context.Context, src, dst *metric.Store, observe storeMigrationObserver) (int64, error) {
|
||||
if src == nil || dst == nil {
|
||||
return 0, fmt.Errorf("source or destination metric store is nil")
|
||||
}
|
||||
|
||||
defs, err := src.ListMetrics(ctx)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("list source metrics: %w", err)
|
||||
}
|
||||
|
||||
var total int64
|
||||
for i, def := range defs {
|
||||
if observe != nil {
|
||||
// 进入下一个指标:已完成 i 个指标。
|
||||
observe(def.Name, i, len(defs), 0)
|
||||
}
|
||||
// 目标库先建立指标定义,保证后续写入的指标存在。
|
||||
if err := dst.UpsertMetric(ctx, def); err != nil {
|
||||
return total, fmt.Errorf("upsert metric %q on target: %w", def.Name, err)
|
||||
}
|
||||
if def.RetentionDays == 0 {
|
||||
continue
|
||||
}
|
||||
var migrated int64
|
||||
_, err := src.ExportRollups(ctx, def.Name, storeMigrationBatchSize, func(batch []metric.PersistedRollup) error {
|
||||
if err := dst.ImportRollups(ctx, batch); err != nil {
|
||||
return fmt.Errorf("write metric %q rollup batch to target: %w", def.Name, err)
|
||||
}
|
||||
count := int64(len(batch))
|
||||
migrated += count
|
||||
total += count
|
||||
if observe != nil {
|
||||
observe(def.Name, i, len(defs), count)
|
||||
}
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
return total, fmt.Errorf("export metric %q rollups: %w", def.Name, err)
|
||||
}
|
||||
if migrated > 0 {
|
||||
logger.Infof("metricstore", "[store-migration] metric %q: migrated %d rollups", def.Name, migrated)
|
||||
}
|
||||
}
|
||||
|
||||
return total, nil
|
||||
}
|
||||
@@ -0,0 +1,40 @@
|
||||
package metricstore
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"golang.org/x/sync/semaphore"
|
||||
)
|
||||
|
||||
// storeOperationGate keeps an active Store stable while allowing regular
|
||||
// reads, report writes, and compaction to proceed together. Store replacement
|
||||
// and maintenance take the whole gate exclusively.
|
||||
type storeOperationGate struct {
|
||||
semaphore *semaphore.Weighted
|
||||
}
|
||||
|
||||
const storeOperationGateWeight int64 = 1024
|
||||
|
||||
func newStoreOperationGate() *storeOperationGate {
|
||||
return &storeOperationGate{semaphore: semaphore.NewWeighted(storeOperationGateWeight)}
|
||||
}
|
||||
|
||||
func (g *storeOperationGate) Acquire(ctx context.Context) error {
|
||||
return g.semaphore.Acquire(ctx, storeOperationGateWeight)
|
||||
}
|
||||
|
||||
func (g *storeOperationGate) TryAcquire() bool {
|
||||
return g.semaphore.TryAcquire(storeOperationGateWeight)
|
||||
}
|
||||
|
||||
func (g *storeOperationGate) Release() {
|
||||
g.semaphore.Release(storeOperationGateWeight)
|
||||
}
|
||||
|
||||
func (g *storeOperationGate) AcquireShared(ctx context.Context) error {
|
||||
return g.semaphore.Acquire(ctx, 1)
|
||||
}
|
||||
|
||||
func (g *storeOperationGate) ReleaseShared() {
|
||||
g.semaphore.Release(1)
|
||||
}
|
||||
@@ -0,0 +1,133 @@
|
||||
package metricstore
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"sort"
|
||||
"time"
|
||||
|
||||
"github.com/komari-monitor/komari/database/models"
|
||||
"github.com/komari-monitor/komari/pkg/metric"
|
||||
)
|
||||
|
||||
// WritePingRecord 将 ping 记录写入 metric store
|
||||
func WritePingRecord(ctx context.Context, rec models.PingRecord) error {
|
||||
s := GetStore()
|
||||
if s == nil {
|
||||
return fmt.Errorf("metric store not enabled")
|
||||
}
|
||||
|
||||
reportBatcherMu.Lock()
|
||||
worker := reportBatcher
|
||||
reportBatcherMu.Unlock()
|
||||
if worker != nil {
|
||||
return worker.enqueuePing(ctx, rec)
|
||||
}
|
||||
return writePingRecords(ctx, []models.PingRecord{rec})
|
||||
}
|
||||
|
||||
func writePingRecords(ctx context.Context, records []models.PingRecord) error {
|
||||
s := GetStore()
|
||||
if s == nil {
|
||||
return fmt.Errorf("metric store not enabled")
|
||||
}
|
||||
if len(records) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
points := make([]metric.Point, 0, len(records)*2)
|
||||
for _, rec := range records {
|
||||
tags := map[string]string{"task_id": fmt.Sprintf("%d", rec.TaskId)}
|
||||
loss := 0.0
|
||||
if rec.Value < 0 {
|
||||
loss = 1
|
||||
}
|
||||
points = append(points,
|
||||
metric.Point{
|
||||
MetricName: MetricPingLatency,
|
||||
EntityID: rec.Client,
|
||||
Timestamp: rec.Time,
|
||||
Value: float64(rec.Value),
|
||||
Tags: tags,
|
||||
},
|
||||
metric.Point{
|
||||
MetricName: MetricPingLoss,
|
||||
EntityID: rec.Client,
|
||||
Timestamp: rec.Time,
|
||||
Value: loss,
|
||||
Tags: tags,
|
||||
},
|
||||
)
|
||||
}
|
||||
return s.WriteBatch(ctx, points)
|
||||
}
|
||||
|
||||
func GetPingRecords(ctx context.Context, clientUUID string, taskID int, start, end time.Time) ([]models.PingRecord, error) {
|
||||
s := GetStore()
|
||||
if s == nil {
|
||||
return nil, fmt.Errorf("metric store not enabled")
|
||||
}
|
||||
|
||||
query := metric.Query{
|
||||
MetricName: MetricPingLatency,
|
||||
Start: start,
|
||||
End: end,
|
||||
Order: metric.OrderAsc,
|
||||
}
|
||||
|
||||
if clientUUID != "" {
|
||||
query.EntityID = clientUUID
|
||||
}
|
||||
|
||||
if taskID >= 0 {
|
||||
query.Tags = map[string]string{"task_id": fmt.Sprintf("%d", taskID)}
|
||||
}
|
||||
|
||||
interval := pingQueryInterval(end.Sub(start), 4000)
|
||||
interval = s.CompatibleSeriesInterval(start, time.Now().UTC(), interval)
|
||||
points, err := s.Series(ctx, metric.AggregateQuery{
|
||||
Query: query,
|
||||
Aggregation: metric.AggLast,
|
||||
Interval: interval,
|
||||
PreserveSeries: true,
|
||||
}, time.Now().UTC())
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
records := make([]models.PingRecord, 0, len(points))
|
||||
for _, p := range points {
|
||||
taskIDVal := uint(0)
|
||||
if tid, ok := p.Tags["task_id"]; ok {
|
||||
var t uint64
|
||||
fmt.Sscanf(tid, "%d", &t)
|
||||
taskIDVal = uint(t)
|
||||
}
|
||||
|
||||
records = append(records, models.PingRecord{
|
||||
Client: p.EntityID,
|
||||
TaskId: taskIDVal,
|
||||
Time: p.Bucket.UTC(),
|
||||
Value: int(p.Value),
|
||||
})
|
||||
}
|
||||
sort.Slice(records, func(i, j int) bool {
|
||||
return records[i].Time.After(records[j].Time)
|
||||
})
|
||||
|
||||
return records, nil
|
||||
}
|
||||
|
||||
func pingQueryInterval(rangeDuration time.Duration, maxPoints int) time.Duration {
|
||||
if maxPoints <= 0 {
|
||||
maxPoints = 4000
|
||||
}
|
||||
if rangeDuration <= 0 {
|
||||
return time.Second
|
||||
}
|
||||
interval := time.Duration((rangeDuration.Nanoseconds() + int64(maxPoints) - 1) / int64(maxPoints))
|
||||
if interval < time.Second {
|
||||
return time.Second
|
||||
}
|
||||
return metric.FloorStandardInterval(interval)
|
||||
}
|
||||
@@ -0,0 +1,164 @@
|
||||
package metricstore
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"errors"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"github.com/komari-monitor/komari/internal/config"
|
||||
"github.com/komari-monitor/komari/pkg/metric"
|
||||
"gorm.io/driver/sqlite"
|
||||
"gorm.io/gorm"
|
||||
)
|
||||
|
||||
func prepareRecoveryTest(t *testing.T) {
|
||||
t.Helper()
|
||||
configDB, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{})
|
||||
if err != nil {
|
||||
t.Fatalf("open config db: %v", err)
|
||||
}
|
||||
config.SetDb(configDB)
|
||||
|
||||
storeInitMu.Lock()
|
||||
storeMigMu.Lock()
|
||||
previousClosing := storeClosing
|
||||
storeClosing = false
|
||||
storeMigMu.Unlock()
|
||||
storeMu.Lock()
|
||||
previousStore := store
|
||||
previousFingerprint := storeFingerprint
|
||||
store = nil
|
||||
storeFingerprint = ""
|
||||
storeMu.Unlock()
|
||||
storeInitMu.Unlock()
|
||||
|
||||
t.Cleanup(func() {
|
||||
_ = CloseStoreContext(context.Background())
|
||||
storeMigMu.Lock()
|
||||
storeClosing = previousClosing
|
||||
storeMigMu.Unlock()
|
||||
storeMu.Lock()
|
||||
store = previousStore
|
||||
storeFingerprint = previousFingerprint
|
||||
storeMu.Unlock()
|
||||
})
|
||||
}
|
||||
|
||||
func TestRecoverStoreFailureKeepsCurrentConfig(t *testing.T) {
|
||||
prepareRecoveryTest(t)
|
||||
if err := config.SetMany(map[string]any{
|
||||
MetricDBDriverKey: "sqlite",
|
||||
MetricDBDSNKey: "old.db",
|
||||
}); err != nil {
|
||||
t.Fatalf("save original config: %v", err)
|
||||
}
|
||||
|
||||
missing := "file:" + filepath.ToSlash(filepath.Join(t.TempDir(), "missing.db")) + "?mode=ro"
|
||||
err := RecoverStore(context.Background(), &MetricStoreConfig{Driver: "sqlite", DSN: missing})
|
||||
if err == nil {
|
||||
t.Fatal("recovery unexpectedly opened a missing read-only database")
|
||||
}
|
||||
if got, _ := config.GetAs[string](MetricDBDSNKey, ""); got != "old.db" {
|
||||
t.Fatalf("DSN changed after failed recovery: %q", got)
|
||||
}
|
||||
if GetStore() != nil {
|
||||
t.Fatal("failed recovery installed a store")
|
||||
}
|
||||
}
|
||||
|
||||
func TestRecoverStoreRecordsManualMigrationSource(t *testing.T) {
|
||||
prepareRecoveryTest(t)
|
||||
dsn := filepath.ToSlash(filepath.Join(t.TempDir(), "recovered.db"))
|
||||
t.Cleanup(func() { _ = CloseStoreContext(context.Background()) })
|
||||
cfg := &MetricStoreConfig{Driver: "sqlite", DSN: dsn}
|
||||
if err := RecoverStore(context.Background(), cfg); err != nil {
|
||||
t.Fatalf("recover store: %v", err)
|
||||
}
|
||||
if GetStore() == nil {
|
||||
t.Fatal("successful recovery did not install a store")
|
||||
}
|
||||
wantTarget := targetFingerprint(cfg)
|
||||
if got, _ := config.GetAs[string](MigrationTargetKey, ""); got != wantTarget {
|
||||
t.Fatalf("migration target = %q, want %q", got, wantTarget)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRecoverStorePersistsLegacyStoreForStructureGuide(t *testing.T) {
|
||||
prepareRecoveryTest(t)
|
||||
dsn := filepath.ToSlash(filepath.Join(t.TempDir(), "legacy.db"))
|
||||
db, err := sql.Open("sqlite3", dsn)
|
||||
if err != nil {
|
||||
t.Fatalf("open legacy store: %v", err)
|
||||
}
|
||||
if _, err := db.Exec(`CREATE TABLE metric_definitions (
|
||||
name TEXT PRIMARY KEY, type TEXT NOT NULL, unit TEXT NOT NULL,
|
||||
description TEXT NOT NULL, retention_days INTEGER NOT NULL,
|
||||
metadata TEXT NOT NULL, created_at INTEGER NOT NULL, updated_at INTEGER NOT NULL
|
||||
)`); err != nil {
|
||||
_ = db.Close()
|
||||
t.Fatalf("create legacy definition table: %v", err)
|
||||
}
|
||||
if err := db.Close(); err != nil {
|
||||
t.Fatalf("close legacy store: %v", err)
|
||||
}
|
||||
|
||||
cfg := &MetricStoreConfig{Driver: "sqlite", DSN: dsn}
|
||||
if err := RecoverStore(context.Background(), cfg); err != nil {
|
||||
t.Fatalf("recover legacy store: %v", err)
|
||||
}
|
||||
if GetStore() != nil {
|
||||
t.Fatal("legacy recovery installed the store before structure upgrade")
|
||||
}
|
||||
if got, _ := config.GetAs[string](MetricDBDSNKey, ""); got != dsn {
|
||||
t.Fatalf("saved legacy DSN = %q, want %q", got, dsn)
|
||||
}
|
||||
required, err := StructureUpgradeRequired(context.Background())
|
||||
if err != nil {
|
||||
t.Fatalf("detect structure upgrade after recovery: %v", err)
|
||||
}
|
||||
if !required {
|
||||
t.Fatal("legacy recovery did not lead to the structure upgrade guide")
|
||||
}
|
||||
}
|
||||
|
||||
func TestReloadRejectsLegacyStoreForStructureUpgrade(t *testing.T) {
|
||||
prepareRecoveryTest(t)
|
||||
active, err := metric.Open(context.Background(), metric.SQLite(":memory:"))
|
||||
if err != nil {
|
||||
t.Fatalf("open active metric store: %v", err)
|
||||
}
|
||||
installTestStore(t, active)
|
||||
|
||||
dsn := filepath.ToSlash(filepath.Join(t.TempDir(), "legacy.db"))
|
||||
db, err := sql.Open("sqlite3", dsn)
|
||||
if err != nil {
|
||||
t.Fatalf("open legacy store: %v", err)
|
||||
}
|
||||
if _, err := db.Exec(`CREATE TABLE metric_definitions (
|
||||
name TEXT PRIMARY KEY, type TEXT NOT NULL, unit TEXT NOT NULL,
|
||||
description TEXT NOT NULL, retention_days INTEGER NOT NULL,
|
||||
metadata TEXT NOT NULL, created_at INTEGER NOT NULL, updated_at INTEGER NOT NULL
|
||||
)`); err != nil {
|
||||
_ = db.Close()
|
||||
t.Fatalf("create legacy definition table: %v", err)
|
||||
}
|
||||
if err := db.Close(); err != nil {
|
||||
t.Fatalf("close legacy store: %v", err)
|
||||
}
|
||||
if err := config.SetMany(map[string]any{
|
||||
MetricDBDriverKey: "sqlite",
|
||||
MetricDBDSNKey: dsn,
|
||||
}); err != nil {
|
||||
t.Fatalf("save legacy metric store config: %v", err)
|
||||
}
|
||||
|
||||
err = Reload(context.Background())
|
||||
if !errors.Is(err, ErrStructureUpgradeRequired) {
|
||||
t.Fatalf("reload error = %v, want %v", err, ErrStructureUpgradeRequired)
|
||||
}
|
||||
if GetStore() != active {
|
||||
t.Fatal("legacy target replaced the active metric store during hot reload")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,22 @@
|
||||
package metricstore
|
||||
|
||||
import (
|
||||
"regexp"
|
||||
"strings"
|
||||
)
|
||||
|
||||
var (
|
||||
metricCredentialPattern = regexp.MustCompile(`(?i)\b(password|passwd)\s*=\s*('[^']*'|"[^"]*"|[^\s,;)&]+)`)
|
||||
metricURLCredentialPattern = regexp.MustCompile(`(?i)([a-z0-9_.-]+):([^@\s]+)@`)
|
||||
)
|
||||
|
||||
// RedactConnectionError removes the supplied DSN and common password fragments
|
||||
// before a connection error is shown to an administrator or written to a log.
|
||||
func RedactConnectionError(message, dsn string) string {
|
||||
message = strings.TrimSpace(message)
|
||||
if dsn != "" {
|
||||
message = strings.ReplaceAll(message, dsn, "[redacted]")
|
||||
}
|
||||
message = metricCredentialPattern.ReplaceAllString(message, "$1=[redacted]")
|
||||
return metricURLCredentialPattern.ReplaceAllString(message, "$1:[redacted]@")
|
||||
}
|
||||
@@ -0,0 +1,414 @@
|
||||
package metricstore
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
logger "github.com/komari-monitor/komari/utils/log"
|
||||
|
||||
"github.com/komari-monitor/komari/database/models"
|
||||
"github.com/komari-monitor/komari/pkg/metric"
|
||||
v1 "github.com/komari-monitor/komari/protocol/v1"
|
||||
)
|
||||
|
||||
type reportTrafficState struct {
|
||||
mu sync.Mutex
|
||||
|
||||
reportTrafficValues
|
||||
}
|
||||
|
||||
type reportTrafficValues struct {
|
||||
initialized bool
|
||||
timestamp time.Time
|
||||
hasUp bool
|
||||
totalUp int64
|
||||
hasDown bool
|
||||
totalDown int64
|
||||
}
|
||||
|
||||
var reportTrafficStates sync.Map
|
||||
|
||||
const (
|
||||
reportBatchInterval = 3 * time.Second
|
||||
reportBatchQueueSize = 4096
|
||||
pingBatchMaxRecords = 512
|
||||
reportBatchWriteTimeout = 10 * time.Second
|
||||
)
|
||||
|
||||
var (
|
||||
reportBatcherMu sync.Mutex
|
||||
reportBatcher *reportBatchWorker
|
||||
)
|
||||
|
||||
var (
|
||||
ErrReportBatchQueueFull = errors.New("metric report batch queue is full")
|
||||
ErrReportBatchStopped = errors.New("metric report batcher is stopped")
|
||||
ErrPingBatchQueueFull = errors.New("ping batch queue is full")
|
||||
)
|
||||
|
||||
type reportBatchRequest struct {
|
||||
ctx context.Context
|
||||
done chan error
|
||||
stop bool
|
||||
}
|
||||
|
||||
type reportBatchWorker struct {
|
||||
mu sync.Mutex
|
||||
queue chan v1.Report
|
||||
pingQueue chan models.PingRecord
|
||||
requests chan reportBatchRequest
|
||||
done chan struct{}
|
||||
stopping bool
|
||||
}
|
||||
|
||||
// StartReportBatcher starts the shared report writer. Exact samples are kept in
|
||||
// the short raw window while the same writes update in-memory minute buckets.
|
||||
func StartReportBatcher() {
|
||||
reportBatcherMu.Lock()
|
||||
defer reportBatcherMu.Unlock()
|
||||
if reportBatcher != nil {
|
||||
return
|
||||
}
|
||||
worker := &reportBatchWorker{
|
||||
queue: make(chan v1.Report, reportBatchQueueSize),
|
||||
pingQueue: make(chan models.PingRecord, reportBatchQueueSize),
|
||||
requests: make(chan reportBatchRequest, 1),
|
||||
done: make(chan struct{}),
|
||||
}
|
||||
reportBatcher = worker
|
||||
go worker.run()
|
||||
}
|
||||
|
||||
// StopReportBatcher stops the report writer after flushing all queued reports.
|
||||
func StopReportBatcher(ctx context.Context) error {
|
||||
reportBatcherMu.Lock()
|
||||
worker := reportBatcher
|
||||
if worker == nil {
|
||||
reportBatcherMu.Unlock()
|
||||
return nil
|
||||
}
|
||||
worker.mu.Lock()
|
||||
worker.stopping = true
|
||||
worker.mu.Unlock()
|
||||
reportBatcherMu.Unlock()
|
||||
|
||||
request := reportBatchRequest{ctx: ctx, done: make(chan error, 1), stop: true}
|
||||
select {
|
||||
case worker.requests <- request:
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
}
|
||||
select {
|
||||
case err := <-request.done:
|
||||
<-worker.done
|
||||
reportBatcherMu.Lock()
|
||||
if reportBatcher == worker {
|
||||
reportBatcher = nil
|
||||
}
|
||||
reportBatcherMu.Unlock()
|
||||
return err
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
}
|
||||
}
|
||||
|
||||
// FlushReportBatch synchronously flushes the current queue. It is useful for
|
||||
// controlled handoff points and deterministic tests; normal operation uses the
|
||||
// worker ticker and the active mode's flush interval.
|
||||
func FlushReportBatch(ctx context.Context) error {
|
||||
reportBatcherMu.Lock()
|
||||
worker := reportBatcher
|
||||
reportBatcherMu.Unlock()
|
||||
if worker == nil {
|
||||
return nil
|
||||
}
|
||||
request := reportBatchRequest{ctx: ctx, done: make(chan error, 1)}
|
||||
worker.mu.Lock()
|
||||
stopping := worker.stopping
|
||||
worker.mu.Unlock()
|
||||
if stopping {
|
||||
return ErrReportBatchStopped
|
||||
}
|
||||
select {
|
||||
case worker.requests <- request:
|
||||
case <-worker.done:
|
||||
return ErrReportBatchStopped
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
}
|
||||
select {
|
||||
case err := <-request.done:
|
||||
return err
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
}
|
||||
}
|
||||
|
||||
// WriteReport persists one agent report and adds it to in-memory minute
|
||||
// summaries using the same server receive time. Traffic deltas remain summable
|
||||
// after rollup.
|
||||
func WriteReport(ctx context.Context, report v1.Report) (v1.Report, error) {
|
||||
if report.UUID == "" {
|
||||
return v1.Report{}, fmt.Errorf("report UUID is required")
|
||||
}
|
||||
if report.UpdatedAt.IsZero() {
|
||||
return v1.Report{}, fmt.Errorf("report receive time is required")
|
||||
}
|
||||
report.UpdatedAt = report.UpdatedAt.UTC()
|
||||
if GetStore() == nil {
|
||||
return v1.Report{}, fmt.Errorf("metric store not enabled")
|
||||
}
|
||||
|
||||
reportBatcherMu.Lock()
|
||||
worker := reportBatcher
|
||||
reportBatcherMu.Unlock()
|
||||
if worker != nil {
|
||||
if err := worker.enqueue(ctx, report); err != nil {
|
||||
return v1.Report{}, err
|
||||
}
|
||||
return report, nil
|
||||
}
|
||||
|
||||
saved, err := writeReportBatch(ctx, []v1.Report{report})
|
||||
if err != nil {
|
||||
return v1.Report{}, err
|
||||
}
|
||||
return saved[0], nil
|
||||
}
|
||||
|
||||
func (w *reportBatchWorker) enqueue(ctx context.Context, report v1.Report) error {
|
||||
w.mu.Lock()
|
||||
defer w.mu.Unlock()
|
||||
if w.stopping {
|
||||
return ErrReportBatchStopped
|
||||
}
|
||||
if err := ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
select {
|
||||
case w.queue <- report:
|
||||
return nil
|
||||
default:
|
||||
return ErrReportBatchQueueFull
|
||||
}
|
||||
}
|
||||
|
||||
func (w *reportBatchWorker) run() {
|
||||
ticker := time.NewTicker(reportBatchInterval)
|
||||
defer ticker.Stop()
|
||||
|
||||
var pending []v1.Report
|
||||
var pendingPings []models.PingRecord
|
||||
for {
|
||||
select {
|
||||
case request := <-w.requests:
|
||||
pending = append(pending, drainReportQueue(w.queue, reportBatchQueueSize)...)
|
||||
pendingPings = append(pendingPings, drainPingQueue(w.pingQueue, reportBatchQueueSize)...)
|
||||
err := errors.Join(
|
||||
writePendingReports(request.ctx, &pending),
|
||||
writePendingPingRecords(request.ctx, &pendingPings),
|
||||
)
|
||||
if request.stop {
|
||||
if err != nil {
|
||||
logger.Errorf("metricstore", "failed to flush metric report batch during shutdown: %v", err)
|
||||
}
|
||||
close(w.done)
|
||||
request.done <- err
|
||||
return
|
||||
}
|
||||
request.done <- err
|
||||
case <-ticker.C:
|
||||
pendingPings = append(pendingPings, drainPingQueue(w.pingQueue, reportBatchQueueSize)...)
|
||||
if err := writePendingPingRecords(context.Background(), &pendingPings); err != nil {
|
||||
logger.Errorf("metricstore", "failed to flush ping batch: %v", err)
|
||||
}
|
||||
pending = append(pending, drainReportQueue(w.queue, reportBatchQueueSize)...)
|
||||
if err := writePendingReports(context.Background(), &pending); err != nil {
|
||||
logger.Errorf("metricstore", "failed to flush metric report batch: %v", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (w *reportBatchWorker) enqueuePing(ctx context.Context, record models.PingRecord) error {
|
||||
w.mu.Lock()
|
||||
defer w.mu.Unlock()
|
||||
if w.stopping {
|
||||
return ErrReportBatchStopped
|
||||
}
|
||||
if err := ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
select {
|
||||
case w.pingQueue <- record:
|
||||
return nil
|
||||
default:
|
||||
return ErrPingBatchQueueFull
|
||||
}
|
||||
}
|
||||
|
||||
func drainReportQueue(queue <-chan v1.Report, limit int) []v1.Report {
|
||||
if limit <= 0 {
|
||||
return nil
|
||||
}
|
||||
capacity := len(queue)
|
||||
if capacity > limit {
|
||||
capacity = limit
|
||||
}
|
||||
if capacity == 0 {
|
||||
return nil
|
||||
}
|
||||
reports := make([]v1.Report, 0, capacity)
|
||||
for len(reports) < limit {
|
||||
select {
|
||||
case report := <-queue:
|
||||
reports = append(reports, report)
|
||||
default:
|
||||
return reports
|
||||
}
|
||||
}
|
||||
return reports
|
||||
}
|
||||
|
||||
func drainPingQueue(queue <-chan models.PingRecord, limit int) []models.PingRecord {
|
||||
if limit <= 0 {
|
||||
return nil
|
||||
}
|
||||
capacity := len(queue)
|
||||
if capacity > limit {
|
||||
capacity = limit
|
||||
}
|
||||
if capacity == 0 {
|
||||
return nil
|
||||
}
|
||||
records := make([]models.PingRecord, 0, capacity)
|
||||
for len(records) < limit {
|
||||
select {
|
||||
case record := <-queue:
|
||||
records = append(records, record)
|
||||
default:
|
||||
return records
|
||||
}
|
||||
}
|
||||
return records
|
||||
}
|
||||
|
||||
func writePendingReports(ctx context.Context, pending *[]v1.Report) error {
|
||||
if len(*pending) == 0 {
|
||||
return nil
|
||||
}
|
||||
batchSize := len(*pending)
|
||||
for len(*pending) > 0 {
|
||||
if batchSize > len(*pending) {
|
||||
batchSize = len(*pending)
|
||||
}
|
||||
writeCtx, cancel := context.WithTimeout(ctx, reportBatchWriteTimeout)
|
||||
_, err := writeReportBatch(writeCtx, (*pending)[:batchSize])
|
||||
cancel()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
*pending = (*pending)[batchSize:]
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func writePendingPingRecords(ctx context.Context, pending *[]models.PingRecord) error {
|
||||
for len(*pending) > 0 {
|
||||
batchSize := len(*pending)
|
||||
if batchSize > pingBatchMaxRecords {
|
||||
batchSize = pingBatchMaxRecords
|
||||
}
|
||||
writeCtx, cancel := context.WithTimeout(ctx, reportBatchWriteTimeout)
|
||||
err := writePingRecords(writeCtx, (*pending)[:batchSize])
|
||||
cancel()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
*pending = (*pending)[batchSize:]
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func writeReportBatch(ctx context.Context, reports []v1.Report) ([]v1.Report, error) {
|
||||
if len(reports) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
if err := storeOperations.AcquireShared(ctx); err != nil {
|
||||
return nil, fmt.Errorf("wait for metric store operation before writing reports: %w", err)
|
||||
}
|
||||
defer storeOperations.ReleaseShared()
|
||||
|
||||
s := GetStore()
|
||||
if s == nil {
|
||||
return nil, fmt.Errorf("metric store not enabled")
|
||||
}
|
||||
|
||||
prepared := make([]v1.Report, len(reports))
|
||||
copy(prepared, reports)
|
||||
points := make([]metric.Point, 0, len(reports)*20)
|
||||
pendingStates := make(map[*reportTrafficState]reportTrafficValues)
|
||||
for i, report := range prepared {
|
||||
stateValue, _ := reportTrafficStates.LoadOrStore(report.UUID, &reportTrafficState{})
|
||||
state := stateValue.(*reportTrafficState)
|
||||
values, ok := pendingStates[state]
|
||||
if !ok {
|
||||
state.mu.Lock()
|
||||
values = state.reportTrafficValues
|
||||
state.mu.Unlock()
|
||||
}
|
||||
if !values.initialized {
|
||||
totalUp, hasUp, err := latestReportCounter(ctx, s, MetricNetTotalUp, report.UUID, report.UpdatedAt)
|
||||
if err == nil {
|
||||
values.totalUp = totalUp
|
||||
values.hasUp = hasUp
|
||||
} else if ctx.Err() == nil {
|
||||
logger.Errorf("metricstore", "failed to restore previous upload counter for %s: %v", report.UUID, err)
|
||||
}
|
||||
totalDown, hasDown, err := latestReportCounter(ctx, s, MetricNetTotalDown, report.UUID, report.UpdatedAt)
|
||||
if err == nil {
|
||||
values.totalDown = totalDown
|
||||
values.hasDown = hasDown
|
||||
} else if ctx.Err() == nil {
|
||||
logger.Errorf("metricstore", "failed to restore previous download counter for %s: %v", report.UUID, err)
|
||||
}
|
||||
if err := ctx.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
values.initialized = true
|
||||
}
|
||||
|
||||
if !values.timestamp.IsZero() && !report.UpdatedAt.After(values.timestamp) {
|
||||
report.UpdatedAt = values.timestamp.Add(time.Millisecond)
|
||||
}
|
||||
trafficUp := int64(0)
|
||||
if values.hasUp {
|
||||
trafficUp = TrafficCounterDelta(report.Network.TotalUp, values.totalUp)
|
||||
}
|
||||
trafficDown := int64(0)
|
||||
if values.hasDown {
|
||||
trafficDown = TrafficCounterDelta(report.Network.TotalDown, values.totalDown)
|
||||
}
|
||||
points = append(points, reportMetricPoints(report, trafficUp, trafficDown)...)
|
||||
values.timestamp = report.UpdatedAt
|
||||
values.hasUp = true
|
||||
values.totalUp = report.Network.TotalUp
|
||||
values.hasDown = true
|
||||
values.totalDown = report.Network.TotalDown
|
||||
pendingStates[state] = values
|
||||
prepared[i] = report
|
||||
}
|
||||
|
||||
if err := s.WriteBatch(ctx, points); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for state, values := range pendingStates {
|
||||
state.mu.Lock()
|
||||
state.reportTrafficValues = values
|
||||
state.mu.Unlock()
|
||||
}
|
||||
return prepared, nil
|
||||
}
|
||||
@@ -0,0 +1,117 @@
|
||||
package metricstore
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"strconv"
|
||||
"time"
|
||||
|
||||
"github.com/komari-monitor/komari/database/models"
|
||||
"github.com/komari-monitor/komari/pkg/metric"
|
||||
v1 "github.com/komari-monitor/komari/protocol/v1"
|
||||
)
|
||||
|
||||
func reportMetricPoints(report v1.Report, trafficUp, trafficDown int64) []metric.Point {
|
||||
entityID := report.UUID
|
||||
ts := report.UpdatedAt
|
||||
points := []metric.Point{
|
||||
{MetricName: MetricCPU, EntityID: entityID, Timestamp: ts, Value: report.CPU.Usage},
|
||||
{MetricName: MetricRAM, EntityID: entityID, Timestamp: ts, Value: float64(report.Ram.Used)},
|
||||
{MetricName: MetricSwap, EntityID: entityID, Timestamp: ts, Value: float64(report.Swap.Used)},
|
||||
{MetricName: MetricLoad, EntityID: entityID, Timestamp: ts, Value: report.Load.Load1},
|
||||
{MetricName: MetricDisk, EntityID: entityID, Timestamp: ts, Value: float64(report.Disk.Used)},
|
||||
{MetricName: MetricNetIn, EntityID: entityID, Timestamp: ts, Value: float64(report.Network.Down)},
|
||||
{MetricName: MetricNetOut, EntityID: entityID, Timestamp: ts, Value: float64(report.Network.Up)},
|
||||
{MetricName: MetricNetTotalUp, EntityID: entityID, Timestamp: ts, Value: float64(report.Network.TotalUp)},
|
||||
{MetricName: MetricNetTotalDown, EntityID: entityID, Timestamp: ts, Value: float64(report.Network.TotalDown)},
|
||||
{MetricName: MetricTrafficUp, EntityID: entityID, Timestamp: ts, Value: float64(trafficUp)},
|
||||
{MetricName: MetricTrafficDown, EntityID: entityID, Timestamp: ts, Value: float64(trafficDown)},
|
||||
{MetricName: MetricProcess, EntityID: entityID, Timestamp: ts, Value: float64(report.Process)},
|
||||
{MetricName: MetricConnections, EntityID: entityID, Timestamp: ts, Value: float64(report.Connections.TCP)},
|
||||
{MetricName: MetricConnectionsUDP, EntityID: entityID, Timestamp: ts, Value: float64(report.Connections.UDP)},
|
||||
}
|
||||
if report.GPU == nil {
|
||||
return points
|
||||
}
|
||||
points = append(points, metric.Point{MetricName: MetricGPU, EntityID: entityID, Timestamp: ts, Value: report.GPU.AverageUsage})
|
||||
for deviceIndex, gpu := range report.GPU.DetailedInfo {
|
||||
tags := map[string]string{
|
||||
"device_index": strconv.Itoa(deviceIndex),
|
||||
"device_name": gpu.Name,
|
||||
}
|
||||
points = append(points,
|
||||
metric.Point{MetricName: MetricGPUMem, EntityID: entityID, Timestamp: ts, Value: float64(gpu.MemoryUsed), Tags: tags},
|
||||
metric.Point{MetricName: MetricGPUMemTotal, EntityID: entityID, Timestamp: ts, Value: float64(gpu.MemoryTotal), Tags: tags},
|
||||
metric.Point{MetricName: MetricGPUDeviceUsage, EntityID: entityID, Timestamp: ts, Value: gpu.Utilization, Tags: tags},
|
||||
metric.Point{MetricName: MetricGPUTemp, EntityID: entityID, Timestamp: ts, Value: float64(gpu.Temperature), Tags: tags},
|
||||
)
|
||||
}
|
||||
return points
|
||||
}
|
||||
|
||||
func latestReportCounter(ctx context.Context, s *metric.Store, metricName, entityID string, before time.Time) (int64, bool, error) {
|
||||
point, ok, err := s.LatestBefore(ctx, metricName, entityID, before)
|
||||
if err != nil {
|
||||
return 0, false, err
|
||||
}
|
||||
if !ok {
|
||||
return 0, false, nil
|
||||
}
|
||||
return int64(point.Value), true, nil
|
||||
}
|
||||
|
||||
// GetLatestTrafficBefore returns the latest retained upload/download counters
|
||||
// before a boundary, transparently reading raw points or rollup summaries.
|
||||
func GetLatestTrafficBefore(ctx context.Context, entityIDs []string, before time.Time) (map[string]models.Record, error) {
|
||||
s := GetStore()
|
||||
if s == nil {
|
||||
return nil, fmt.Errorf("metric store not enabled")
|
||||
}
|
||||
result := make(map[string]models.Record, len(entityIDs))
|
||||
for _, entityID := range entityIDs {
|
||||
if entityID == "" {
|
||||
continue
|
||||
}
|
||||
up, hasUp, err := latestReportCounter(ctx, s, MetricNetTotalUp, entityID, before)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
down, hasDown, err := latestReportCounter(ctx, s, MetricNetTotalDown, entityID, before)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if !hasUp && !hasDown {
|
||||
continue
|
||||
}
|
||||
result[entityID] = models.Record{
|
||||
Client: entityID,
|
||||
Time: before.UTC().Add(-time.Nanosecond),
|
||||
NetTotalUp: up,
|
||||
NetTotalDown: down,
|
||||
}
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
// TrafficCounterDelta returns a reset-aware increase between two cumulative
|
||||
// traffic counters. After a reset, the current counter is the new increase.
|
||||
func TrafficCounterDelta(current, previous int64) int64 {
|
||||
if current < 0 || previous < 0 {
|
||||
return 0
|
||||
}
|
||||
if current >= previous {
|
||||
return current - previous
|
||||
}
|
||||
return current
|
||||
}
|
||||
|
||||
func deleteReportTrafficState(entityID string) {
|
||||
reportTrafficStates.Delete(entityID)
|
||||
}
|
||||
|
||||
func clearReportTrafficStates() {
|
||||
reportTrafficStates.Range(func(key, _ any) bool {
|
||||
reportTrafficStates.Delete(key)
|
||||
return true
|
||||
})
|
||||
}
|
||||
@@ -2,10 +2,12 @@ package metricstore
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/komari-monitor/komari/database/models"
|
||||
"github.com/komari-monitor/komari/pkg/metric"
|
||||
v1 "github.com/komari-monitor/komari/protocol/v1"
|
||||
)
|
||||
@@ -42,12 +44,12 @@ func useReportTestStore(t *testing.T, policy *metric.RollupPolicy) *metric.Store
|
||||
return s
|
||||
}
|
||||
|
||||
func TestWriteReportStoresRawMetricsAndResetAwareTraffic(t *testing.T) {
|
||||
func TestWriteReportStoresMinuteMetricsAndResetAwareTraffic(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
policy := defaultRollupPolicy()
|
||||
s := useReportTestStore(t, &policy)
|
||||
now := time.Now().UTC().Truncate(time.Minute)
|
||||
base := now.Add(-30 * time.Minute)
|
||||
base := time.Now().UTC().Truncate(time.Minute).Add(5 * time.Second)
|
||||
now := base.Add(45 * time.Second)
|
||||
|
||||
report := v1.Report{
|
||||
UUID: "node-a",
|
||||
@@ -88,6 +90,8 @@ func TestWriteReportStoresRawMetricsAndResetAwareTraffic(t *testing.T) {
|
||||
assertMetricValues(t, s, MetricTrafficUp, report.UUID, base.Add(-time.Second), base.Add(time.Minute), []float64{0, 50, 20})
|
||||
assertMetricValues(t, s, MetricTrafficDown, report.UUID, base.Add(-time.Second), base.Add(time.Minute), []float64{0, 60, 30})
|
||||
assertMetricValues(t, s, MetricNetTotalUp, report.UUID, base.Add(-time.Second), base.Add(time.Minute), []float64{100, 150, 20})
|
||||
assertMetricAggregate(t, s, MetricTrafficUp, report.UUID, base.Add(-time.Second), base.Add(time.Minute), metric.AggSum, 70, 3)
|
||||
assertMetricAggregate(t, s, MetricTrafficDown, report.UUID, base.Add(-time.Second), base.Add(time.Minute), metric.AggSum, 90, 3)
|
||||
|
||||
gpuPoints, err := s.Query(ctx, metric.Query{
|
||||
MetricName: MetricGPUDeviceUsage,
|
||||
@@ -100,7 +104,7 @@ func TestWriteReportStoresRawMetricsAndResetAwareTraffic(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatalf("query GPU points: %v", err)
|
||||
}
|
||||
if len(gpuPoints) != 3 || gpuPoints[0].Timestamp != base || gpuPoints[0].Tags["device_name"] != "GPU 0" {
|
||||
if len(gpuPoints) != 3 || !gpuPoints[2].Timestamp.Equal(base.Add(6*time.Second)) || gpuPoints[2].Tags["device_name"] != "GPU 0" {
|
||||
t.Fatalf("unexpected GPU points: %#v", gpuPoints)
|
||||
}
|
||||
|
||||
@@ -118,6 +122,27 @@ func TestWriteReportStoresRawMetricsAndResetAwareTraffic(t *testing.T) {
|
||||
assertMetricValues(t, s, MetricTrafficDown, report.UUID, now.Add(-time.Second), now.Add(time.Second), []float64{20})
|
||||
}
|
||||
|
||||
func TestWriteReportSkipsMetricsWithoutAgentData(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
s := useReportTestStore(t, nil)
|
||||
timestamp := time.Now().UTC()
|
||||
if _, err := WriteReport(ctx, v1.Report{
|
||||
UUID: "node-without-gpu", UpdatedAt: timestamp,
|
||||
}); err != nil {
|
||||
t.Fatalf("write report: %v", err)
|
||||
}
|
||||
points, err := s.Query(ctx, metric.Query{
|
||||
MetricName: MetricGPU, EntityID: "node-without-gpu",
|
||||
Start: timestamp.Add(-time.Second), End: timestamp.Add(time.Second),
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("query GPU metric: %v", err)
|
||||
}
|
||||
if len(points) != 0 {
|
||||
t.Fatalf("GPU metric was written without GPU data: %#v", points)
|
||||
}
|
||||
}
|
||||
|
||||
func TestReportBatcherFlushesQueuedReports(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
s := useReportTestStore(t, nil)
|
||||
@@ -128,7 +153,7 @@ func TestReportBatcherFlushesQueuedReports(t *testing.T) {
|
||||
}
|
||||
})
|
||||
|
||||
base := time.Now().UTC().Truncate(time.Second)
|
||||
base := time.Now().UTC().Truncate(time.Minute).Add(10 * time.Second)
|
||||
first := v1.Report{
|
||||
UUID: "batched-node",
|
||||
UpdatedAt: base,
|
||||
@@ -167,6 +192,130 @@ func TestReportBatcherFlushesQueuedReports(t *testing.T) {
|
||||
assertMetricValues(t, s, MetricCPU, first.UUID, base.Add(-time.Second), base.Add(time.Minute), []float64{10, 20})
|
||||
assertMetricValues(t, s, MetricTrafficUp, first.UUID, base.Add(-time.Second), base.Add(time.Minute), []float64{0, 50})
|
||||
assertMetricValues(t, s, MetricTrafficDown, first.UUID, base.Add(-time.Second), base.Add(time.Minute), []float64{0, 60})
|
||||
assertMetricAggregate(t, s, MetricCPU, first.UUID, base.Add(-time.Second), base.Add(time.Minute), metric.AggAvg, 15, 2)
|
||||
}
|
||||
|
||||
func TestPingBatcherFlushesLatencyAndLossTogether(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
s := useReportTestStore(t, nil)
|
||||
StartReportBatcher()
|
||||
t.Cleanup(func() {
|
||||
if err := StopReportBatcher(ctx); err != nil {
|
||||
t.Errorf("stop report batcher: %v", err)
|
||||
}
|
||||
})
|
||||
|
||||
base := time.Now().UTC().Truncate(time.Second)
|
||||
records := []models.PingRecord{
|
||||
{Client: "ping-node", TaskId: 7, Time: base, Value: 24},
|
||||
{Client: "ping-node", TaskId: 7, Time: base.Add(time.Minute), Value: -1},
|
||||
{Client: "ping-node", TaskId: 8, Time: base, Value: 31},
|
||||
}
|
||||
for _, record := range records {
|
||||
if err := WritePingRecord(ctx, record); err != nil {
|
||||
t.Fatalf("queue ping record: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
for _, name := range []string{MetricPingLatency, MetricPingLoss} {
|
||||
points, err := s.Query(ctx, metric.Query{
|
||||
MetricName: name,
|
||||
EntityID: "ping-node",
|
||||
Start: base.Add(-time.Second),
|
||||
End: base.Add(2 * time.Minute),
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("query queued %s points: %v", name, err)
|
||||
}
|
||||
if len(points) != 0 {
|
||||
t.Fatalf("queued %s points were written before flush: %#v", name, points)
|
||||
}
|
||||
}
|
||||
|
||||
if err := FlushReportBatch(ctx); err != nil {
|
||||
t.Fatalf("flush ping batch: %v", err)
|
||||
}
|
||||
|
||||
latency, err := s.Query(ctx, metric.Query{
|
||||
MetricName: MetricPingLatency,
|
||||
EntityID: "ping-node",
|
||||
Tags: map[string]string{"task_id": "7"},
|
||||
Start: base.Add(-time.Second),
|
||||
End: base.Add(2 * time.Minute),
|
||||
Order: metric.OrderAsc,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("query flushed latency points: %v", err)
|
||||
}
|
||||
if len(latency) != 2 || latency[0].Value != 24 || latency[1].Value != -1 {
|
||||
t.Fatalf("latency points = %#v, want both original samples", latency)
|
||||
}
|
||||
|
||||
loss, err := s.Query(ctx, metric.Query{
|
||||
MetricName: MetricPingLoss,
|
||||
EntityID: "ping-node",
|
||||
Tags: map[string]string{"task_id": "7"},
|
||||
Start: base.Add(-time.Second),
|
||||
End: base.Add(2 * time.Minute),
|
||||
Order: metric.OrderAsc,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("query flushed loss points: %v", err)
|
||||
}
|
||||
if len(loss) != 2 || loss[0].Value != 0 || loss[1].Value != 1 {
|
||||
t.Fatalf("loss points = %#v, want success and loss samples", loss)
|
||||
}
|
||||
}
|
||||
|
||||
func TestReportBatchKeepsEverySample(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
s := useReportTestStore(t, nil)
|
||||
base := time.Now().UTC().Truncate(time.Second)
|
||||
pending := []v1.Report{
|
||||
{UUID: "node-a", UpdatedAt: base, CPU: v1.CPUReport{Usage: 10}, Network: v1.NetworkReport{TotalUp: 100}},
|
||||
{UUID: "node-a", UpdatedAt: base, CPU: v1.CPUReport{Usage: 20}, Network: v1.NetworkReport{TotalUp: 150}},
|
||||
{UUID: "node-b", UpdatedAt: base, CPU: v1.CPUReport{Usage: 30}, Network: v1.NetworkReport{TotalUp: 200}},
|
||||
{UUID: "node-b", UpdatedAt: base.Add(time.Second), CPU: v1.CPUReport{Usage: 40}, Network: v1.NetworkReport{TotalUp: 260}},
|
||||
}
|
||||
|
||||
if err := writePendingReports(ctx, &pending); err != nil {
|
||||
t.Fatalf("write report batch: %v", err)
|
||||
}
|
||||
if len(pending) != 0 {
|
||||
t.Fatalf("pending reports = %d, want 0", len(pending))
|
||||
}
|
||||
assertMetricValues(t, s, MetricCPU, "node-a", base.Add(-time.Second), base.Add(time.Minute), []float64{10, 20})
|
||||
assertMetricValues(t, s, MetricCPU, "node-b", base.Add(-time.Second), base.Add(time.Minute), []float64{30, 40})
|
||||
}
|
||||
|
||||
func TestReportQueueFullReturnsError(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
useReportTestStore(t, nil)
|
||||
worker := &reportBatchWorker{
|
||||
queue: make(chan v1.Report, 1),
|
||||
requests: make(chan reportBatchRequest, 1),
|
||||
done: make(chan struct{}),
|
||||
}
|
||||
worker.queue <- v1.Report{UUID: "already-queued"}
|
||||
reportBatcherMu.Lock()
|
||||
reportBatcher = worker
|
||||
reportBatcherMu.Unlock()
|
||||
t.Cleanup(func() {
|
||||
reportBatcherMu.Lock()
|
||||
if reportBatcher == worker {
|
||||
reportBatcher = nil
|
||||
}
|
||||
reportBatcherMu.Unlock()
|
||||
})
|
||||
|
||||
report := v1.Report{
|
||||
UUID: "realtime-node",
|
||||
UpdatedAt: time.Now().UTC(),
|
||||
}
|
||||
_, err := WriteReport(ctx, report)
|
||||
if !errors.Is(err, ErrReportBatchQueueFull) {
|
||||
t.Fatalf("queue-full error = %v, want %v", err, ErrReportBatchQueueFull)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRecordReconstructionUsesMetricSpecificAggregation(t *testing.T) {
|
||||
@@ -225,7 +374,7 @@ func TestWriteReportNormalizesReceiveTimeToUTC(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
s := useReportTestStore(t, nil)
|
||||
local := time.FixedZone("UTC+8", 8*60*60)
|
||||
receiveTime := time.Date(2026, 7, 17, 9, 30, 0, 123456789, local)
|
||||
receiveTime := time.Now().In(local).Add(-10 * time.Second)
|
||||
report := v1.Report{
|
||||
UUID: "utc-report",
|
||||
UpdatedAt: receiveTime,
|
||||
@@ -249,8 +398,8 @@ func TestWriteReportNormalizesReceiveTimeToUTC(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatalf("query stored point: %v", err)
|
||||
}
|
||||
if len(points) != 1 || points[0].Timestamp.Location() != time.UTC || points[0].Timestamp.Nanosecond() != 123456789 {
|
||||
t.Fatalf("stored points = %#v, want one UTC nanosecond point", points)
|
||||
if len(points) != 1 || points[0].Timestamp.Location() != time.UTC || points[0].Timestamp.UnixMilli() != receiveTime.UnixMilli() {
|
||||
t.Fatalf("stored points = %#v, want one UTC millisecond point", points)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -275,3 +424,17 @@ func assertMetricValues(t *testing.T, s *metric.Store, metricName, entityID stri
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func assertMetricAggregate(t *testing.T, s *metric.Store, metricName, entityID string, start, end time.Time, aggregation metric.Aggregation, want float64, wantCount int) {
|
||||
t.Helper()
|
||||
points, err := s.Series(context.Background(), metric.AggregateQuery{
|
||||
Query: metric.Query{MetricName: metricName, EntityID: entityID, Start: start, End: end},
|
||||
Aggregation: aggregation, Interval: time.Minute, PreserveSeries: true,
|
||||
}, end)
|
||||
if err != nil {
|
||||
t.Fatalf("aggregate %s: %v", metricName, err)
|
||||
}
|
||||
if len(points) != 1 || points[0].Value != want || points[0].Count != wantCount {
|
||||
t.Fatalf("aggregate %s = %#v, want value=%v count=%d", metricName, points, want, wantCount)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,137 @@
|
||||
package metricstore
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
|
||||
"github.com/komari-monitor/komari/internal/config"
|
||||
"github.com/komari-monitor/komari/pkg/metric"
|
||||
)
|
||||
|
||||
// RestructureProgress is the authenticated guide's stable progress payload.
|
||||
type RestructureProgress struct {
|
||||
Phase string
|
||||
CurrentMetric string
|
||||
RowsDone int64
|
||||
RowsTotal int64
|
||||
MetricsDone int
|
||||
MetricsTotal int
|
||||
}
|
||||
|
||||
type RestructureResult struct {
|
||||
BeforeBytes int64
|
||||
AfterBytes int64
|
||||
RowsCopied int64
|
||||
Metrics int
|
||||
}
|
||||
|
||||
// StructureUpgradeRequired checks the configured metric store without creating
|
||||
// tables. It is called before normal metric-store initialization so an existing
|
||||
// installation is never rebuilt at startup without an administrator action.
|
||||
func StructureUpgradeRequired(ctx context.Context) (bool, error) {
|
||||
cfg, err := config.GetManyAs[MetricStoreConfig]()
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
return structureUpgradeRequiredForConfig(ctx, cfg)
|
||||
}
|
||||
|
||||
func structureUpgradeRequiredForConfig(ctx context.Context, cfg *MetricStoreConfig) (bool, error) {
|
||||
metricCfg, err := buildMetricConfig(cfg, false)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
store, err := metric.Open(ctx, metricCfg)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
defer store.Close()
|
||||
return store.NeedsRestructure(ctx)
|
||||
}
|
||||
|
||||
// RestructureConfiguredStore performs the explicit data copy, table swap and
|
||||
// one physical reclaim pass used by the upgrade guide.
|
||||
func RestructureConfiguredStore(ctx context.Context, report func(RestructureProgress)) (RestructureResult, error) {
|
||||
cfg, err := config.GetManyAs[MetricStoreConfig]()
|
||||
if err != nil {
|
||||
return RestructureResult{}, err
|
||||
}
|
||||
metricCfg, err := buildMetricConfig(cfg, false)
|
||||
if err != nil {
|
||||
return RestructureResult{}, err
|
||||
}
|
||||
store, err := metric.Open(ctx, metricCfg)
|
||||
if err != nil {
|
||||
return RestructureResult{}, err
|
||||
}
|
||||
defer store.Close()
|
||||
|
||||
before, err := store.LegacyStorageSize(ctx)
|
||||
if err != nil {
|
||||
return RestructureResult{}, fmt.Errorf("measure legacy metric storage: %w", err)
|
||||
}
|
||||
result, err := store.Restructure(ctx, func(progress metric.RestructureProgress) {
|
||||
if report == nil {
|
||||
return
|
||||
}
|
||||
report(RestructureProgress{Phase: progress.Phase, CurrentMetric: progress.Current, RowsDone: progress.RowsDone, RowsTotal: progress.RowsTotal, MetricsDone: progress.MetricsDone, MetricsTotal: progress.MetricsTotal})
|
||||
})
|
||||
if err != nil {
|
||||
return RestructureResult{}, err
|
||||
}
|
||||
if report != nil {
|
||||
report(RestructureProgress{Phase: "reclaiming", RowsDone: result.RowsCopied, RowsTotal: result.RowsCopied, MetricsDone: result.Metrics, MetricsTotal: result.Metrics})
|
||||
}
|
||||
if err := store.ReclaimSpace(ctx); err != nil {
|
||||
return RestructureResult{}, fmt.Errorf("reclaim metric storage: %w", err)
|
||||
}
|
||||
after, err := store.StorageSize(ctx)
|
||||
if err != nil {
|
||||
return RestructureResult{}, fmt.Errorf("measure rebuilt metric storage: %w", err)
|
||||
}
|
||||
return RestructureResult{BeforeBytes: before, AfterBytes: after, RowsCopied: result.RowsCopied, Metrics: result.Metrics}, nil
|
||||
}
|
||||
|
||||
// DiscardConfiguredStoreHistory completes the same schema upgrade as
|
||||
// RestructureConfiguredStore but drops every historical sample instead of
|
||||
// copying it. Metric definitions remain available for new ingest.
|
||||
func DiscardConfiguredStoreHistory(ctx context.Context, report func(RestructureProgress)) (RestructureResult, error) {
|
||||
cfg, err := config.GetManyAs[MetricStoreConfig]()
|
||||
if err != nil {
|
||||
return RestructureResult{}, err
|
||||
}
|
||||
metricCfg, err := buildMetricConfig(cfg, false)
|
||||
if err != nil {
|
||||
return RestructureResult{}, err
|
||||
}
|
||||
store, err := metric.Open(ctx, metricCfg)
|
||||
if err != nil {
|
||||
return RestructureResult{}, err
|
||||
}
|
||||
defer store.Close()
|
||||
|
||||
before, err := store.LegacyStorageSize(ctx)
|
||||
if err != nil {
|
||||
return RestructureResult{}, fmt.Errorf("measure legacy metric storage: %w", err)
|
||||
}
|
||||
result, err := store.DiscardHistory(ctx, func(progress metric.RestructureProgress) {
|
||||
if report == nil {
|
||||
return
|
||||
}
|
||||
report(RestructureProgress{Phase: progress.Phase, CurrentMetric: progress.Current, RowsDone: progress.RowsDone, RowsTotal: progress.RowsTotal, MetricsDone: progress.MetricsDone, MetricsTotal: progress.MetricsTotal})
|
||||
})
|
||||
if err != nil {
|
||||
return RestructureResult{}, err
|
||||
}
|
||||
if report != nil {
|
||||
report(RestructureProgress{Phase: "reclaiming", RowsDone: result.RowsCopied, RowsTotal: result.RowsCopied, MetricsDone: result.Metrics, MetricsTotal: result.Metrics})
|
||||
}
|
||||
if err := store.ReclaimSpace(ctx); err != nil {
|
||||
return RestructureResult{}, fmt.Errorf("reclaim metric storage: %w", err)
|
||||
}
|
||||
after, err := store.StorageSize(ctx)
|
||||
if err != nil {
|
||||
return RestructureResult{}, fmt.Errorf("measure rebuilt metric storage: %w", err)
|
||||
}
|
||||
return RestructureResult{BeforeBytes: before, AfterBytes: after, RowsCopied: result.RowsCopied, Metrics: result.Metrics}, nil
|
||||
}
|
||||
@@ -0,0 +1,290 @@
|
||||
package metricstore
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
logger "github.com/komari-monitor/komari/utils/log"
|
||||
|
||||
"github.com/komari-monitor/komari/internal/config"
|
||||
"github.com/komari-monitor/komari/pkg/metric"
|
||||
)
|
||||
|
||||
var (
|
||||
store *metric.Store
|
||||
storeFingerprint string
|
||||
storeMu sync.RWMutex
|
||||
storeInitMu sync.Mutex
|
||||
storeOperations = newStoreOperationGate()
|
||||
compactOperations = newStoreOperationGate()
|
||||
)
|
||||
|
||||
var ErrCompactInProgress = errors.New("metric store compact already in progress")
|
||||
|
||||
// ErrStructureUpgradeRequired reports that the configured store must be
|
||||
// migrated by the restricted startup guide before it can be opened normally.
|
||||
var ErrStructureUpgradeRequired = errors.New("metric store structure upgrade is required")
|
||||
|
||||
// openStore 按配置打开 metric store 并创建指标定义。
|
||||
func openStore(ctx context.Context, cfg *MetricStoreConfig) (*metric.Store, error) {
|
||||
return openStoreWithDefaultRetention(ctx, cfg, defaultBuiltinMetricRetentionDays)
|
||||
}
|
||||
|
||||
func openStoreWithDefaultRetention(ctx context.Context, cfg *MetricStoreConfig, defaultRetentionDays int) (*metric.Store, error) {
|
||||
metricCfg, err := buildMetricConfig(cfg, true)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
s, err := metric.Open(ctx, metricCfg)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to open metric store: %w", err)
|
||||
}
|
||||
|
||||
if err := createMetricDefinitionsWithDefaultRetention(ctx, s, defaultRetentionDays); err != nil {
|
||||
s.Close()
|
||||
return nil, fmt.Errorf("failed to create metric definitions: %w", err)
|
||||
}
|
||||
|
||||
return s, nil
|
||||
}
|
||||
|
||||
// OpenStore opens an isolated metric store using the supplied configuration.
|
||||
// It is used by the pre-start upgrade flow before the process-wide store is
|
||||
// initialized. The caller owns the returned store and must close it.
|
||||
// func OpenStore(ctx context.Context, cfg *MetricStoreConfig) (*metric.Store, error) {
|
||||
// return openStore(ctx, cfg)
|
||||
// }
|
||||
|
||||
// OpenStoreForMigration opens an isolated target and uses the legacy data span
|
||||
// as the initial retention for definitions that do not exist yet. Existing
|
||||
// definitions keep their configured retention, including an explicit zero.
|
||||
func OpenStoreForMigration(ctx context.Context, cfg *MetricStoreConfig, legacyRetentionDays int) (*metric.Store, error) {
|
||||
if legacyRetentionDays < defaultBuiltinMetricRetentionDays {
|
||||
legacyRetentionDays = defaultBuiltinMetricRetentionDays
|
||||
}
|
||||
return openStoreWithDefaultRetention(ctx, cfg, legacyRetentionDays)
|
||||
}
|
||||
|
||||
// TestConnection 使用给定配置尝试连接 metrics 数据库(不影响当前运行的 store)。
|
||||
// 仅打开连接并 Ping,不执行自动建表,连接成功后立即关闭。失败时返回可读错误。
|
||||
func TestConnection(ctx context.Context, cfg *MetricStoreConfig) error {
|
||||
metricCfg, err := buildMetricConfig(cfg, false)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
s, err := metric.Open(ctx, metricCfg)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer s.Close()
|
||||
|
||||
return s.Ping(ctx)
|
||||
}
|
||||
|
||||
// InitializeStore 初始化 metric store(启动时调用,可在失败后重试)。
|
||||
func InitializeStore() error {
|
||||
storeInitMu.Lock()
|
||||
defer storeInitMu.Unlock()
|
||||
|
||||
// A previous failed connection must remain retryable. The old sync.Once
|
||||
// implementation consumed the one-time call on failure and returned nil on
|
||||
// every later call while the store was still nil.
|
||||
storeMu.RLock()
|
||||
initialized := store != nil
|
||||
storeMu.RUnlock()
|
||||
if initialized {
|
||||
return nil
|
||||
}
|
||||
|
||||
cfg, err := config.GetManyAs[MetricStoreConfig]()
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to load metric store config: %w", err)
|
||||
}
|
||||
|
||||
// metric store 始终启用;未配置时默认 SQLite(./data/metrics.db)。
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||||
defer cancel()
|
||||
|
||||
s, err := openStore(ctx, cfg)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
storeMu.Lock()
|
||||
store = s
|
||||
storeFingerprint = targetFingerprint(cfg)
|
||||
storeMu.Unlock()
|
||||
clearStoreClosing()
|
||||
|
||||
logger.Infof("metricstore", "Metric store initialized successfully (driver=%s)", ResolveDriverFromConfig(cfg.Driver, cfg.DSN))
|
||||
return nil
|
||||
}
|
||||
|
||||
// RecoverStore opens, persists, and activates a replacement store selected
|
||||
// from the restricted recovery page. It records that target as the current
|
||||
// manual-migration source, but does not copy data from the unavailable store.
|
||||
func RecoverStore(ctx context.Context, cfg *MetricStoreConfig) error {
|
||||
if cfg == nil {
|
||||
return fmt.Errorf("metric store recovery config is nil")
|
||||
}
|
||||
|
||||
storeInitMu.Lock()
|
||||
defer storeInitMu.Unlock()
|
||||
if err := storeOperations.Acquire(ctx); err != nil {
|
||||
return fmt.Errorf("wait for metric store operations before recovery: %w", err)
|
||||
}
|
||||
defer storeOperations.Release()
|
||||
if isStoreClosing() {
|
||||
return ErrStoreBusy
|
||||
}
|
||||
|
||||
recovered := *cfg
|
||||
recovered.DSN = strings.TrimSpace(recovered.DSN)
|
||||
recovered.Driver = string(ResolveDriverFromConfig(recovered.Driver, recovered.DSN))
|
||||
restructureRequired, err := structureUpgradeRequiredForConfig(ctx, &recovered)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if restructureRequired {
|
||||
target := targetFingerprint(&recovered)
|
||||
if err := config.SetMany(map[string]any{
|
||||
MetricDBDriverKey: recovered.Driver,
|
||||
MetricDBDSNKey: recovered.DSN,
|
||||
MigrationTargetKey: target,
|
||||
}); err != nil {
|
||||
return fmt.Errorf("save recovered metric store config: %w", err)
|
||||
}
|
||||
logger.Infof("metricstore", "Metric store connection recovered; structure upgrade is required (driver=%s)", recovered.Driver)
|
||||
return nil
|
||||
}
|
||||
s, err := openStore(ctx, &recovered)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
target := targetFingerprint(&recovered)
|
||||
if err := config.SetMany(map[string]any{
|
||||
MetricDBDriverKey: recovered.Driver,
|
||||
MetricDBDSNKey: recovered.DSN,
|
||||
MigrationTargetKey: target,
|
||||
}); err != nil {
|
||||
_ = s.Close()
|
||||
return fmt.Errorf("save recovered metric store config: %w", err)
|
||||
}
|
||||
|
||||
storeMu.Lock()
|
||||
old := store
|
||||
store = s
|
||||
storeFingerprint = target
|
||||
storeMu.Unlock()
|
||||
clearStoreClosing()
|
||||
|
||||
if old != nil {
|
||||
if closeErr := old.Close(); closeErr != nil {
|
||||
logger.Errorf("metricstore", "Failed to close previous metric store during recovery: %v", closeErr)
|
||||
}
|
||||
}
|
||||
logger.Infof("metricstore", "Metric store recovered successfully (driver=%s)", recovered.Driver)
|
||||
return nil
|
||||
}
|
||||
|
||||
// Reload 根据最新配置热重载 metric store,无需重启进程。
|
||||
// metric store 始终启用:用新配置打开并建表(内部已 Ping 校验连接),
|
||||
// 成功后再替换运行中的 store,最后关闭旧实例。任何失败都会保留旧 store 不变。
|
||||
//
|
||||
// 注意:Reload 只切换运行中的连接,不会把旧目标(如 SQLite)中的历史数据
|
||||
// 搬运到新目标(如 MySQL/PostgreSQL)。跨库数据迁移必须由管理员手动启动。
|
||||
func Reload(ctx context.Context) error {
|
||||
if err := storeOperations.Acquire(ctx); err != nil {
|
||||
return fmt.Errorf("wait for metric store operations before reload: %w", err)
|
||||
}
|
||||
defer storeOperations.Release()
|
||||
if isStoreClosing() {
|
||||
return ErrStoreBusy
|
||||
}
|
||||
|
||||
cfg, err := config.GetManyAs[MetricStoreConfig]()
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to load metric store config: %w", err)
|
||||
}
|
||||
|
||||
// Do not let the normal hot-reload path run AutoMigrate against an old
|
||||
// point-backed schema. That schema must go through the authenticated
|
||||
// structure-upgrade guide first; otherwise index creation can reference
|
||||
// columns such as resolution_id that do not exist yet.
|
||||
checkCfg, err := buildMetricConfig(cfg, false)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
checkStore, err := metric.Open(ctx, checkCfg)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
needsRestructure, err := checkStore.NeedsRestructure(ctx)
|
||||
_ = checkStore.Close()
|
||||
if err != nil {
|
||||
return fmt.Errorf("inspect metric store structure: %w", err)
|
||||
}
|
||||
if needsRestructure {
|
||||
return fmt.Errorf("%w before hot reload", ErrStructureUpgradeRequired)
|
||||
}
|
||||
|
||||
// 用新配置打开并建表(内部已 Ping 校验连接)。
|
||||
s, err := openStore(ctx, cfg)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
storeMu.Lock()
|
||||
old := store
|
||||
store = s
|
||||
storeFingerprint = targetFingerprint(cfg)
|
||||
storeMu.Unlock()
|
||||
if old != nil {
|
||||
if cerr := old.Close(); cerr != nil {
|
||||
logger.Errorf("metricstore", "Failed to close previous metric store on reload: %v", cerr)
|
||||
}
|
||||
}
|
||||
|
||||
logger.Infof("metricstore", "Metric store reloaded successfully (driver=%s)", ResolveDriverFromConfig(cfg.Driver, cfg.DSN))
|
||||
return nil
|
||||
}
|
||||
|
||||
// GetStore 获取 metric store 实例(如果未启用返回 nil)
|
||||
func GetStore() *metric.Store {
|
||||
storeMu.RLock()
|
||||
defer storeMu.RUnlock()
|
||||
return store
|
||||
}
|
||||
|
||||
// CloseStoreContext stops the asynchronous store migration before taking the
|
||||
// store write lock, so shutdown cannot wait forever on the migration's lease.
|
||||
func CloseStoreContext(ctx context.Context) error {
|
||||
if err := stopStoreMigrationForClose(ctx); err != nil {
|
||||
clearStoreClosing()
|
||||
return err
|
||||
}
|
||||
if err := storeOperations.Acquire(ctx); err != nil {
|
||||
clearStoreClosing()
|
||||
return fmt.Errorf("wait for metric store operations before close: %w", err)
|
||||
}
|
||||
defer storeOperations.Release()
|
||||
|
||||
storeMu.Lock()
|
||||
defer storeMu.Unlock()
|
||||
|
||||
if store != nil {
|
||||
err := store.Close()
|
||||
store = nil
|
||||
storeFingerprint = ""
|
||||
return err
|
||||
}
|
||||
storeFingerprint = ""
|
||||
return nil
|
||||
}
|
||||
@@ -3,12 +3,13 @@ package metricstore
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"log"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/komari-monitor/komari/pkg/config"
|
||||
logger "github.com/komari-monitor/komari/utils/log"
|
||||
|
||||
"github.com/komari-monitor/komari/internal/config"
|
||||
"github.com/komari-monitor/komari/pkg/metric"
|
||||
)
|
||||
|
||||
@@ -18,18 +19,15 @@ import (
|
||||
// ./data/metrics.db)中的全部指标数据搬运到当前正在运行的 metrics 目标库
|
||||
// (通常是用户新配置并热重载后的 MySQL/PostgreSQL)。
|
||||
//
|
||||
// 与启动迁移(RunStartupMigration)的区别:
|
||||
// - 启动迁移在进程启动时自动、同步执行(komari.db → 当前 metrics 目标,或上一个
|
||||
// metrics 目标 → 当前目标)。
|
||||
// - 此处的 store 迁移由管理员在 WebUI 中「切换数据库后」手动触发,异步执行并可
|
||||
// 查询进度 / 取消。数据以 upsert 写入,可安全重复执行。
|
||||
// 此迁移仅能由管理员在 WebUI 中「切换数据库后」手动触发,异步执行并可查询进度 /
|
||||
// 取消。服务启动和指标库热重载均不会复制历史数据。数据以 upsert 写入,可安全重复执行。
|
||||
//
|
||||
// 典型流程:
|
||||
// 1. 管理员在设置中把 metric_db_dsn 改为 MySQL/PostgreSQL,保存 → 后端连接测试
|
||||
// 通过后热重载,当前 store 切到远端(此时远端为空)。
|
||||
// 2. 管理员调用 startMetricMigration(不带 source 参数),系统以上一个 metrics
|
||||
// 目标(默认 SQLite metrics.db)为源,把历史数据搬运到远端。
|
||||
// 3. 完成后登记目标指纹,下次启动不再重复搬运。
|
||||
// 3. 完成后登记目标指纹,供下一次手动迁移默认选择源库。
|
||||
|
||||
// StoreMigrationProgress 描述一次 store-to-store 迁移的实时进度。
|
||||
type StoreMigrationProgress struct {
|
||||
@@ -191,19 +189,19 @@ func runStoreMigration(ctx context.Context, cancel context.CancelFunc, done chan
|
||||
if err != nil {
|
||||
if ctx.Err() != nil {
|
||||
finishStoreMigration("canceled", nil)
|
||||
log.Printf("[store-migration] canceled after %d points", total)
|
||||
logger.Warnf("metricstore", "[store-migration] canceled after %d points", total)
|
||||
return
|
||||
}
|
||||
finishStoreMigration("failed", err)
|
||||
return
|
||||
}
|
||||
|
||||
// 搬运成功:登记当前目标指纹,避免下次启动重复搬运。
|
||||
// 搬运成功:登记当前目标指纹,供下一次手动迁移推断源库。
|
||||
if err := config.Set(MigrationTargetKey, targetFP); err != nil {
|
||||
log.Printf("[store-migration] failed to persist migration target fingerprint: %v", err)
|
||||
logger.Errorf("metricstore", "[store-migration] failed to persist migration target fingerprint: %v", err)
|
||||
}
|
||||
finishStoreMigration("completed", nil)
|
||||
log.Printf("[store-migration] completed via API (%d points) target_driver=%s", total, ResolveDriverFromConfig(cfg.Driver, cfg.DSN))
|
||||
logger.Infof("metricstore", "[store-migration] completed via API (%d points) target_driver=%s", total, ResolveDriverFromConfig(cfg.Driver, cfg.DSN))
|
||||
}
|
||||
|
||||
// finishStoreMigration 统一收尾:设置终态状态、结束时间与错误信息。
|
||||
@@ -270,6 +268,12 @@ func stopStoreMigrationForClose(ctx context.Context) error {
|
||||
}
|
||||
}
|
||||
|
||||
// StopStoreMigrationForShutdown prevents a new store migration and waits for
|
||||
// an active one to release its exclusive lease before report batches flush.
|
||||
func StopStoreMigrationForShutdown(ctx context.Context) error {
|
||||
return stopStoreMigrationForClose(ctx)
|
||||
}
|
||||
|
||||
func clearStoreClosing() {
|
||||
storeMigMu.Lock()
|
||||
storeClosing = false
|
||||
File renamed without changes.
@@ -2,14 +2,18 @@ package migrations
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"log"
|
||||
logger "github.com/komari-monitor/komari/utils/log"
|
||||
"math"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/komari-monitor/komari/database/metricstore"
|
||||
"github.com/komari-monitor/komari/internal/metricstore"
|
||||
"github.com/komari-monitor/komari/database/models"
|
||||
appconfig "github.com/komari-monitor/komari/pkg/config"
|
||||
appconfig "github.com/komari-monitor/komari/internal/config"
|
||||
"github.com/komari-monitor/komari/pkg/metric"
|
||||
"gorm.io/gorm"
|
||||
)
|
||||
@@ -17,25 +21,20 @@ import (
|
||||
const (
|
||||
legacyMonitoringMigrationDoneKey = "migration_legacy_monitoring_to_metric_store_done"
|
||||
legacyMonitoringBatchSize = 500
|
||||
legacyMonitoringInterval = time.Hour
|
||||
)
|
||||
|
||||
var legacyMonitoringTables = []string{"records", "records_long_term", "gpu_records", "ping_records"}
|
||||
|
||||
type MetricContext struct {
|
||||
DB *gorm.DB
|
||||
Store *metric.Store
|
||||
}
|
||||
|
||||
type LegacyMonitoringStats struct {
|
||||
Records int64
|
||||
GPU int64
|
||||
Ping int64
|
||||
}
|
||||
|
||||
// LegacyMonitoringSummary describes the legacy source data shown by the
|
||||
// pre-start upgrade wizard. Row counts deliberately stay separate from the
|
||||
// estimated metric point count because one legacy row expands into multiple
|
||||
// metric points.
|
||||
// LegacyMonitoringSummary describes the full legacy source data shown by the
|
||||
// pre-start migration guide. Every source row participates in the migration;
|
||||
// EstimatedPoints is the number of hourly P95 points expected after grouping.
|
||||
type LegacyMonitoringSummary struct {
|
||||
LoadRows int64 `json:"load_rows"`
|
||||
GPURows int64 `json:"gpu_rows"`
|
||||
@@ -56,12 +55,6 @@ type LegacyMonitoringProgress struct {
|
||||
WrittenPoints int64 `json:"written_points"`
|
||||
}
|
||||
|
||||
type LegacyMonitoringDeleted struct {
|
||||
LoadRows int64 `json:"load_rows"`
|
||||
GPURows int64 `json:"gpu_rows"`
|
||||
LatencyRows int64 `json:"latency_rows"`
|
||||
}
|
||||
|
||||
type legacyMonitoringBounds struct {
|
||||
Oldest string `gorm:"column:oldest"`
|
||||
Newest string `gorm:"column:newest"`
|
||||
@@ -69,45 +62,23 @@ type legacyMonitoringBounds struct {
|
||||
|
||||
type legacyBatchProgress func(table string, rows, points int64)
|
||||
|
||||
type legacyRecordRow struct {
|
||||
RowID int64 `gorm:"column:row_id"`
|
||||
models.Record
|
||||
type legacyP95Group struct {
|
||||
metricName string
|
||||
entityID string
|
||||
tags map[string]string
|
||||
values []float64
|
||||
}
|
||||
|
||||
type legacyGPURecordRow struct {
|
||||
RowID int64 `gorm:"column:row_id"`
|
||||
models.GPURecord
|
||||
}
|
||||
|
||||
type legacyPingRecordRow struct {
|
||||
RowID int64 `gorm:"column:row_id"`
|
||||
models.PingRecord
|
||||
}
|
||||
|
||||
// RunMetricStoreMigrations executes one-shot migrations that need the metric store
|
||||
// to be initialized. These cannot run inside Run because Run executes before the
|
||||
// metric store is opened.
|
||||
func RunMetricStoreMigrations(ctx MetricContext) error {
|
||||
done, err := appconfig.GetAs[bool](legacyMonitoringMigrationDoneKey, false)
|
||||
if err != nil {
|
||||
return fmt.Errorf("read legacy monitoring migration marker: %w", err)
|
||||
}
|
||||
|
||||
stats, err := runLegacyMonitoringMigration(context.Background(), ctx.DB, ctx.Store, done, func() error {
|
||||
return appconfig.Set(legacyMonitoringMigrationDoneKey, true)
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if stats.Records > 0 || stats.GPU > 0 || stats.Ping > 0 {
|
||||
log.Printf("Legacy monitoring table migration completed (records=%d, gpu=%d, ping=%d)", stats.Records, stats.GPU, stats.Ping)
|
||||
}
|
||||
return nil
|
||||
type legacyHourlyP95Aggregator struct {
|
||||
store *metric.Store
|
||||
partitionKey string
|
||||
bucket time.Time
|
||||
groups map[string]*legacyP95Group
|
||||
}
|
||||
|
||||
// LegacyMonitoringMigrationRequired reports whether startup must enter the
|
||||
// restricted 1.2.7 upgrade flow. Empty legacy tables are handled by the normal
|
||||
// one-shot migration so fresh installations never see the wizard.
|
||||
// restricted 1.2.7 upgrade flow. The migration itself runs only after an
|
||||
// administrator explicitly starts it from that guide.
|
||||
func LegacyMonitoringMigrationRequired(db *gorm.DB) (bool, LegacyMonitoringSummary, error) {
|
||||
done, err := appconfig.GetAs[bool](legacyMonitoringMigrationDoneKey, false)
|
||||
if err != nil {
|
||||
@@ -171,7 +142,6 @@ func InspectLegacyMonitoring(db *gorm.DB) (LegacyMonitoringSummary, error) {
|
||||
}
|
||||
|
||||
summary.MonitoringRows = summary.LoadRows + summary.GPURows + summary.LatencyRows
|
||||
summary.EstimatedPoints = summary.LoadRows*19 + summary.GPURows*4 + summary.LatencyRows*2
|
||||
if !oldest.IsZero() {
|
||||
oldestCopy := oldest
|
||||
summary.OldestAt = &oldestCopy
|
||||
@@ -181,6 +151,11 @@ func InspectLegacyMonitoring(db *gorm.DB) (LegacyMonitoringSummary, error) {
|
||||
summary.NewestAt = &newestCopy
|
||||
}
|
||||
if !oldest.IsZero() && !newest.IsZero() {
|
||||
estimatedPoints, err := estimateLegacyHourlyP95Points(db, summary, oldest, newest)
|
||||
if err != nil {
|
||||
return summary, err
|
||||
}
|
||||
summary.EstimatedPoints = estimatedPoints
|
||||
retentionEnd := time.Now().UTC()
|
||||
if newest.After(retentionEnd) {
|
||||
retentionEnd = newest
|
||||
@@ -193,41 +168,10 @@ func InspectLegacyMonitoring(db *gorm.DB) (LegacyMonitoringSummary, error) {
|
||||
return summary, nil
|
||||
}
|
||||
|
||||
// DeleteLegacyMonitoringBefore removes only monitoring history older than the
|
||||
// cutoff. Clients, users, ping tasks and other business data are untouched.
|
||||
func DeleteLegacyMonitoringBefore(db *gorm.DB, cutoff time.Time) (LegacyMonitoringDeleted, error) {
|
||||
var deleted LegacyMonitoringDeleted
|
||||
if db == nil {
|
||||
return deleted, fmt.Errorf("migration database is nil")
|
||||
}
|
||||
if cutoff.IsZero() {
|
||||
return deleted, fmt.Errorf("cleanup cutoff is required")
|
||||
}
|
||||
|
||||
err := db.Transaction(func(tx *gorm.DB) error {
|
||||
for _, table := range legacyMonitoringTables {
|
||||
if !tx.Migrator().HasTable(table) {
|
||||
continue
|
||||
}
|
||||
result := tx.Exec("DELETE FROM "+table+" WHERE time < ?", cutoff.UTC())
|
||||
if result.Error != nil {
|
||||
return fmt.Errorf("delete legacy %s before cutoff: %w", table, result.Error)
|
||||
}
|
||||
switch table {
|
||||
case "records", "records_long_term":
|
||||
deleted.LoadRows += result.RowsAffected
|
||||
case "gpu_records":
|
||||
deleted.GPURows += result.RowsAffected
|
||||
case "ping_records":
|
||||
deleted.LatencyRows += result.RowsAffected
|
||||
}
|
||||
}
|
||||
return nil
|
||||
})
|
||||
return deleted, err
|
||||
}
|
||||
|
||||
func MigrateLegacyMonitoring(ctx context.Context, db *gorm.DB, s *metric.Store, progress func(LegacyMonitoringProgress)) (LegacyMonitoringStats, error) {
|
||||
if err := metricstore.EnsureBuiltinMetricDefinitions(ctx, s); err != nil {
|
||||
return LegacyMonitoringStats{}, fmt.Errorf("ensure built-in metric definitions: %w", err)
|
||||
}
|
||||
return migrateLegacyMonitoringTables(ctx, db, s, progress)
|
||||
}
|
||||
|
||||
@@ -301,15 +245,13 @@ func migrateLegacyMonitoringTables(ctx context.Context, db *gorm.DB, s *metric.S
|
||||
if progress != nil {
|
||||
emit("", 0, 0)
|
||||
}
|
||||
for _, table := range []string{"records", "records_long_term"} {
|
||||
n, err := migrateLegacyRecordTable(ctx, s, db, table, emit)
|
||||
if err != nil {
|
||||
return stats, err
|
||||
}
|
||||
stats.Records += n
|
||||
n, err := migrateLegacyRecordTables(ctx, s, db, []string{"records", "records_long_term"}, emit)
|
||||
if err != nil {
|
||||
return stats, err
|
||||
}
|
||||
stats.Records += n
|
||||
|
||||
n, err := migrateLegacyGPURecordTable(ctx, s, db, "gpu_records", emit)
|
||||
n, err = migrateLegacyGPURecordTable(ctx, s, db, "gpu_records", emit)
|
||||
if err != nil {
|
||||
return stats, err
|
||||
}
|
||||
@@ -324,56 +266,49 @@ func migrateLegacyMonitoringTables(ctx context.Context, db *gorm.DB, s *metric.S
|
||||
return stats, nil
|
||||
}
|
||||
|
||||
func migrateLegacyRecordTable(ctx context.Context, s *metric.Store, db *gorm.DB, table string, progress legacyBatchProgress) (int64, error) {
|
||||
if !db.Migrator().HasTable(table) {
|
||||
return 0, nil
|
||||
}
|
||||
|
||||
func migrateLegacyRecordTables(ctx context.Context, s *metric.Store, db *gorm.DB, tables []string, progress legacyBatchProgress) (int64, error) {
|
||||
var existing []string
|
||||
var total int64
|
||||
if err := db.Table(table).Count(&total).Error; err != nil {
|
||||
return 0, fmt.Errorf("count legacy %s: %w", table, err)
|
||||
for _, table := range tables {
|
||||
if !db.Migrator().HasTable(table) {
|
||||
continue
|
||||
}
|
||||
var count int64
|
||||
if err := db.Table(table).Count(&count).Error; err != nil {
|
||||
return 0, fmt.Errorf("count legacy %s: %w", table, err)
|
||||
}
|
||||
if count > 0 {
|
||||
existing = append(existing, table)
|
||||
total += count
|
||||
}
|
||||
}
|
||||
if total == 0 {
|
||||
if len(existing) == 0 {
|
||||
return 0, nil
|
||||
}
|
||||
|
||||
log.Printf("[legacy-migration] importing %d rows from %s", total, table)
|
||||
var migrated int64
|
||||
var lastRowID int64
|
||||
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return migrated, ctx.Err()
|
||||
default:
|
||||
}
|
||||
|
||||
var rows []legacyRecordRow
|
||||
q := db.Table(table).Select("rowid AS row_id, *").Where("rowid > ?", lastRowID).Order("rowid ASC").Limit(legacyMonitoringBatchSize)
|
||||
if err := q.Find(&rows).Error; err != nil {
|
||||
return migrated, fmt.Errorf("read legacy %s: %w", table, err)
|
||||
}
|
||||
if len(rows) == 0 {
|
||||
break
|
||||
}
|
||||
|
||||
points := make([]metric.Point, 0, len(rows)*19)
|
||||
for i := range rows {
|
||||
points = append(points, recordToPoints(rows[i].Record)...)
|
||||
}
|
||||
if err := s.WriteBatch(ctx, points); err != nil {
|
||||
return migrated, fmt.Errorf("write legacy %s to metric store: %w", table, err)
|
||||
}
|
||||
|
||||
last := rows[len(rows)-1]
|
||||
lastRowID = last.RowID
|
||||
migrated += int64(len(rows))
|
||||
if progress != nil {
|
||||
progress(table, int64(len(rows)), int64(len(points)))
|
||||
parts := make([]string, 0, len(existing))
|
||||
for _, table := range existing {
|
||||
projection, err := legacyRecordProjection(db, table)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
parts = append(parts, projection)
|
||||
}
|
||||
query := strings.Join(parts, " UNION ALL ") + " ORDER BY client ASC, time ASC"
|
||||
rows, err := db.Raw(query).Rows()
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("stream legacy record tables: %w", err)
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
log.Printf("[legacy-migration] imported %d rows from %s", migrated, table)
|
||||
logger.Infof("migration", "[legacy-migration] aggregating %d rows from %s into 1h P95 points", total, strings.Join(existing, ","))
|
||||
migrated, err := migrateLegacyStream(ctx, db, s, rows, "records", func() *models.Record { return &models.Record{} }, func(value models.Record) []metric.Point {
|
||||
return recordToPoints(value)
|
||||
}, progress)
|
||||
if err != nil {
|
||||
return migrated, fmt.Errorf("aggregate legacy record tables: %w", err)
|
||||
}
|
||||
logger.Infof("migration", "[legacy-migration] aggregated %d rows from %s", migrated, strings.Join(existing, ","))
|
||||
return migrated, nil
|
||||
}
|
||||
|
||||
@@ -390,43 +325,19 @@ func migrateLegacyGPURecordTable(ctx context.Context, s *metric.Store, db *gorm.
|
||||
return 0, nil
|
||||
}
|
||||
|
||||
log.Printf("[legacy-migration] importing %d rows from %s", total, table)
|
||||
var migrated int64
|
||||
var lastRowID int64
|
||||
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return migrated, ctx.Err()
|
||||
default:
|
||||
}
|
||||
|
||||
var rows []legacyGPURecordRow
|
||||
q := db.Table(table).Select("rowid AS row_id, *").Where("rowid > ?", lastRowID).Order("rowid ASC").Limit(legacyMonitoringBatchSize)
|
||||
if err := q.Find(&rows).Error; err != nil {
|
||||
return migrated, fmt.Errorf("read legacy %s: %w", table, err)
|
||||
}
|
||||
if len(rows) == 0 {
|
||||
break
|
||||
}
|
||||
|
||||
points := make([]metric.Point, 0, len(rows)*4)
|
||||
for i := range rows {
|
||||
points = append(points, gpuRecordToPoints(rows[i].GPURecord)...)
|
||||
}
|
||||
if err := s.WriteBatch(ctx, points); err != nil {
|
||||
return migrated, fmt.Errorf("write legacy %s to metric store: %w", table, err)
|
||||
}
|
||||
|
||||
last := rows[len(rows)-1]
|
||||
lastRowID = last.RowID
|
||||
migrated += int64(len(rows))
|
||||
if progress != nil {
|
||||
progress(table, int64(len(rows)), int64(len(points)))
|
||||
}
|
||||
rows, err := db.Table(table).Select("*").Order("client ASC, device_index ASC, device_name ASC, time ASC").Rows()
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("stream legacy %s: %w", table, err)
|
||||
}
|
||||
|
||||
log.Printf("[legacy-migration] imported %d rows from %s", migrated, table)
|
||||
defer rows.Close()
|
||||
logger.Infof("migration", "[legacy-migration] aggregating %d rows from %s into 1h P95 points", total, table)
|
||||
migrated, err := migrateLegacyStream(ctx, db, s, rows, table, func() *models.GPURecord { return &models.GPURecord{} }, func(value models.GPURecord) []metric.Point {
|
||||
return gpuRecordToPoints(value)
|
||||
}, progress)
|
||||
if err != nil {
|
||||
return migrated, fmt.Errorf("aggregate legacy %s: %w", table, err)
|
||||
}
|
||||
logger.Infof("migration", "[legacy-migration] aggregated %d rows from %s", migrated, table)
|
||||
return migrated, nil
|
||||
}
|
||||
|
||||
@@ -443,46 +354,254 @@ func migrateLegacyPingRecordTable(ctx context.Context, s *metric.Store, db *gorm
|
||||
return 0, nil
|
||||
}
|
||||
|
||||
log.Printf("[legacy-migration] importing %d rows from %s", total, table)
|
||||
var migrated int64
|
||||
var lastRowID int64
|
||||
rows, err := db.Table(table).Select("*").Order("client ASC, task_id ASC, time ASC").Rows()
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("stream legacy %s: %w", table, err)
|
||||
}
|
||||
defer rows.Close()
|
||||
logger.Infof("migration", "[legacy-migration] aggregating %d rows from %s into 1h P95 points", total, table)
|
||||
migrated, err := migrateLegacyStream(ctx, db, s, rows, table, func() *models.PingRecord { return &models.PingRecord{} }, func(value models.PingRecord) []metric.Point {
|
||||
return pingRecordToPoints(value)
|
||||
}, progress)
|
||||
if err != nil {
|
||||
return migrated, fmt.Errorf("aggregate legacy %s: %w", table, err)
|
||||
}
|
||||
logger.Infof("migration", "[legacy-migration] aggregated %d rows from %s", migrated, table)
|
||||
return migrated, nil
|
||||
}
|
||||
|
||||
for {
|
||||
func migrateLegacyStream[T any](ctx context.Context, db *gorm.DB, s *metric.Store, rows *sql.Rows, table string, newValue func() *T, toPoints func(T) []metric.Point, progress legacyBatchProgress) (int64, error) {
|
||||
aggregator := &legacyHourlyP95Aggregator{store: s}
|
||||
var migrated, batchRows, batchPoints int64
|
||||
emit := func(force bool) {
|
||||
if progress != nil && (batchRows > 0 || force && batchPoints > 0) {
|
||||
progress(table, batchRows, batchPoints)
|
||||
}
|
||||
batchRows = 0
|
||||
batchPoints = 0
|
||||
}
|
||||
for rows.Next() {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return migrated, ctx.Err()
|
||||
default:
|
||||
}
|
||||
|
||||
var rows []legacyPingRecordRow
|
||||
q := db.Table(table).Select("rowid AS row_id, *").Where("rowid > ?", lastRowID).Order("rowid ASC").Limit(legacyMonitoringBatchSize)
|
||||
if err := q.Find(&rows).Error; err != nil {
|
||||
return migrated, fmt.Errorf("read legacy %s: %w", table, err)
|
||||
value := newValue()
|
||||
if err := db.ScanRows(rows, value); err != nil {
|
||||
return migrated, err
|
||||
}
|
||||
if len(rows) == 0 {
|
||||
break
|
||||
written, err := aggregator.Add(ctx, toPoints(*value))
|
||||
if err != nil {
|
||||
return migrated, err
|
||||
}
|
||||
|
||||
points := make([]metric.Point, 0, len(rows)*2)
|
||||
for i := range rows {
|
||||
points = append(points, pingRecordToPoints(rows[i].PingRecord)...)
|
||||
}
|
||||
if err := s.WriteBatch(ctx, points); err != nil {
|
||||
return migrated, fmt.Errorf("write legacy %s to metric store: %w", table, err)
|
||||
}
|
||||
|
||||
last := rows[len(rows)-1]
|
||||
lastRowID = last.RowID
|
||||
migrated += int64(len(rows))
|
||||
if progress != nil {
|
||||
progress(table, int64(len(rows)), int64(len(points)))
|
||||
migrated++
|
||||
batchRows++
|
||||
batchPoints += written
|
||||
if batchRows >= legacyMonitoringBatchSize {
|
||||
emit(false)
|
||||
}
|
||||
}
|
||||
|
||||
log.Printf("[legacy-migration] imported %d rows from %s", migrated, table)
|
||||
if err := rows.Err(); err != nil {
|
||||
return migrated, err
|
||||
}
|
||||
written, err := aggregator.Flush(ctx)
|
||||
if err != nil {
|
||||
return migrated, err
|
||||
}
|
||||
batchPoints += written
|
||||
emit(true)
|
||||
return migrated, nil
|
||||
}
|
||||
|
||||
func legacyRecordProjection(db *gorm.DB, table string) (string, error) {
|
||||
columnTypes, err := db.Migrator().ColumnTypes(table)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("inspect legacy %s columns: %w", table, err)
|
||||
}
|
||||
available := make(map[string]struct{}, len(columnTypes))
|
||||
for _, column := range columnTypes {
|
||||
available[strings.ToLower(column.Name())] = struct{}{}
|
||||
}
|
||||
for _, required := range []string{"client", "time"} {
|
||||
if _, ok := available[required]; !ok {
|
||||
return "", fmt.Errorf("legacy %s is missing required column %s", table, required)
|
||||
}
|
||||
}
|
||||
columns := []string{
|
||||
"client", "time", "cpu", "gpu", "ram", "swap", "load", "disk",
|
||||
"net_in", "net_out", "net_total_up", "net_total_down", "traffic_up",
|
||||
"traffic_down", "process", "connections", "connections_udp",
|
||||
}
|
||||
projection := make([]string, 0, len(columns))
|
||||
for _, column := range columns {
|
||||
if _, ok := available[column]; ok {
|
||||
projection = append(projection, column)
|
||||
} else {
|
||||
projection = append(projection, "0 AS "+column)
|
||||
}
|
||||
}
|
||||
return "SELECT " + strings.Join(projection, ", ") + " FROM " + table, nil
|
||||
}
|
||||
|
||||
func (a *legacyHourlyP95Aggregator) Add(ctx context.Context, points []metric.Point) (int64, error) {
|
||||
var written int64
|
||||
for _, point := range points {
|
||||
bucket := point.Timestamp.UTC().Truncate(legacyMonitoringInterval)
|
||||
partition, err := legacyPointPartition(point)
|
||||
if err != nil {
|
||||
return written, err
|
||||
}
|
||||
if a.groups != nil && (partition != a.partitionKey || !bucket.Equal(a.bucket)) {
|
||||
if partition == a.partitionKey && bucket.Before(a.bucket) {
|
||||
return written, fmt.Errorf("legacy points are not ordered within series: %s before %s", bucket, a.bucket)
|
||||
}
|
||||
n, err := a.Flush(ctx)
|
||||
written += n
|
||||
if err != nil {
|
||||
return written, err
|
||||
}
|
||||
}
|
||||
if a.groups == nil {
|
||||
a.partitionKey = partition
|
||||
a.bucket = bucket
|
||||
a.groups = make(map[string]*legacyP95Group)
|
||||
}
|
||||
group := a.groups[point.MetricName]
|
||||
if group == nil {
|
||||
group = &legacyP95Group{
|
||||
metricName: point.MetricName,
|
||||
entityID: point.EntityID,
|
||||
tags: cloneLegacyTags(point.Tags),
|
||||
}
|
||||
a.groups[point.MetricName] = group
|
||||
}
|
||||
group.values = append(group.values, point.Value)
|
||||
}
|
||||
return written, nil
|
||||
}
|
||||
|
||||
func (a *legacyHourlyP95Aggregator) Flush(ctx context.Context) (int64, error) {
|
||||
if len(a.groups) == 0 {
|
||||
return 0, nil
|
||||
}
|
||||
names := make([]string, 0, len(a.groups))
|
||||
for name := range a.groups {
|
||||
names = append(names, name)
|
||||
}
|
||||
sort.Strings(names)
|
||||
points := make([]metric.Point, 0, len(names))
|
||||
for _, name := range names {
|
||||
group := a.groups[name]
|
||||
points = append(points, metric.Point{
|
||||
MetricName: group.metricName,
|
||||
EntityID: group.entityID,
|
||||
Timestamp: a.bucket,
|
||||
Value: legacyP95(group.values),
|
||||
Tags: group.tags,
|
||||
})
|
||||
}
|
||||
if err := a.store.ReplaceRollupPoints(ctx, legacyMonitoringInterval, points); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
a.partitionKey = ""
|
||||
a.bucket = time.Time{}
|
||||
a.groups = nil
|
||||
return int64(len(points)), nil
|
||||
}
|
||||
|
||||
func legacyP95(values []float64) float64 {
|
||||
if len(values) == 0 {
|
||||
return 0
|
||||
}
|
||||
sort.Float64s(values)
|
||||
if len(values) == 1 {
|
||||
return values[0]
|
||||
}
|
||||
position := 0.95 * float64(len(values)-1)
|
||||
lower := int(math.Floor(position))
|
||||
upper := int(math.Ceil(position))
|
||||
if lower == upper {
|
||||
return values[lower]
|
||||
}
|
||||
weight := position - float64(lower)
|
||||
return values[lower]*(1-weight) + values[upper]*weight
|
||||
}
|
||||
|
||||
func legacyPointPartition(point metric.Point) (string, error) {
|
||||
tags, err := json.Marshal(point.Tags)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("encode legacy point tags: %w", err)
|
||||
}
|
||||
return point.EntityID + "\x00" + string(tags), nil
|
||||
}
|
||||
|
||||
func cloneLegacyTags(tags map[string]string) map[string]string {
|
||||
if len(tags) == 0 {
|
||||
return nil
|
||||
}
|
||||
cloned := make(map[string]string, len(tags))
|
||||
for key, value := range tags {
|
||||
cloned[key] = value
|
||||
}
|
||||
return cloned
|
||||
}
|
||||
|
||||
func estimateLegacyHourlyP95Points(db *gorm.DB, summary LegacyMonitoringSummary, oldest, newest time.Time) (int64, error) {
|
||||
type estimate struct {
|
||||
Count int64 `gorm:"column:count"`
|
||||
}
|
||||
countSeries := func(query string) (int64, error) {
|
||||
var result estimate
|
||||
if err := db.Raw("SELECT COUNT(*) AS count FROM (" + query + ")").Scan(&result).Error; err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return result.Count, nil
|
||||
}
|
||||
firstHour := oldest.UTC().Truncate(legacyMonitoringInterval)
|
||||
lastHour := newest.UTC().Truncate(legacyMonitoringInterval)
|
||||
hours := int64(lastHour.Sub(firstHour)/legacyMonitoringInterval) + 1
|
||||
if hours < 1 {
|
||||
hours = 1
|
||||
}
|
||||
|
||||
var recordParts []string
|
||||
for _, table := range []string{"records", "records_long_term"} {
|
||||
if db.Migrator().HasTable(table) {
|
||||
recordParts = append(recordParts, "SELECT client FROM "+table)
|
||||
}
|
||||
}
|
||||
var total int64
|
||||
if len(recordParts) > 0 {
|
||||
series, err := countSeries(strings.Join(recordParts, " UNION "))
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("estimate load series: %w", err)
|
||||
}
|
||||
total += minLegacyBuckets(summary.LoadRows, series*hours) * 15
|
||||
}
|
||||
if db.Migrator().HasTable("gpu_records") {
|
||||
series, err := countSeries("SELECT client, device_index, device_name FROM gpu_records GROUP BY client, device_index, device_name")
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("estimate GPU series: %w", err)
|
||||
}
|
||||
total += minLegacyBuckets(summary.GPURows, series*hours) * 4
|
||||
}
|
||||
if db.Migrator().HasTable("ping_records") {
|
||||
series, err := countSeries("SELECT client, task_id FROM ping_records GROUP BY client, task_id")
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("estimate ping series: %w", err)
|
||||
}
|
||||
total += minLegacyBuckets(summary.LatencyRows, series*hours) * 2
|
||||
}
|
||||
return total, nil
|
||||
}
|
||||
|
||||
func minLegacyBuckets(sourceRows, estimatedBuckets int64) int64 {
|
||||
if sourceRows < estimatedBuckets {
|
||||
return sourceRows
|
||||
}
|
||||
return estimatedBuckets
|
||||
}
|
||||
|
||||
func recordToPoints(rec models.Record) []metric.Point {
|
||||
ts := rec.Time
|
||||
entityID := rec.Client
|
||||
@@ -490,13 +609,9 @@ func recordToPoints(rec models.Record) []metric.Point {
|
||||
{MetricName: metricstore.MetricCPU, EntityID: entityID, Timestamp: ts, Value: float64(rec.Cpu)},
|
||||
{MetricName: metricstore.MetricGPU, EntityID: entityID, Timestamp: ts, Value: float64(rec.Gpu)},
|
||||
{MetricName: metricstore.MetricRAM, EntityID: entityID, Timestamp: ts, Value: float64(rec.Ram)},
|
||||
{MetricName: metricstore.MetricRAMTotal, EntityID: entityID, Timestamp: ts, Value: float64(rec.RamTotal)},
|
||||
{MetricName: metricstore.MetricSwap, EntityID: entityID, Timestamp: ts, Value: float64(rec.Swap)},
|
||||
{MetricName: metricstore.MetricSwapTotal, EntityID: entityID, Timestamp: ts, Value: float64(rec.SwapTotal)},
|
||||
{MetricName: metricstore.MetricLoad, EntityID: entityID, Timestamp: ts, Value: float64(rec.Load)},
|
||||
{MetricName: metricstore.MetricTemp, EntityID: entityID, Timestamp: ts, Value: float64(rec.Temp)},
|
||||
{MetricName: metricstore.MetricDisk, EntityID: entityID, Timestamp: ts, Value: float64(rec.Disk)},
|
||||
{MetricName: metricstore.MetricDiskTotal, EntityID: entityID, Timestamp: ts, Value: float64(rec.DiskTotal)},
|
||||
{MetricName: metricstore.MetricNetIn, EntityID: entityID, Timestamp: ts, Value: float64(rec.NetIn)},
|
||||
{MetricName: metricstore.MetricNetOut, EntityID: entityID, Timestamp: ts, Value: float64(rec.NetOut)},
|
||||
{MetricName: metricstore.MetricNetTotalUp, EntityID: entityID, Timestamp: ts, Value: float64(rec.NetTotalUp)},
|
||||
@@ -553,7 +668,7 @@ func dropLegacyMonitoringTables(db *gorm.DB) error {
|
||||
if !db.Migrator().HasTable(table) {
|
||||
continue
|
||||
}
|
||||
log.Printf("[legacy-migration] dropping legacy table %s", table)
|
||||
logger.Infof("migration", "[legacy-migration] dropping legacy table %s", table)
|
||||
if err := db.Migrator().DropTable(table); err != nil {
|
||||
return fmt.Errorf("drop legacy table %s: %w", table, err)
|
||||
}
|
||||
+93
-67
@@ -4,13 +4,14 @@ import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"errors"
|
||||
"math"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/komari-monitor/komari/database/metricstore"
|
||||
"github.com/komari-monitor/komari/internal/metricstore"
|
||||
"github.com/komari-monitor/komari/database/models"
|
||||
appconfig "github.com/komari-monitor/komari/pkg/config"
|
||||
appconfig "github.com/komari-monitor/komari/internal/config"
|
||||
"github.com/komari-monitor/komari/pkg/metric"
|
||||
"gorm.io/driver/sqlite"
|
||||
"gorm.io/gorm"
|
||||
@@ -57,7 +58,7 @@ func TestLegacyMonitoringTablesMigratedByOneShotMigration(t *testing.T) {
|
||||
if summary.LoadRows != 2 || summary.GPURows != 1 || summary.LatencyRows != 2 || summary.MonitoringRows != 5 {
|
||||
t.Fatalf("unexpected legacy monitoring summary: %#v", summary)
|
||||
}
|
||||
if summary.EstimatedPoints != 46 || summary.RetentionDays < 1 {
|
||||
if summary.EstimatedPoints != 21 || summary.RetentionDays < 1 {
|
||||
t.Fatalf("unexpected legacy point estimate or retention: %#v", summary)
|
||||
}
|
||||
|
||||
@@ -81,7 +82,7 @@ func TestLegacyMonitoringTablesMigratedByOneShotMigration(t *testing.T) {
|
||||
}); err != nil {
|
||||
t.Fatalf("migrate legacy monitoring with progress: %v", err)
|
||||
}
|
||||
if lastProgress.SourceRowsDone != 5 || lastProgress.SourceRowsTotal != 5 || lastProgress.WrittenPoints != 46 {
|
||||
if lastProgress.SourceRowsDone != 5 || lastProgress.SourceRowsTotal != 5 || lastProgress.WrittenPoints != 21 {
|
||||
t.Fatalf("unexpected final migration progress: %#v", lastProgress)
|
||||
}
|
||||
|
||||
@@ -99,35 +100,24 @@ func TestLegacyMonitoringTablesMigratedByOneShotMigration(t *testing.T) {
|
||||
t.Fatalf("unexpected stats: %#v", stats)
|
||||
}
|
||||
|
||||
cpuPoints, err := metricStore.Query(ctx, metric.Query{MetricName: metricstore.MetricCPU, EntityID: "client-a", Start: base.Add(-time.Second), End: base.Add(2 * time.Minute)})
|
||||
if err != nil {
|
||||
t.Fatalf("query cpu points: %v", err)
|
||||
}
|
||||
if len(cpuPoints) != 2 || cpuPoints[0].Value != 12.5 || cpuPoints[1].Value != 22.5 {
|
||||
hour := base.Truncate(time.Hour)
|
||||
cpuPoints := queryLegacyRollups(t, ctx, metricStore, metric.Query{MetricName: metricstore.MetricCPU, EntityID: "client-a", Start: hour.Add(-time.Second), End: hour.Add(time.Hour)})
|
||||
if len(cpuPoints) != 1 || math.Abs(cpuPoints[0].Value-22) > 1e-9 || !cpuPoints[0].Bucket.Equal(hour) {
|
||||
t.Fatalf("unexpected cpu points: %#v", cpuPoints)
|
||||
}
|
||||
|
||||
gpuPoints, err := metricStore.Query(ctx, metric.Query{MetricName: metricstore.MetricGPUDeviceUsage, EntityID: "client-a", Start: base.Add(-time.Second), End: base.Add(time.Second), Tags: map[string]string{"device_index": "0"}})
|
||||
if err != nil {
|
||||
t.Fatalf("query gpu points: %v", err)
|
||||
}
|
||||
gpuPoints := queryLegacyRollups(t, ctx, metricStore, metric.Query{MetricName: metricstore.MetricGPUDeviceUsage, EntityID: "client-a", Start: hour.Add(-time.Second), End: hour.Add(time.Hour), Tags: map[string]string{"device_index": "0"}})
|
||||
if len(gpuPoints) != 1 || gpuPoints[0].Value != 67 {
|
||||
t.Fatalf("unexpected gpu points: %#v", gpuPoints)
|
||||
}
|
||||
|
||||
pingPoints, err := metricStore.Query(ctx, metric.Query{MetricName: metricstore.MetricPingLatency, EntityID: "client-a", Start: base.Add(-time.Second), End: base.Add(time.Minute), Tags: map[string]string{"task_id": "7"}})
|
||||
if err != nil {
|
||||
t.Fatalf("query ping points: %v", err)
|
||||
}
|
||||
if len(pingPoints) != 2 || pingPoints[0].Value != 36 || pingPoints[1].Value != -1 {
|
||||
pingPoints := queryLegacyRollups(t, ctx, metricStore, metric.Query{MetricName: metricstore.MetricPingLatency, EntityID: "client-a", Start: hour.Add(-time.Second), End: hour.Add(time.Hour), Tags: map[string]string{"task_id": "7"}})
|
||||
if len(pingPoints) != 1 || math.Abs(pingPoints[0].Value-34.15) > 1e-9 || !pingPoints[0].Bucket.Equal(hour) {
|
||||
t.Fatalf("unexpected ping points: %#v", pingPoints)
|
||||
}
|
||||
|
||||
pingLossPoints, err := metricStore.Query(ctx, metric.Query{MetricName: metricstore.MetricPingLoss, EntityID: "client-a", Start: base.Add(-time.Second), End: base.Add(time.Minute), Tags: map[string]string{"task_id": "7"}})
|
||||
if err != nil {
|
||||
t.Fatalf("query ping loss points: %v", err)
|
||||
}
|
||||
if len(pingLossPoints) != 2 || pingLossPoints[0].Value != 0 || pingLossPoints[1].Value != 1 {
|
||||
pingLossPoints := queryLegacyRollups(t, ctx, metricStore, metric.Query{MetricName: metricstore.MetricPingLoss, EntityID: "client-a", Start: hour.Add(-time.Second), End: hour.Add(time.Hour), Tags: map[string]string{"task_id": "7"}})
|
||||
if len(pingLossPoints) != 1 || math.Abs(pingLossPoints[0].Value-0.95) > 1e-9 || !pingLossPoints[0].Bucket.Equal(hour) {
|
||||
t.Fatalf("unexpected ping loss points: %#v", pingLossPoints)
|
||||
}
|
||||
|
||||
@@ -152,62 +142,98 @@ func TestLegacyMonitoringTablesMigratedByOneShotMigration(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestDeleteLegacyMonitoringBeforeOnlyRemovesOldHistory(t *testing.T) {
|
||||
db, err := gorm.Open(sqlite.Open("file:"+filepath.ToSlash(filepath.Join(t.TempDir(), "cleanup.db"))+"?mode=rwc"), &gorm.Config{})
|
||||
func TestLegacyHourlyP95PreservesHoursAndTaggedSeries(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
metricDB, err := sql.Open("sqlite3", "file:"+filepath.ToSlash(filepath.Join(t.TempDir(), "hourly-p95.db"))+"?mode=rwc")
|
||||
if err != nil {
|
||||
t.Fatalf("open cleanup db: %v", err)
|
||||
t.Fatalf("open metric db: %v", err)
|
||||
}
|
||||
store, err := metric.Open(ctx, metric.SQLite("", metric.WithDB(metricDB)))
|
||||
if err != nil {
|
||||
t.Fatalf("open metric store: %v", err)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
_ = store.Close()
|
||||
_ = metricDB.Close()
|
||||
})
|
||||
if err := store.UpsertMetric(ctx, metric.Definition{Name: "test.p95", Type: metric.TypeGauge, RetentionDays: 1}); err != nil {
|
||||
t.Fatalf("create test metric: %v", err)
|
||||
}
|
||||
|
||||
base := time.Date(2026, 7, 8, 10, 0, 0, 0, time.UTC)
|
||||
aggregator := &legacyHourlyP95Aggregator{store: store}
|
||||
inputs := [][]metric.Point{
|
||||
{{MetricName: "test.p95", EntityID: "node-a", Timestamp: base.Add(time.Minute), Value: 10, Tags: map[string]string{"task_id": "1"}}},
|
||||
{{MetricName: "test.p95", EntityID: "node-a", Timestamp: base.Add(2 * time.Minute), Value: 20, Tags: map[string]string{"task_id": "1"}}},
|
||||
{{MetricName: "test.p95", EntityID: "node-a", Timestamp: base.Add(time.Hour), Value: 30, Tags: map[string]string{"task_id": "1"}}},
|
||||
{{MetricName: "test.p95", EntityID: "node-a", Timestamp: base.Add(time.Minute), Value: 100, Tags: map[string]string{"task_id": "2"}}},
|
||||
}
|
||||
for _, points := range inputs {
|
||||
if _, err := aggregator.Add(ctx, points); err != nil {
|
||||
t.Fatalf("add aggregate input: %v", err)
|
||||
}
|
||||
}
|
||||
if _, err := aggregator.Flush(ctx); err != nil {
|
||||
t.Fatalf("flush aggregate input: %v", err)
|
||||
}
|
||||
|
||||
taskOne := queryLegacyRollups(t, ctx, store, metric.Query{
|
||||
MetricName: "test.p95", EntityID: "node-a", Start: base.Add(-time.Second), End: base.Add(2 * time.Hour),
|
||||
Tags: map[string]string{"task_id": "1"}, Order: metric.OrderAsc,
|
||||
})
|
||||
if len(taskOne) != 2 || math.Abs(taskOne[0].Value-19.5) > 1e-9 || taskOne[1].Value != 30 {
|
||||
t.Fatalf("unexpected task one aggregates: %#v", taskOne)
|
||||
}
|
||||
taskTwo := queryLegacyRollups(t, ctx, store, metric.Query{
|
||||
MetricName: "test.p95", EntityID: "node-a", Start: base.Add(-time.Second), End: base.Add(2 * time.Hour),
|
||||
Tags: map[string]string{"task_id": "2"}, Order: metric.OrderAsc,
|
||||
})
|
||||
if len(taskTwo) != 1 || taskTwo[0].Value != 100 || !taskTwo[0].Bucket.Equal(base) {
|
||||
t.Fatalf("unexpected task two aggregates: %#v", taskTwo)
|
||||
}
|
||||
}
|
||||
|
||||
func queryLegacyRollups(t *testing.T, ctx context.Context, store *metric.Store, query metric.Query) []metric.AggregatePoint {
|
||||
t.Helper()
|
||||
points, err := store.Series(ctx, metric.AggregateQuery{
|
||||
Query: query, Aggregation: metric.AggLast, Interval: time.Hour, PreserveSeries: true,
|
||||
}, query.End)
|
||||
if err != nil {
|
||||
t.Fatalf("query %s rollups: %v", query.MetricName, err)
|
||||
}
|
||||
return points
|
||||
}
|
||||
|
||||
func TestLegacyRecordProjectionFillsMissingMetricColumns(t *testing.T) {
|
||||
db, err := gorm.Open(sqlite.Open("file:"+filepath.ToSlash(filepath.Join(t.TempDir(), "minimal-legacy.db"))+"?mode=rwc"), &gorm.Config{})
|
||||
if err != nil {
|
||||
t.Fatalf("open minimal legacy db: %v", err)
|
||||
}
|
||||
if sqlDB, err := db.DB(); err == nil {
|
||||
t.Cleanup(func() { _ = sqlDB.Close() })
|
||||
} else {
|
||||
t.Fatalf("cleanup sql db: %v", err)
|
||||
t.Fatalf("minimal legacy sql db: %v", err)
|
||||
}
|
||||
if err := db.AutoMigrate(&models.Client{}, &models.Record{}, &models.GPURecord{}, &models.PingRecord{}); err != nil {
|
||||
t.Fatalf("migrate cleanup tables: %v", err)
|
||||
if err := db.Exec("CREATE TABLE records_minimal (client TEXT NOT NULL, time DATETIME NOT NULL, cpu REAL NOT NULL)").Error; err != nil {
|
||||
t.Fatalf("create minimal legacy table: %v", err)
|
||||
}
|
||||
if err := db.Table("records_long_term").AutoMigrate(&models.Record{}); err != nil {
|
||||
t.Fatalf("migrate cleanup long-term table: %v", err)
|
||||
base := time.Date(2026, 7, 8, 10, 0, 0, 0, time.UTC)
|
||||
if err := db.Exec("INSERT INTO records_minimal (client, time, cpu) VALUES (?, ?, ?)", "node-a", base, 42.5).Error; err != nil {
|
||||
t.Fatalf("insert minimal legacy row: %v", err)
|
||||
}
|
||||
if err := db.Create(&models.Client{UUID: "client-a", Token: "token-a", Name: "A"}).Error; err != nil {
|
||||
t.Fatalf("seed client: %v", err)
|
||||
}
|
||||
|
||||
cutoff := time.Date(2026, 7, 10, 0, 0, 0, 0, time.UTC)
|
||||
oldTime := cutoff.Add(-time.Hour)
|
||||
newTime := cutoff.Add(time.Hour)
|
||||
for _, record := range []models.Record{{Client: "client-a", Time: oldTime}, {Client: "client-a", Time: newTime}} {
|
||||
if err := db.Create(&record).Error; err != nil {
|
||||
t.Fatalf("seed load record: %v", err)
|
||||
}
|
||||
}
|
||||
for _, record := range []models.GPURecord{{Client: "client-a", Time: oldTime}, {Client: "client-a", Time: newTime}} {
|
||||
if err := db.Create(&record).Error; err != nil {
|
||||
t.Fatalf("seed GPU record: %v", err)
|
||||
}
|
||||
}
|
||||
for _, record := range []models.PingRecord{{Client: "client-a", TaskId: 1, Time: oldTime}, {Client: "client-a", TaskId: 1, Time: newTime}} {
|
||||
if err := db.Create(&record).Error; err != nil {
|
||||
t.Fatalf("seed ping record: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
deleted, err := DeleteLegacyMonitoringBefore(db, cutoff)
|
||||
projection, err := legacyRecordProjection(db, "records_minimal")
|
||||
if err != nil {
|
||||
t.Fatalf("delete legacy monitoring before cutoff: %v", err)
|
||||
t.Fatalf("build legacy projection: %v", err)
|
||||
}
|
||||
if deleted.LoadRows != 1 || deleted.GPURows != 1 || deleted.LatencyRows != 1 {
|
||||
t.Fatalf("unexpected deleted rows: %#v", deleted)
|
||||
var record models.Record
|
||||
if err := db.Raw(projection).Scan(&record).Error; err != nil {
|
||||
t.Fatalf("scan projected legacy row: %v", err)
|
||||
}
|
||||
summary, err := InspectLegacyMonitoring(db)
|
||||
if err != nil {
|
||||
t.Fatalf("inspect remaining legacy monitoring: %v", err)
|
||||
if record.Client != "node-a" || !record.Time.Equal(base) || record.Cpu != 42.5 {
|
||||
t.Fatalf("unexpected projected record: %#v", record)
|
||||
}
|
||||
if summary.LoadRows != 1 || summary.GPURows != 1 || summary.LatencyRows != 1 || summary.ServerCount != 1 {
|
||||
t.Fatalf("unexpected remaining legacy rows: %#v", summary)
|
||||
}
|
||||
var clients int64
|
||||
if err := db.Model(&models.Client{}).Count(&clients).Error; err != nil || clients != 1 {
|
||||
t.Fatalf("cleanup changed clients: count=%d err=%v", clients, err)
|
||||
if record.ConnectionsUdp != 0 || record.NetTotalDown != 0 {
|
||||
t.Fatalf("missing metric columns were not zero-filled: %#v", record)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -3,13 +3,13 @@ package migrations
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"log"
|
||||
logger "github.com/komari-monitor/komari/utils/log"
|
||||
"reflect"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/komari-monitor/komari/database/models"
|
||||
appconfig "github.com/komari-monitor/komari/pkg/config"
|
||||
appconfig "github.com/komari-monitor/komari/internal/config"
|
||||
"gorm.io/gorm"
|
||||
"gorm.io/gorm/clause"
|
||||
)
|
||||
@@ -150,6 +150,7 @@ func migrateRemovedCompatibilityConfig(db *gorm.DB) error {
|
||||
return db.Delete(&appconfig.ConfigItem{}, "key IN ?", []string{
|
||||
"nezha_compat_enabled",
|
||||
"nezha_compat_listen",
|
||||
"low_resource_mode",
|
||||
}).Error
|
||||
}
|
||||
|
||||
@@ -176,7 +177,7 @@ func hasTableColumn(db *gorm.DB, tableName, columnName string) bool {
|
||||
|
||||
func migrateLegacyLoadNotification(db *gorm.DB) error {
|
||||
if db.Migrator().HasColumn(&models.LoadNotification{}, "client") {
|
||||
log.Println("[>0.1.4] Rebuilding LoadNotification table....")
|
||||
logger.InfoArgs("migration", "[>0.1.4] Rebuilding LoadNotification table....")
|
||||
return db.Migrator().DropTable(&models.LoadNotification{})
|
||||
}
|
||||
return nil
|
||||
@@ -187,7 +188,7 @@ func migrateLegacyOidcConfig(db *gorm.DB) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
log.Println("[>1.0.2] Merge OidcProvider table....")
|
||||
logger.InfoArgs("migration", "[>1.0.2] Merge OidcProvider table....")
|
||||
var oldData struct {
|
||||
OAuthClientID string `gorm:"column:o_auth_client_id"`
|
||||
OAuthClientSecret string `gorm:"column:o_auth_client_secret"`
|
||||
@@ -224,7 +225,7 @@ func migrateLegacyMessageSenderConfig(db *gorm.DB) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
log.Println("[>1.0.2] Migrate MessageSender configuration....")
|
||||
logger.InfoArgs("migration", "[>1.0.2] Migrate MessageSender configuration....")
|
||||
var oldData struct {
|
||||
TelegramBotToken string `gorm:"column:telegram_bot_token"`
|
||||
TelegramChatID string `gorm:"column:telegram_chat_id"`
|
||||
@@ -308,7 +309,7 @@ func saveLegacyMessageSenderConfig(db *gorm.DB, name string, config map[string]i
|
||||
}
|
||||
|
||||
func migrateLegacyConfigToItems(db *gorm.DB) error {
|
||||
log.Println("[>1.1.4] Moving legacy config data...")
|
||||
logger.InfoArgs("migration", "[>1.1.4] Moving legacy config data...")
|
||||
|
||||
var oldData legacyConfig
|
||||
if err := db.Order("id desc").First(&oldData).Error; err != nil {
|
||||
@@ -371,7 +372,7 @@ func migrateLegacyClientInfo(db *gorm.DB) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
log.Println("[>0.0.5] Legacy ClientInfo table detected, starting data migration...")
|
||||
logger.InfoArgs("migration", "[>0.0.5] Legacy ClientInfo table detected, starting data migration...")
|
||||
if err := db.AutoMigrate(&models.Client{}); err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -384,7 +385,7 @@ func migrateLegacyClientInfo(db *gorm.DB) error {
|
||||
for _, info := range clientInfos {
|
||||
var client models.Client
|
||||
if err := db.Where("uuid = ?", info.UUID).First(&client).Error; err != nil {
|
||||
log.Printf("Could not find Client record with UUID %s: %v", info.UUID, err)
|
||||
logger.Errorf("migration", "Could not find Client record with UUID %s: %v", info.UUID, err)
|
||||
continue
|
||||
}
|
||||
|
||||
@@ -416,7 +417,7 @@ func migrateLegacyClientInfo(db *gorm.DB) error {
|
||||
if err := db.Migrator().RenameTable("client_infos", "client_infos_backup"); err != nil {
|
||||
return fmt.Errorf("backup legacy ClientInfo table: %w", err)
|
||||
}
|
||||
log.Println("Data migration completed, old table has been backed up as client_infos_backup")
|
||||
logger.InfoArgs("migration", "Data migration completed, old table has been backed up as client_infos_backup")
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -7,7 +7,7 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/komari-monitor/komari/database/models"
|
||||
appconfig "github.com/komari-monitor/komari/pkg/config"
|
||||
appconfig "github.com/komari-monitor/komari/internal/config"
|
||||
"gorm.io/driver/sqlite"
|
||||
"gorm.io/gorm"
|
||||
)
|
||||
@@ -104,7 +104,7 @@ func TestRunRemovesCompatibilityConfig(t *testing.T) {
|
||||
if err := db.AutoMigrate(&appconfig.ConfigItem{}); err != nil {
|
||||
t.Fatalf("migrate config item table: %v", err)
|
||||
}
|
||||
for _, key := range []string{"nezha_compat_enabled", "nezha_compat_listen"} {
|
||||
for _, key := range []string{"nezha_compat_enabled", "nezha_compat_listen", "low_resource_mode"} {
|
||||
if err := db.Create(&appconfig.ConfigItem{Key: key, Value: "true"}).Error; err != nil {
|
||||
t.Fatalf("seed removed config %q: %v", key, err)
|
||||
}
|
||||
@@ -121,6 +121,7 @@ func TestRunRemovesCompatibilityConfig(t *testing.T) {
|
||||
if err := db.Model(&appconfig.ConfigItem{}).Where("key IN ?", []string{
|
||||
"nezha_compat_enabled",
|
||||
"nezha_compat_listen",
|
||||
"low_resource_mode",
|
||||
}).Count(&count).Error; err != nil {
|
||||
t.Fatalf("count removed config: %v", err)
|
||||
}
|
||||
@@ -3,14 +3,14 @@ package migrations
|
||||
import (
|
||||
"database/sql"
|
||||
"fmt"
|
||||
"log"
|
||||
logger "github.com/komari-monitor/komari/utils/log"
|
||||
"math"
|
||||
"os"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
appconfig "github.com/komari-monitor/komari/pkg/config"
|
||||
appconfig "github.com/komari-monitor/komari/internal/config"
|
||||
"gorm.io/gorm"
|
||||
"gorm.io/gorm/clause"
|
||||
)
|
||||
@@ -90,7 +90,7 @@ func migrateLegacyTimestampColumns(db *gorm.DB) error {
|
||||
return err
|
||||
}
|
||||
if converted > 0 {
|
||||
log.Printf("Converted %d legacy timestamp values to explicit UTC", converted)
|
||||
logger.Infof("migration", "Converted %d legacy timestamp values to explicit UTC", converted)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -204,7 +204,7 @@ func legacyTimestampLocation() *time.Location {
|
||||
}
|
||||
location, err := time.LoadLocation(name)
|
||||
if err != nil {
|
||||
log.Printf("Legacy timezone %q cannot be loaded; interpreting old timestamps as UTC: %v", name, err)
|
||||
logger.Infof("migration", "Legacy timezone %q cannot be loaded; interpreting old timestamps as UTC: %v", name, err)
|
||||
return time.UTC
|
||||
}
|
||||
return location
|
||||
File renamed without changes.
@@ -0,0 +1,118 @@
|
||||
package plugin
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
|
||||
"github.com/komari-monitor/komari/database/dbcore"
|
||||
"github.com/komari-monitor/komari/database/models"
|
||||
"gorm.io/gorm"
|
||||
)
|
||||
|
||||
// GetConfiguration returns the saved configuration values of one plugin,
|
||||
// merged with the manifest-declared defaults for keys that were never saved.
|
||||
// This mirrors the theme configuration behavior so plugins and the admin UI
|
||||
// always see a complete value set. Missing rows yield only defaults; a
|
||||
// missing manifest yields only the saved values.
|
||||
func GetConfiguration(short string) (map[string]any, error) {
|
||||
values, err := savedConfiguration(short)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
info, err := readManifest(filepath.Join(DataDir, short))
|
||||
if err != nil {
|
||||
return values, nil // not installed (or unreadable): keep saved values
|
||||
}
|
||||
mergeConfigurationDefaults(values, info.Configuration)
|
||||
return values, nil
|
||||
}
|
||||
|
||||
func savedConfiguration(short string) (map[string]any, error) {
|
||||
db := dbcore.GetDBInstance()
|
||||
var cfg models.PluginConfiguration
|
||||
if err := db.Where("short = ?", short).First(&cfg).Error; err != nil {
|
||||
if errors.Is(err, gorm.ErrRecordNotFound) {
|
||||
return map[string]any{}, nil
|
||||
}
|
||||
return nil, fmt.Errorf("read plugin configuration: %w", err)
|
||||
}
|
||||
var data map[string]any
|
||||
if err := json.Unmarshal([]byte(cfg.Data), &data); err != nil || data == nil {
|
||||
return map[string]any{}, nil
|
||||
}
|
||||
return data, nil
|
||||
}
|
||||
|
||||
// mergeConfigurationDefaults fills manifest-declared defaults for keys that
|
||||
// are missing from the saved values, following the theme merge rules.
|
||||
func mergeConfigurationDefaults(values map[string]any, configuration models.Configuration) {
|
||||
if configuration.Type != models.ThemeConfigurationManaged {
|
||||
return
|
||||
}
|
||||
raw, err := json.Marshal(configuration.Data)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
var items []models.ManagedThemeConfigurationItem
|
||||
if err := json.Unmarshal(raw, &items); err != nil {
|
||||
return
|
||||
}
|
||||
for _, item := range items {
|
||||
if item.Key == "" {
|
||||
continue
|
||||
}
|
||||
if _, exists := values[item.Key]; exists {
|
||||
continue
|
||||
}
|
||||
values[item.Key] = configurationDefault(item)
|
||||
}
|
||||
}
|
||||
|
||||
// configurationDefault resolves one item's effective default, mirroring the
|
||||
// theme rules: select falls back to its first option; nil defaults become
|
||||
// 0 for number, false for switch and "" otherwise.
|
||||
func configurationDefault(item models.ManagedThemeConfigurationItem) any {
|
||||
def := item.Default
|
||||
if item.Type == "select" {
|
||||
if def == nil || def == "" {
|
||||
if item.Options != "" {
|
||||
opts := strings.Split(item.Options, ",")
|
||||
if len(opts) > 0 {
|
||||
return strings.TrimSpace(opts[0])
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if def == nil {
|
||||
switch item.Type {
|
||||
case "number":
|
||||
return float64(0) // JSON numbers decode as float64
|
||||
case "switch":
|
||||
return false
|
||||
default:
|
||||
return ""
|
||||
}
|
||||
}
|
||||
return def
|
||||
}
|
||||
|
||||
// SaveConfiguration upserts the configuration values of one plugin.
|
||||
func SaveConfiguration(short string, data map[string]any) error {
|
||||
if data == nil {
|
||||
data = map[string]any{}
|
||||
}
|
||||
raw, err := json.Marshal(data)
|
||||
if err != nil {
|
||||
return fmt.Errorf("marshal plugin configuration: %w", err)
|
||||
}
|
||||
db := dbcore.GetDBInstance()
|
||||
if err := db.Where("short = ?", short).
|
||||
Assign(models.PluginConfiguration{Short: short, Data: string(raw)}).
|
||||
FirstOrCreate(&models.PluginConfiguration{}).Error; err != nil {
|
||||
return fmt.Errorf("save plugin configuration: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,97 @@
|
||||
package plugin
|
||||
|
||||
import (
|
||||
"net"
|
||||
"net/http"
|
||||
"strings"
|
||||
|
||||
"github.com/dop251/goja"
|
||||
"github.com/gin-gonic/gin"
|
||||
"github.com/komari-monitor/komari/pkg/rpc"
|
||||
)
|
||||
|
||||
// principalContextKey stays in sync with web/api/principal.go:
|
||||
// IdentityMiddleware stores the resolved principal under this gin key.
|
||||
const principalContextKey = "principal"
|
||||
|
||||
// routeRequestContext builds the JS request context with the caller identity
|
||||
// resolved by IdentityMiddleware. Raw credentials (session/client tokens) are
|
||||
// intentionally not exposed.
|
||||
func routeRequestContext(vm *goja.Runtime, c *gin.Context) *goja.Object {
|
||||
ctx := vm.NewObject()
|
||||
principal := rpcPrincipalFromGin(c)
|
||||
p := vm.NewObject()
|
||||
if principal != nil {
|
||||
_ = p.Set("type", principalTypeString(principal.Type))
|
||||
_ = p.Set("roles", principal.Roles)
|
||||
_ = p.Set("user_uuid", principal.UserUUID)
|
||||
_ = p.Set("client_uuid", principal.ClientUUID)
|
||||
_ = p.Set("is_api_key", principal.IsAPIKey)
|
||||
}
|
||||
_ = ctx.Set("principal", p)
|
||||
if role, ok := c.Get("role"); ok {
|
||||
if s, ok := role.(string); ok {
|
||||
_ = ctx.Set("role", s)
|
||||
}
|
||||
}
|
||||
if uuid, ok := c.Get("uuid"); ok {
|
||||
_ = ctx.Set("user_uuid", uuid)
|
||||
}
|
||||
if clientUUID, ok := c.Get("client_uuid"); ok {
|
||||
_ = ctx.Set("client_uuid", clientUUID)
|
||||
}
|
||||
_ = ctx.Set("remote_ip", c.ClientIP())
|
||||
_ = ctx.Set("user_agent", c.GetHeader("User-Agent"))
|
||||
return ctx
|
||||
}
|
||||
|
||||
// hookRequestContext builds the JS request context at the raw HTTP layer
|
||||
// (before the gin identity middleware runs): only network metadata is
|
||||
// available there.
|
||||
func hookRequestContext(vm *goja.Runtime, r *http.Request) *goja.Object {
|
||||
ctx := vm.NewObject()
|
||||
_ = ctx.Set("remote_ip", remoteIP(r))
|
||||
_ = ctx.Set("user_agent", r.Header.Get("User-Agent"))
|
||||
return ctx
|
||||
}
|
||||
|
||||
func rpcPrincipalFromGin(c *gin.Context) *rpc.Principal {
|
||||
value, ok := c.Get(principalContextKey)
|
||||
if !ok {
|
||||
return nil
|
||||
}
|
||||
principal, _ := value.(*rpc.Principal)
|
||||
return principal
|
||||
}
|
||||
|
||||
func principalTypeString(t rpc.PrincipalType) string {
|
||||
switch t {
|
||||
case rpc.PrincipalAgent:
|
||||
return "agent"
|
||||
case rpc.PrincipalUser:
|
||||
return "user"
|
||||
case rpc.PrincipalAPIKey:
|
||||
return "api_key"
|
||||
default:
|
||||
return "anonymous"
|
||||
}
|
||||
}
|
||||
|
||||
// remoteIP mirrors gin's ClientIP for the raw request layer: first
|
||||
// X-Forwarded-For entry, then X-Real-IP, then the RemoteAddr host.
|
||||
func remoteIP(r *http.Request) string {
|
||||
if forwarded := r.Header.Get("X-Forwarded-For"); forwarded != "" {
|
||||
if first, _, ok := strings.Cut(forwarded, ","); ok {
|
||||
return strings.TrimSpace(first)
|
||||
}
|
||||
return strings.TrimSpace(forwarded)
|
||||
}
|
||||
if real := r.Header.Get("X-Real-IP"); real != "" {
|
||||
return strings.TrimSpace(real)
|
||||
}
|
||||
host, _, err := net.SplitHostPort(r.RemoteAddr)
|
||||
if err != nil {
|
||||
return r.RemoteAddr
|
||||
}
|
||||
return host
|
||||
}
|
||||
@@ -0,0 +1,72 @@
|
||||
package plugin
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"sync/atomic"
|
||||
|
||||
"github.com/dop251/goja"
|
||||
"github.com/komari-monitor/komari/internal/scheduler"
|
||||
)
|
||||
|
||||
// cronJobSeq gives every registered cron job a process-unique id. Job names
|
||||
// stay unique across plugin reloads because unload cancels every job of the
|
||||
// plugin before a new load registers replacements.
|
||||
var cronJobSeq atomic.Uint64
|
||||
|
||||
// registerCron registers one cron job for a plugin. The handler is a
|
||||
// JavaScript function that runs on the plugin's own event loop each time the
|
||||
// schedule fires. It is called from the plugin's own event loop during script
|
||||
// evaluation and takes the manager lock itself; the critical section is short
|
||||
// and never waits on the event loop, so no lock cycle is possible. The job is
|
||||
// removed automatically when the plugin is unloaded or fails to load.
|
||||
func (m *Manager) registerCron(short, spec string, fn goja.Callable) error {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
inst, ok := m.instances[short]
|
||||
if !ok {
|
||||
return fmt.Errorf("plugin %q is not loaded", short)
|
||||
}
|
||||
inst.mu.RLock()
|
||||
host := inst.host
|
||||
alive := inst.runtime != nil
|
||||
inst.mu.RUnlock()
|
||||
if !alive || host == nil {
|
||||
return fmt.Errorf("plugin %q runtime is not ready", short)
|
||||
}
|
||||
|
||||
name := fmt.Sprintf("plugin:%s:%d", short, cronJobSeq.Add(1))
|
||||
if err := scheduler.AddContextFunc(name, spec, false, func(ctx context.Context) {
|
||||
m.runCronJob(short, name, fn)
|
||||
}); err != nil {
|
||||
return err
|
||||
}
|
||||
inst.mu.Lock()
|
||||
inst.cronJobs = append(inst.cronJobs, name)
|
||||
inst.mu.Unlock()
|
||||
return nil
|
||||
}
|
||||
|
||||
// runCronJob bridges one scheduled fire into the plugin event loop. It runs
|
||||
// on a scheduler goroutine and drops the fire when the plugin was unloaded
|
||||
// (the job is cancelled with the instance, but a fire may already be queued).
|
||||
// Errors thrown by the handler are reported to the plugin log by RunJob.
|
||||
func (m *Manager) runCronJob(short, name string, fn goja.Callable) {
|
||||
inst := m.instanceFor(short)
|
||||
if inst == nil {
|
||||
return
|
||||
}
|
||||
inst.mu.RLock()
|
||||
host := inst.host
|
||||
alive := inst.runtime != nil
|
||||
inst.mu.RUnlock()
|
||||
if !alive || host == nil {
|
||||
return
|
||||
}
|
||||
host.RunOnLoop(func(vm *goja.Runtime) {
|
||||
_ = host.RunJob(vm, "plugin cron "+name, func() error {
|
||||
_, err := fn(goja.Undefined())
|
||||
return err
|
||||
})
|
||||
})
|
||||
}
|
||||
@@ -0,0 +1,842 @@
|
||||
package plugin
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/gin-gonic/gin"
|
||||
"github.com/komari-monitor/komari/pkg/rpc"
|
||||
)
|
||||
|
||||
const featureManifest = `{"name":"Feat","short":"feat","version":"1.0.0","permissions":{"timeout":5,"allowRoutes":true,"allowHooks":true}}`
|
||||
|
||||
func TestHookRequestAndResponse(t *testing.T) {
|
||||
withTempDataDir(t)
|
||||
gin.SetMode(gin.TestMode)
|
||||
engine := gin.New()
|
||||
engine.GET("/ping", func(c *gin.Context) {
|
||||
if c.GetHeader("x-hooked") != "yes" {
|
||||
c.String(http.StatusBadRequest, "missing x-hooked")
|
||||
return
|
||||
}
|
||||
c.String(http.StatusOK, "pong")
|
||||
})
|
||||
Init(engine)
|
||||
|
||||
zipPath := writePluginZip(t, map[string]string{
|
||||
"komari-plugin.json": featureManifest,
|
||||
"script.js": `
|
||||
const server = require("server");
|
||||
function load() {
|
||||
server.hook("request", (req) => {
|
||||
req.headers["x-hooked"] = "yes";
|
||||
});
|
||||
server.hook("response", (req, res) => {
|
||||
res.statusCode = 201;
|
||||
res.headers["x-res"] = "1";
|
||||
res.body = res.body + "|hooked";
|
||||
});
|
||||
}
|
||||
`,
|
||||
})
|
||||
if _, err := InstallZip(zipPath); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := SetEnabled("feat", true, true); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// 用 WrapHandler 包裹(对应 internal/server 的接入点)
|
||||
wrapped := WrapHandler(engine)
|
||||
|
||||
rec := httptest.NewRecorder()
|
||||
wrapped.ServeHTTP(rec, httptest.NewRequest(http.MethodGet, "/ping", nil))
|
||||
if rec.Code != http.StatusCreated {
|
||||
t.Fatalf("status = %d, body = %s", rec.Code, rec.Body.String())
|
||||
}
|
||||
if rec.Body.String() != "pong|hooked" {
|
||||
t.Fatalf("body = %q", rec.Body.String())
|
||||
}
|
||||
if rec.Header().Get("x-res") != "1" {
|
||||
t.Fatalf("headers = %v", rec.Header())
|
||||
}
|
||||
|
||||
// 卸载后 hook 移除:请求不再带 x-hooked,handler 返回 400
|
||||
if err := SetEnabled("feat", false, false); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
rec2 := httptest.NewRecorder()
|
||||
wrapped.ServeHTTP(rec2, httptest.NewRequest(http.MethodGet, "/ping", nil))
|
||||
if rec2.Code != http.StatusBadRequest || !strings.Contains(rec2.Body.String(), "missing x-hooked") {
|
||||
t.Fatalf("after unload status = %d, body = %q", rec2.Code, rec2.Body.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestHookUpgradePassesThrough(t *testing.T) {
|
||||
withTempDataDir(t)
|
||||
gin.SetMode(gin.TestMode)
|
||||
engine := gin.New()
|
||||
engine.GET("/ws", func(c *gin.Context) {
|
||||
c.String(http.StatusOK, "direct")
|
||||
})
|
||||
Init(engine)
|
||||
|
||||
zipPath := writePluginZip(t, map[string]string{
|
||||
"komari-plugin.json": featureManifest,
|
||||
"script.js": `
|
||||
const server = require("server");
|
||||
function load() {
|
||||
server.hook("request", (req) => { req.headers["x"] = "y"; });
|
||||
server.hook("response", (req, res) => { res.body = "changed"; });
|
||||
}
|
||||
`,
|
||||
})
|
||||
if _, err := InstallZip(zipPath); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := SetEnabled("feat", true, true); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
rec := httptest.NewRecorder()
|
||||
req := httptest.NewRequest(http.MethodGet, "/ws", nil)
|
||||
req.Header.Set("Connection", "upgrade")
|
||||
req.Header.Set("Upgrade", "websocket")
|
||||
WrapHandler(engine).ServeHTTP(rec, req)
|
||||
if rec.Code != http.StatusOK || rec.Body.String() != "direct" {
|
||||
t.Fatalf("upgrade passthrough failed: status = %d, body = %q", rec.Code, rec.Body.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestRegisterRPCCallableThroughRegistry(t *testing.T) {
|
||||
withTempDataDir(t)
|
||||
gin.SetMode(gin.TestMode)
|
||||
Init(gin.New())
|
||||
|
||||
zipPath := writePluginZip(t, map[string]string{
|
||||
"komari-plugin.json": featureManifest,
|
||||
"script.js": `
|
||||
const server = require("server");
|
||||
function load() {
|
||||
server.registerRPC("plugin:featEcho", (params) => {
|
||||
return { echo: params, from: "feat" };
|
||||
});
|
||||
}
|
||||
`,
|
||||
})
|
||||
if _, err := InstallZip(zipPath); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := SetEnabled("feat", true, true); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer func() { _ = SetEnabled("feat", false, false) }()
|
||||
|
||||
resp := rpc.CallWithContext(context.Background(), nil, "plugin:featEcho", map[string]any{"x": 1})
|
||||
if resp.Error != nil {
|
||||
t.Fatalf("rpc error: %v", resp.Error)
|
||||
}
|
||||
result, ok := resp.Result.(map[string]any)
|
||||
if !ok || result["from"] != "feat" {
|
||||
t.Fatalf("result = %v", resp.Result)
|
||||
}
|
||||
echo, ok := result["echo"].(map[string]any)
|
||||
if !ok || fmt.Sprint(echo["x"]) != "1" {
|
||||
t.Fatalf("echo = %v", result["echo"])
|
||||
}
|
||||
|
||||
// 卸载后方法注销
|
||||
if err := SetEnabled("feat", false, false); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
resp2 := rpc.CallWithContext(context.Background(), nil, "plugin:featEcho", nil)
|
||||
if resp2.Error == nil || resp2.Error.Code != rpc.MethodNotFound {
|
||||
t.Fatalf("expected MethodNotFound after unload, got %v", resp2.Error)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRegisterRPCErrorCarriesCode(t *testing.T) {
|
||||
withTempDataDir(t)
|
||||
gin.SetMode(gin.TestMode)
|
||||
Init(gin.New())
|
||||
|
||||
zipPath := writePluginZip(t, map[string]string{
|
||||
"komari-plugin.json": featureManifest,
|
||||
"script.js": `
|
||||
const server = require("server");
|
||||
function load() {
|
||||
server.registerRPC("plugin:featFail", () => {
|
||||
const err = new Error("boom");
|
||||
err.code = -32045;
|
||||
err.data = { detail: "x" };
|
||||
throw err;
|
||||
});
|
||||
}
|
||||
`,
|
||||
})
|
||||
if _, err := InstallZip(zipPath); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := SetEnabled("feat", true, true); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer func() { _ = SetEnabled("feat", false, false) }()
|
||||
|
||||
resp := rpc.CallWithContext(context.Background(), nil, "plugin:featFail", nil)
|
||||
if resp.Error == nil {
|
||||
t.Fatal("expected error")
|
||||
}
|
||||
if resp.Error.Code != -32045 || resp.Error.Message != "boom" {
|
||||
t.Fatalf("error = %+v", resp.Error)
|
||||
}
|
||||
data, ok := resp.Error.Data.(map[string]any)
|
||||
if !ok || data["detail"] != "x" {
|
||||
t.Fatalf("error data = %v", resp.Error.Data)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDeleteRemovesPlugin(t *testing.T) {
|
||||
withTempDataDir(t)
|
||||
gin.SetMode(gin.TestMode)
|
||||
engine := gin.New()
|
||||
Init(engine)
|
||||
|
||||
zipPath := writePluginZip(t, map[string]string{
|
||||
"komari-plugin.json": featureManifest,
|
||||
"script.js": `function load() { console.log("hi"); }`,
|
||||
})
|
||||
if _, err := InstallZip(zipPath); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := SetEnabled("feat", true, true); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := Delete("feat"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if infos := List(); len(infos) != 0 {
|
||||
t.Fatalf("List = %+v", infos)
|
||||
}
|
||||
if _, err := Manifest("feat"); err == nil {
|
||||
t.Fatal("manifest should be gone")
|
||||
}
|
||||
if err := Delete("feat"); err == nil {
|
||||
t.Fatal("second delete should fail")
|
||||
}
|
||||
}
|
||||
|
||||
// TestCronRunsOnScheduleAndStopsOnUnload 验证 server.cron 按表达式在插件事件
|
||||
// 循环上执行 handler,且卸载后任务被移除、不再触发。
|
||||
func TestCronRunsOnScheduleAndStopsOnUnload(t *testing.T) {
|
||||
withTempDataDir(t)
|
||||
gin.SetMode(gin.TestMode)
|
||||
Init(gin.New())
|
||||
|
||||
zipPath := writePluginZip(t, map[string]string{
|
||||
"komari-plugin.json": `{"name":"Cron","short":"cron","version":"1.0.0","permissions":{"node":true,"timeout":5}}`,
|
||||
"script.js": `
|
||||
const fs = require("fs");
|
||||
const server = require("server");
|
||||
function load() {
|
||||
server.cron("@every 100ms", () => {
|
||||
fs.appendFileSync("tick.txt", "x");
|
||||
});
|
||||
}
|
||||
`,
|
||||
})
|
||||
if _, err := InstallZip(zipPath); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := SetEnabled("cron", true, true); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
tickFile := filepath.Join(DataDir, "cron", "tick.txt")
|
||||
waitFor := func(minTicks int) int {
|
||||
deadline := time.Now().Add(5 * time.Second)
|
||||
for time.Now().Before(deadline) {
|
||||
if data, err := os.ReadFile(tickFile); err == nil && len(data) >= minTicks {
|
||||
return len(data)
|
||||
}
|
||||
time.Sleep(20 * time.Millisecond)
|
||||
}
|
||||
t.Fatalf("cron did not fire within 5s (need at least %d ticks)", minTicks)
|
||||
return 0
|
||||
}
|
||||
if ticks := waitFor(3); ticks == 0 {
|
||||
t.Fatal("cron fired but wrote no ticks")
|
||||
}
|
||||
|
||||
// 卸载后任务被移除:两个采样窗口之间的计数不得再增长。
|
||||
if err := SetEnabled("cron", false, false); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
time.Sleep(300 * time.Millisecond)
|
||||
first, _ := os.ReadFile(tickFile)
|
||||
time.Sleep(300 * time.Millisecond)
|
||||
second, _ := os.ReadFile(tickFile)
|
||||
if len(second) != len(first) {
|
||||
t.Fatalf("cron kept firing after unload: %d -> %d", len(first), len(second))
|
||||
}
|
||||
}
|
||||
|
||||
// TestCronInvalidSpecFailsLoad 验证非法 cron 表达式使插件加载失败并自动禁用。
|
||||
func TestCronInvalidSpecFailsLoad(t *testing.T) {
|
||||
withTempDataDir(t)
|
||||
gin.SetMode(gin.TestMode)
|
||||
Init(gin.New())
|
||||
|
||||
zipPath := writePluginZip(t, map[string]string{
|
||||
"komari-plugin.json": `{"name":"Cron","short":"cron","version":"1.0.0","permissions":{"node":true,"timeout":5}}`,
|
||||
"script.js": `
|
||||
const server = require("server");
|
||||
function load() {
|
||||
server.cron("not a cron spec", () => {});
|
||||
}
|
||||
`,
|
||||
})
|
||||
if _, err := InstallZip(zipPath); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := SetEnabled("cron", true, true); err == nil {
|
||||
t.Fatal("invalid cron spec must fail the load")
|
||||
}
|
||||
infos := List()
|
||||
if len(infos) != 1 || infos[0].Enabled || !strings.Contains(infos[0].LastError, "invalid") {
|
||||
t.Fatalf("state after failed load = %+v", infos)
|
||||
}
|
||||
}
|
||||
|
||||
func TestConfigurationSaveAndGet(t *testing.T) {
|
||||
withTempDataDir(t)
|
||||
Init(gin.New())
|
||||
|
||||
if err := SaveConfiguration("feat", map[string]any{"greeting": "hello", "n": float64(3)}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
values, err := GetConfiguration("feat")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if values["greeting"] != "hello" || values["n"] != float64(3) {
|
||||
t.Fatalf("values = %v", values)
|
||||
}
|
||||
// 覆盖保存
|
||||
if err := SaveConfiguration("feat", map[string]any{"greeting": "bye"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
values, err = GetConfiguration("feat")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if values["greeting"] != "bye" || len(values) != 1 {
|
||||
t.Fatalf("values = %v", values)
|
||||
}
|
||||
}
|
||||
|
||||
// TestGetConfigurationMergesManifestDefaults 验证与主题一致的行为:未保存的
|
||||
// key 返回 manifest 声明的默认值(select 取首个选项、number->0、switch->false、
|
||||
// string->""),已保存的 key 保留保存值。
|
||||
func TestGetConfigurationMergesManifestDefaults(t *testing.T) {
|
||||
withTempDataDir(t)
|
||||
zipPath := writePluginZip(t, map[string]string{
|
||||
"komari-plugin.json": `{"name":"Cfg","short":"cfg","version":"1.0.0","configuration":{"type":"managed","data":[
|
||||
{"key":"greeting","name":"Greeting","type":"string","default":"Hello"},
|
||||
{"key":"count","name":"Count","type":"number"},
|
||||
{"key":"enabled","name":"Enabled","type":"switch","default":true},
|
||||
{"key":"mode","name":"Mode","type":"select","options":"json,text"},
|
||||
{"key":"note","name":"Note","type":"string"}
|
||||
]}}`,
|
||||
"script.js": `function load() {}`,
|
||||
})
|
||||
if _, err := InstallZip(zipPath); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// 未保存任何值时,全部返回默认值。
|
||||
values, err := GetConfiguration("cfg")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if values["greeting"] != "Hello" {
|
||||
t.Errorf("greeting default = %v", values["greeting"])
|
||||
}
|
||||
if values["count"] != float64(0) {
|
||||
t.Errorf("count default = %v", values["count"])
|
||||
}
|
||||
if values["enabled"] != true {
|
||||
t.Errorf("enabled default = %v", values["enabled"])
|
||||
}
|
||||
if values["mode"] != "json" {
|
||||
t.Errorf("mode default = %v", values["mode"])
|
||||
}
|
||||
if values["note"] != "" {
|
||||
t.Errorf("note default = %v", values["note"])
|
||||
}
|
||||
|
||||
// 保存部分值:缺失 key 仍回退默认值,已保存 key 不被默认覆盖。
|
||||
if err := SaveConfiguration("cfg", map[string]any{"greeting": "saved"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
values, err = GetConfiguration("cfg")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if values["greeting"] != "saved" {
|
||||
t.Errorf("greeting saved = %v", values["greeting"])
|
||||
}
|
||||
if values["mode"] != "json" {
|
||||
t.Errorf("mode still default = %v", values["mode"])
|
||||
}
|
||||
}
|
||||
func TestGetConfigFromScript(t *testing.T) {
|
||||
withTempDataDir(t)
|
||||
gin.SetMode(gin.TestMode)
|
||||
Init(gin.New())
|
||||
|
||||
if err := SaveConfiguration("feat", map[string]any{"greeting": "saved-value"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
zipPath := writePluginZip(t, map[string]string{
|
||||
"komari-plugin.json": featureManifest,
|
||||
"script.js": `
|
||||
const server = require("server");
|
||||
function load() {
|
||||
server.route("GET", "/cfg", async (req, res) => {
|
||||
const cfg = await server.getConfig();
|
||||
res.end(JSON.stringify(cfg));
|
||||
});
|
||||
}
|
||||
`,
|
||||
})
|
||||
if _, err := InstallZip(zipPath); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := SetEnabled("feat", true, true); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer func() { _ = SetEnabled("feat", false, false) }()
|
||||
|
||||
rec := httptest.NewRecorder()
|
||||
engine := global.engine
|
||||
engine.ServeHTTP(rec, httptest.NewRequest(http.MethodGet, "/cfg", nil))
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("status = %d, body = %s", rec.Code, rec.Body.String())
|
||||
}
|
||||
var body map[string]any
|
||||
if err := json.Unmarshal(rec.Body.Bytes(), &body); err != nil || body["greeting"] != "saved-value" {
|
||||
t.Fatalf("body = %s", rec.Body.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestResolvePublicFileScopedToEnabledPublicPages(t *testing.T) {
|
||||
withTempDataDir(t)
|
||||
zipPath := writePluginZip(t, map[string]string{
|
||||
"komari-plugin.json": `{"name":"Pub","short":"pub","version":"1.0.0","pages":[{"file":"pages/pub.html","title":"Pub","visibility":"public"},{"file":"admin.html","title":"Admin"}]}`,
|
||||
"script.js": `function load() {}`,
|
||||
"pages/pub.html": `<h1>pub</h1>`,
|
||||
"pages/pub.js": `console.log("asset")`,
|
||||
"admin.html": `<h1>admin</h1>`,
|
||||
})
|
||||
if _, err := InstallZip(zipPath); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// 未启用:公开文件不可访问。
|
||||
if _, err := ResolvePublicFile("pub", "pages/pub.html"); err == nil {
|
||||
t.Fatal("disabled plugin public page must not resolve")
|
||||
}
|
||||
|
||||
// 启用后:页面与同目录资源可访问,admin 页面与目录外文件不可访问。
|
||||
if err := SetEnabled("pub", true, true); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := ResolvePublicFile("pub", "pages/pub.html"); err != nil {
|
||||
t.Fatalf("public page must resolve when enabled: %v", err)
|
||||
}
|
||||
if _, err := ResolvePublicFile("pub", "pages/pub.js"); err != nil {
|
||||
t.Fatalf("relative asset must resolve: %v", err)
|
||||
}
|
||||
for _, bad := range []string{"admin.html", "script.js", "../evil"} {
|
||||
if _, err := ResolvePublicFile("pub", bad); err == nil {
|
||||
t.Fatalf("ResolvePublicFile(%q) should fail", bad)
|
||||
}
|
||||
}
|
||||
}
|
||||
func TestResolveFileRejectsTraversal(t *testing.T) {
|
||||
withTempDataDir(t)
|
||||
zipPath := writePluginZip(t, map[string]string{
|
||||
"komari-plugin.json": featureManifest,
|
||||
"script.js": `function load() {}`,
|
||||
"pages/hello.html": `<h1>hello</h1>`,
|
||||
})
|
||||
if _, err := InstallZip(zipPath); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
full, err := ResolveFile("feat", "pages/hello.html")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !strings.HasSuffix(full, "hello.html") {
|
||||
t.Fatalf("path = %q", full)
|
||||
}
|
||||
for _, bad := range []string{"../evil", "/abs", "..", "pages/../evil"} {
|
||||
if _, err := ResolveFile("feat", bad); err == nil {
|
||||
t.Fatalf("ResolveFile(%q) should fail", bad)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestRouteRequestContextCarriesIdentity(t *testing.T) {
|
||||
withTempDataDir(t)
|
||||
gin.SetMode(gin.TestMode)
|
||||
engine := gin.New()
|
||||
// 模拟 IdentityMiddleware:写入 principal/role/uuid(与 web/api 一致)
|
||||
engine.Use(func(c *gin.Context) {
|
||||
c.Set("principal", rpc.NewUserPrincipal("user-1"))
|
||||
c.Set("role", "admin")
|
||||
c.Set("uuid", "user-1")
|
||||
c.Next()
|
||||
})
|
||||
Init(engine)
|
||||
|
||||
zipPath := writePluginZip(t, map[string]string{
|
||||
"komari-plugin.json": featureManifest,
|
||||
"script.js": `
|
||||
const server = require("server");
|
||||
function load() {
|
||||
server.route("GET", "/who", (req, res) => {
|
||||
res.end(JSON.stringify(req.context));
|
||||
});
|
||||
}
|
||||
`,
|
||||
})
|
||||
if _, err := InstallZip(zipPath); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := SetEnabled("feat", true, true); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer func() { _ = SetEnabled("feat", false, false) }()
|
||||
|
||||
rec := httptest.NewRecorder()
|
||||
req := httptest.NewRequest(http.MethodGet, "/who", nil)
|
||||
req.RemoteAddr = "203.0.113.9:1234"
|
||||
engine.ServeHTTP(rec, req)
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("status = %d, body = %s", rec.Code, rec.Body.String())
|
||||
}
|
||||
var body map[string]any
|
||||
if err := json.Unmarshal(rec.Body.Bytes(), &body); err != nil {
|
||||
t.Fatalf("body = %s", rec.Body.String())
|
||||
}
|
||||
principal, ok := body["principal"].(map[string]any)
|
||||
if !ok || principal["type"] != "user" || principal["user_uuid"] != "user-1" {
|
||||
t.Fatalf("principal = %v", body["principal"])
|
||||
}
|
||||
if body["role"] != "admin" || body["user_uuid"] != "user-1" {
|
||||
t.Fatalf("context = %v", body)
|
||||
}
|
||||
if body["remote_ip"] != "203.0.113.9" {
|
||||
t.Fatalf("remote_ip = %v", body["remote_ip"])
|
||||
}
|
||||
}
|
||||
|
||||
func TestHookRequestContextHasNetworkOnly(t *testing.T) {
|
||||
withTempDataDir(t)
|
||||
gin.SetMode(gin.TestMode)
|
||||
engine := gin.New()
|
||||
engine.GET("/ping", func(c *gin.Context) { c.String(http.StatusOK, "pong") })
|
||||
Init(engine)
|
||||
|
||||
zipPath := writePluginZip(t, map[string]string{
|
||||
"komari-plugin.json": featureManifest,
|
||||
"script.js": `
|
||||
const server = require("server");
|
||||
function load() {
|
||||
server.hook("request", (req) => {
|
||||
req.headers["x-ip"] = req.context.remote_ip;
|
||||
});
|
||||
}
|
||||
`,
|
||||
})
|
||||
if _, err := InstallZip(zipPath); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := SetEnabled("feat", true, true); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer func() { _ = SetEnabled("feat", false, false) }()
|
||||
|
||||
engine.GET("/check", func(c *gin.Context) {
|
||||
if c.GetHeader("x-ip") != "198.51.100.7" {
|
||||
c.String(http.StatusBadRequest, "bad ip: "+c.GetHeader("x-ip"))
|
||||
return
|
||||
}
|
||||
c.String(http.StatusOK, "ok")
|
||||
})
|
||||
rec := httptest.NewRecorder()
|
||||
req := httptest.NewRequest(http.MethodGet, "/check", nil)
|
||||
req.RemoteAddr = "198.51.100.7:4567"
|
||||
WrapHandler(engine).ServeHTTP(rec, req)
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("status = %d, body = %s", rec.Code, rec.Body.String())
|
||||
}
|
||||
}
|
||||
|
||||
// TestServerModulePermissionEnforcement 验证 server 模块的权限门:
|
||||
// server.route/server.hook 缺权限时加载失败;server.call 缺权限时 Promise
|
||||
// reject;registerRPC/getConfig 始终可用;无危险权限的插件无需批准即可启用。
|
||||
func TestServerModulePermissionEnforcement(t *testing.T) {
|
||||
withTempDataDir(t)
|
||||
gin.SetMode(gin.TestMode)
|
||||
engine := gin.New()
|
||||
Init(engine)
|
||||
|
||||
// 无危险权限:直接启用,不要求批准。
|
||||
zipPath := writePluginZip(t, map[string]string{
|
||||
"komari-plugin.json": `{"name":"Plain","short":"plain","version":"1.0.0","permissions":{"node":true,"timeout":5}}`,
|
||||
"script.js": `const server = require("server"); function load() { server.registerRPC("plugin:plain", () => ({ok:true})); }`,
|
||||
})
|
||||
if _, err := InstallZip(zipPath); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := SetEnabled("plain", true, false); err != nil {
|
||||
t.Fatalf("plain plugin without dangerous permissions must enable without approval: %v", err)
|
||||
}
|
||||
defer func() { _ = SetEnabled("plain", false, false) }()
|
||||
resp := rpc.CallWithContext(context.Background(), nil, "plugin:plain", nil)
|
||||
if resp.Error != nil {
|
||||
t.Fatalf("default-granted registerRPC failed: %v", resp.Error)
|
||||
}
|
||||
|
||||
// server.route 缺 allowRoutes:加载失败并提示权限名。
|
||||
zipPath = writePluginZip(t, map[string]string{
|
||||
"komari-plugin.json": `{"name":"NoRoute","short":"noroute","version":"1.0.0"}`,
|
||||
"script.js": `const server = require("server");
|
||||
function load() { server.route("GET", "/x", (req, res) => res.end("x")); }`,
|
||||
})
|
||||
if _, err := InstallZip(zipPath); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := SetEnabled("noroute", true, true); err == nil || !strings.Contains(err.Error(), "allowRoutes") {
|
||||
t.Fatalf("server.route without allowRoutes must fail loading: %v", err)
|
||||
}
|
||||
|
||||
// server.hook 缺 allowHooks:加载失败并提示权限名。
|
||||
zipPath = writePluginZip(t, map[string]string{
|
||||
"komari-plugin.json": `{"name":"NoHook","short":"nohook","version":"1.0.0"}`,
|
||||
"script.js": `const server = require("server");
|
||||
function load() { server.hook("request", (req) => {}); }`,
|
||||
})
|
||||
if _, err := InstallZip(zipPath); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := SetEnabled("nohook", true, true); err == nil || !strings.Contains(err.Error(), "allowHooks") {
|
||||
t.Fatalf("server.hook without allowHooks must fail loading: %v", err)
|
||||
}
|
||||
|
||||
// server.call 缺 allowSystemRPC:路由可用但 Promise reject。
|
||||
zipPath = writePluginZip(t, map[string]string{
|
||||
"komari-plugin.json": `{"name":"NoRPC","short":"norpc","version":"1.0.0","permissions":{"allowRoutes":true}}`,
|
||||
"script.js": `const server = require("server");
|
||||
function load() {
|
||||
server.route("GET", "/call", async (req, res) => {
|
||||
try {
|
||||
await server.call("common:getVersion");
|
||||
res.end("ok");
|
||||
} catch (e) {
|
||||
res.end("denied:" + e.message);
|
||||
}
|
||||
});
|
||||
}`,
|
||||
})
|
||||
if _, err := InstallZip(zipPath); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := SetEnabled("norpc", true, true); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer func() { _ = SetEnabled("norpc", false, false) }()
|
||||
rec := httptest.NewRecorder()
|
||||
engine.ServeHTTP(rec, httptest.NewRequest(http.MethodGet, "/call", nil))
|
||||
if rec.Code != http.StatusOK || !strings.Contains(rec.Body.String(), "allowSystemRPC") {
|
||||
t.Fatalf("server.call without allowSystemRPC must reject: status = %d, body = %q", rec.Code, rec.Body.String())
|
||||
}
|
||||
}
|
||||
|
||||
// TestHookFiltering 验证 server.hook(kind, matcher, fn) 的路径/方法过滤:
|
||||
// 不匹配的请求完全跳过 hook 链(不做 body 缓冲),匹配的请求按规则执行。
|
||||
func TestHookFiltering(t *testing.T) {
|
||||
withTempDataDir(t)
|
||||
gin.SetMode(gin.TestMode)
|
||||
engine := gin.New()
|
||||
check := func(name, header string) gin.HandlerFunc {
|
||||
return func(c *gin.Context) {
|
||||
if c.GetHeader(header) == "" {
|
||||
c.String(http.StatusBadRequest, "missing "+header)
|
||||
return
|
||||
}
|
||||
c.String(http.StatusOK, name)
|
||||
}
|
||||
}
|
||||
engine.GET("/ping", func(c *gin.Context) { c.String(http.StatusOK, "no-hook") })
|
||||
engine.GET("/hooked", check("hooked", "x-hooked"))
|
||||
engine.GET("/hooked/sub", check("hooked", "x-hooked"))
|
||||
engine.GET("/hookedx", func(c *gin.Context) { c.String(http.StatusOK, "no-hook") })
|
||||
engine.POST("/exact", check("exact", "x-exact"))
|
||||
engine.GET("/exact", func(c *gin.Context) { c.String(http.StatusOK, "no-hook") })
|
||||
engine.GET("/prefix/sub", check("prefix", "x-prefix"))
|
||||
engine.GET("/prefixx", func(c *gin.Context) { c.String(http.StatusOK, "no-hook") })
|
||||
engine.GET("/resp/a", func(c *gin.Context) { c.String(http.StatusOK, "r") })
|
||||
Init(engine)
|
||||
|
||||
zipPath := writePluginZip(t, map[string]string{
|
||||
"komari-plugin.json": featureManifest,
|
||||
"script.js": `
|
||||
const server = require("server");
|
||||
function load() {
|
||||
server.hook("request", "/hooked/*", (req) => { req.headers["x-hooked"] = "yes"; });
|
||||
server.hook("request", "POST /exact", (req) => { req.headers["x-exact"] = "yes"; });
|
||||
server.hook("request", "/prefix/*", (req) => { req.headers["x-prefix"] = "yes"; });
|
||||
server.hook("response", "/resp/*", (req, res) => { res.body = res.body + "|resp"; });
|
||||
}
|
||||
`,
|
||||
})
|
||||
if _, err := InstallZip(zipPath); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := SetEnabled("feat", true, true); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer func() { _ = SetEnabled("feat", false, false) }()
|
||||
wrapped := WrapHandler(engine)
|
||||
|
||||
serve := func(method, path string) *httptest.ResponseRecorder {
|
||||
rec := httptest.NewRecorder()
|
||||
wrapped.ServeHTTP(rec, httptest.NewRequest(method, path, nil))
|
||||
return rec
|
||||
}
|
||||
assert := func(method, path, want string) {
|
||||
rec := serve(method, path)
|
||||
if rec.Code != http.StatusOK || rec.Body.String() != want {
|
||||
t.Fatalf("%s %s = %d %q, want %q", method, path, rec.Code, rec.Body.String(), want)
|
||||
}
|
||||
}
|
||||
assert(http.MethodGet, "/ping", "no-hook")
|
||||
assert(http.MethodGet, "/hooked", "hooked")
|
||||
assert(http.MethodGet, "/hooked/sub", "hooked")
|
||||
assert(http.MethodGet, "/hookedx", "no-hook")
|
||||
assert(http.MethodPost, "/exact", "exact")
|
||||
assert(http.MethodGet, "/exact", "no-hook")
|
||||
assert(http.MethodGet, "/prefix/sub", "prefix")
|
||||
assert(http.MethodGet, "/prefixx", "no-hook")
|
||||
assert(http.MethodGet, "/resp/a", "r|resp")
|
||||
assert(http.MethodGet, "/ping", "no-hook")
|
||||
}
|
||||
|
||||
// TestHookResponseContentLength 验证响应钩子改写 body 后不会留下失效的
|
||||
// Content-Length(否则客户端会截断/挂起);未改 body 时保留原长度。
|
||||
func TestHookResponseContentLength(t *testing.T) {
|
||||
withTempDataDir(t)
|
||||
gin.SetMode(gin.TestMode)
|
||||
engine := gin.New()
|
||||
engine.GET("/changed", func(c *gin.Context) {
|
||||
c.Header("Content-Length", strconv.Itoa(len("abcdef")))
|
||||
c.String(http.StatusOK, "abcdef")
|
||||
})
|
||||
engine.GET("/kept", func(c *gin.Context) {
|
||||
c.Header("Content-Length", strconv.Itoa(len("abcdef")))
|
||||
c.String(http.StatusOK, "abcdef")
|
||||
})
|
||||
Init(engine)
|
||||
|
||||
zipPath := writePluginZip(t, map[string]string{
|
||||
"komari-plugin.json": featureManifest,
|
||||
"script.js": `
|
||||
const server = require("server");
|
||||
function load() {
|
||||
server.hook("response", (req, res) => {
|
||||
if (req.url === "/changed") { res.body = "modified body"; }
|
||||
});
|
||||
}
|
||||
`,
|
||||
})
|
||||
if _, err := InstallZip(zipPath); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := SetEnabled("feat", true, true); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer func() { _ = SetEnabled("feat", false, false) }()
|
||||
wrapped := WrapHandler(engine)
|
||||
|
||||
rec := httptest.NewRecorder()
|
||||
wrapped.ServeHTTP(rec, httptest.NewRequest(http.MethodGet, "/changed", nil))
|
||||
if rec.Body.String() != "modified body" {
|
||||
t.Fatalf("body = %q", rec.Body.String())
|
||||
}
|
||||
if cl := rec.Header().Get("Content-Length"); cl != "" {
|
||||
t.Fatalf("stale Content-Length %q kept after body rewrite", cl)
|
||||
}
|
||||
|
||||
rec2 := httptest.NewRecorder()
|
||||
wrapped.ServeHTTP(rec2, httptest.NewRequest(http.MethodGet, "/kept", nil))
|
||||
if rec2.Body.String() != "abcdef" || rec2.Header().Get("Content-Length") != "6" {
|
||||
t.Fatalf("unmodified body: len header = %q, body = %q", rec2.Header().Get("Content-Length"), rec2.Body.String())
|
||||
}
|
||||
}
|
||||
|
||||
// TestHookBodyLimit 验证请求钩子按插件声明的 maxHTTPBodyBytes 读取请求体:
|
||||
// 超过限制的请求返回 413,而非沿用全局默认 32 MiB。
|
||||
func TestHookBodyLimit(t *testing.T) {
|
||||
withTempDataDir(t)
|
||||
gin.SetMode(gin.TestMode)
|
||||
engine := gin.New()
|
||||
engine.POST("/echo", func(c *gin.Context) {
|
||||
if c.GetHeader("x-hooked") != "yes" {
|
||||
c.String(http.StatusBadRequest, "missing x-hooked")
|
||||
return
|
||||
}
|
||||
c.String(http.StatusOK, "ok")
|
||||
})
|
||||
Init(engine)
|
||||
|
||||
zipPath := writePluginZip(t, map[string]string{
|
||||
"komari-plugin.json": `{"name":"Limit","short":"limit","version":"1.0.0","permissions":{"allowHooks":true,"maxHTTPBodyBytes":256,"timeout":5}}`,
|
||||
"script.js": `
|
||||
const server = require("server");
|
||||
function load() {
|
||||
server.hook("request", (req) => { req.headers["x-hooked"] = "yes"; });
|
||||
}
|
||||
`,
|
||||
})
|
||||
if _, err := InstallZip(zipPath); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := SetEnabled("limit", true, true); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer func() { _ = SetEnabled("limit", false, false) }()
|
||||
wrapped := WrapHandler(engine)
|
||||
|
||||
rec := httptest.NewRecorder()
|
||||
wrapped.ServeHTTP(rec, httptest.NewRequest(http.MethodPost, "/echo", strings.NewReader(strings.Repeat("x", 128))))
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("small body = %d, want 200", rec.Code)
|
||||
}
|
||||
rec2 := httptest.NewRecorder()
|
||||
wrapped.ServeHTTP(rec2, httptest.NewRequest(http.MethodPost, "/echo", strings.NewReader(strings.Repeat("x", 1024))))
|
||||
if rec2.Code != http.StatusRequestEntityTooLarge {
|
||||
t.Fatalf("large body = %d %q, want 413", rec2.Code, rec2.Body.String())
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,594 @@
|
||||
package plugin
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"bytes"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/dop251/goja"
|
||||
"github.com/komari-monitor/komari/pkg/jsruntime"
|
||||
"github.com/komari-monitor/komari/pkg/jsruntime/httpbody"
|
||||
)
|
||||
|
||||
// hookKind identifies the HTTP hook phase.
|
||||
type hookKind string
|
||||
|
||||
const (
|
||||
hookRequest hookKind = "request"
|
||||
hookResponse hookKind = "response"
|
||||
)
|
||||
|
||||
// hookEntry is one registered JavaScript hook.
|
||||
type hookEntry struct {
|
||||
short string
|
||||
kind hookKind
|
||||
fn goja.Callable
|
||||
host *jsruntime.Host
|
||||
matcher *hookMatcher // nil matches every request
|
||||
bodyLimit int64 // plugin-declared maxHTTPBodyBytes (0 = default)
|
||||
}
|
||||
|
||||
// hookMatcher optionally restricts a hook to matching requests. A nil
|
||||
// matcher matches every request.
|
||||
type hookMatcher struct {
|
||||
method string // optional HTTP method; "" matches any
|
||||
path string // cleaned request path, lowercase
|
||||
prefix bool // subtree match when the pattern ended with "*"
|
||||
}
|
||||
|
||||
// parseHookMatcher parses a server.hook path filter: "POST /api/foo",
|
||||
// "/api/foo" or "/api/foo/*". The optional leading HTTP method restricts the
|
||||
// hook to one request method; a trailing "*" matches the path and its whole
|
||||
// subtree; any other pattern is an exact path match.
|
||||
func parseHookMatcher(pattern string) (*hookMatcher, error) {
|
||||
pattern = strings.TrimSpace(pattern)
|
||||
if pattern == "" {
|
||||
return nil, errors.New("hook path must not be empty")
|
||||
}
|
||||
m := &hookMatcher{}
|
||||
if method, rest, ok := strings.Cut(pattern, " "); ok {
|
||||
method = strings.ToUpper(strings.TrimSpace(method))
|
||||
if isHTTPMethod(method) {
|
||||
m.method = method
|
||||
pattern = strings.TrimSpace(rest)
|
||||
}
|
||||
}
|
||||
if !strings.HasPrefix(pattern, "/") {
|
||||
return nil, fmt.Errorf("hook path %q must start with /", pattern)
|
||||
}
|
||||
if strings.HasSuffix(pattern, "*") {
|
||||
m.prefix = true
|
||||
pattern = strings.TrimSuffix(pattern, "*")
|
||||
}
|
||||
pattern = strings.TrimRight(pattern, "/")
|
||||
if pattern == "" {
|
||||
if m.prefix {
|
||||
return m, nil // "/*" matches every path
|
||||
}
|
||||
return nil, fmt.Errorf("hook path %q is invalid", pattern)
|
||||
}
|
||||
if !filepath.IsLocal(strings.TrimPrefix(pattern, "/")) {
|
||||
return nil, fmt.Errorf("hook path %q contains invalid segments", pattern)
|
||||
}
|
||||
m.path = strings.ToLower(pattern)
|
||||
return m, nil
|
||||
}
|
||||
|
||||
// matchesPath reports whether a request path matches the filter. A nil
|
||||
// matcher matches everything.
|
||||
func (m *hookMatcher) matchesPath(path string) bool {
|
||||
if m == nil {
|
||||
return true
|
||||
}
|
||||
if path == "" {
|
||||
path = "/"
|
||||
}
|
||||
path = strings.ToLower(path)
|
||||
if m.prefix {
|
||||
return path == m.path || strings.HasPrefix(path, m.path+"/")
|
||||
}
|
||||
return path == m.path
|
||||
}
|
||||
|
||||
// matches reports whether the request matches the filter. A nil matcher
|
||||
// matches everything.
|
||||
func (m *hookMatcher) matches(r *http.Request) bool {
|
||||
if m == nil {
|
||||
return true
|
||||
}
|
||||
if m.method != "" && r.Method != m.method {
|
||||
return false
|
||||
}
|
||||
return m.matchesPath(r.URL.Path)
|
||||
}
|
||||
|
||||
// isHTTPMethod reports whether s is a known HTTP request method.
|
||||
func isHTTPMethod(s string) bool {
|
||||
switch s {
|
||||
case http.MethodGet, http.MethodHead, http.MethodPost, http.MethodPut,
|
||||
http.MethodPatch, http.MethodDelete, http.MethodOptions:
|
||||
return true
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
// matches reports whether the hook applies to the request: no matcher, or a
|
||||
// matching filter.
|
||||
func (h *hookEntry) matches(r *http.Request) bool {
|
||||
return h.matcher == nil || h.matcher.matches(r)
|
||||
}
|
||||
|
||||
// filterHooks drops the hooks whose matcher does not match the request.
|
||||
func filterHooks(hooks []*hookEntry, r *http.Request) []*hookEntry {
|
||||
kept := hooks[:0]
|
||||
for _, hook := range hooks {
|
||||
if hook.matches(r) {
|
||||
kept = append(kept, hook)
|
||||
}
|
||||
}
|
||||
return kept
|
||||
}
|
||||
|
||||
// maxHookBodyLimit returns the body read limit for the request hooks that
|
||||
// will run: the largest plugin-declared maxHTTPBodyBytes, or the default
|
||||
// when no hook declares a limit.
|
||||
func maxHookBodyLimit(hooks []*hookEntry) int64 {
|
||||
var limit int64
|
||||
for _, hook := range hooks {
|
||||
if hook.bodyLimit > limit {
|
||||
limit = hook.bodyLimit
|
||||
}
|
||||
}
|
||||
if limit < 1 {
|
||||
return defaultMaxHTTPBodyBytes
|
||||
}
|
||||
return limit
|
||||
}
|
||||
|
||||
// wrapHandler installs the hook chain around the application handler. WebSocket
|
||||
// upgrades pass through untouched because the response cannot be buffered.
|
||||
// Hooks with a path/method filter only run for matching requests; unfiltered
|
||||
// requests skip both the hook chain and its body buffering.
|
||||
func (m *Manager) wrapHandler(next http.Handler) http.Handler {
|
||||
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if isUpgradeRequest(r) {
|
||||
next.ServeHTTP(w, r)
|
||||
return
|
||||
}
|
||||
reqHooks := filterHooks(m.hooksOf(hookRequest), r)
|
||||
respHooks := filterHooks(m.hooksOf(hookResponse), r)
|
||||
if len(reqHooks) == 0 && len(respHooks) == 0 {
|
||||
next.ServeHTTP(w, r)
|
||||
return
|
||||
}
|
||||
|
||||
req := r
|
||||
if len(reqHooks) > 0 {
|
||||
body, err := httpbody.ReadAll(r.Body, maxHookBodyLimit(reqHooks))
|
||||
if err != nil {
|
||||
http.Error(w, err.Error(), http.StatusRequestEntityTooLarge)
|
||||
return
|
||||
}
|
||||
for _, hook := range reqHooks {
|
||||
body, err = m.runRequestHook(req, body, hook)
|
||||
if err != nil {
|
||||
m.logHookError(hook, err)
|
||||
http.Error(w, "plugin request hook failed", http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
}
|
||||
req = requestWithBody(req, body)
|
||||
}
|
||||
|
||||
if len(respHooks) == 0 {
|
||||
next.ServeHTTP(w, req)
|
||||
return
|
||||
}
|
||||
bw := newBufferedResponseWriter(w)
|
||||
next.ServeHTTP(bw, req)
|
||||
for _, hook := range respHooks {
|
||||
if err := m.runResponseHook(req, bw, hook); err != nil {
|
||||
m.logHookError(hook, err)
|
||||
}
|
||||
}
|
||||
bw.flushTo(w)
|
||||
})
|
||||
}
|
||||
|
||||
// runRequestHook runs one request hook on its plugin event loop, applies the
|
||||
// JS-side modifications to the request, and returns the (possibly replaced)
|
||||
// request body.
|
||||
func (m *Manager) runRequestHook(r *http.Request, body []byte, h *hookEntry) ([]byte, error) {
|
||||
nextBody := body
|
||||
var hookErr error
|
||||
queued, timedOut := runHookTurn(h.host, "plugin request hook "+h.short, func(vm *goja.Runtime) {
|
||||
reqObj := hookRequestObject(vm, r, nextBody)
|
||||
hookErr = h.host.RunJob(vm, "plugin request hook "+h.short, func() error {
|
||||
_, err := h.fn(goja.Undefined(), reqObj)
|
||||
return err
|
||||
})
|
||||
if hookErr != nil {
|
||||
return
|
||||
}
|
||||
if value := reqObj.Get("method"); value != nil && !goja.IsUndefined(value) {
|
||||
if method := strings.ToUpper(strings.TrimSpace(value.String())); method != "" {
|
||||
r.Method = method
|
||||
}
|
||||
}
|
||||
if value := reqObj.Get("url"); value != nil && !goja.IsUndefined(value) {
|
||||
if raw := strings.TrimSpace(value.String()); raw != "" {
|
||||
if parsed, err := url.Parse(raw); err == nil {
|
||||
r.URL = parsed
|
||||
r.RequestURI = raw
|
||||
}
|
||||
}
|
||||
}
|
||||
if value := reqObj.Get("headers"); value != nil && !goja.IsUndefined(value) {
|
||||
var raw map[string]any
|
||||
if err := vm.ExportTo(value, &raw); err == nil {
|
||||
r.Header = normalizeHeader(raw)
|
||||
}
|
||||
}
|
||||
if value := reqObj.Get("body"); value != nil && !goja.IsUndefined(value) {
|
||||
nextBody = []byte(value.String())
|
||||
}
|
||||
})
|
||||
if !queued {
|
||||
return body, errors.New("plugin runtime is closed")
|
||||
}
|
||||
if timedOut {
|
||||
return body, errors.New("plugin request hook timed out")
|
||||
}
|
||||
return nextBody, hookErr
|
||||
}
|
||||
|
||||
// runResponseHook runs one response hook on its plugin event loop and applies
|
||||
// the JS-side modifications to the captured response.
|
||||
func (m *Manager) runResponseHook(r *http.Request, bw *bufferedResponseWriter, h *hookEntry) error {
|
||||
var hookErr error
|
||||
queued, timedOut := runHookTurn(h.host, "plugin response hook "+h.short, func(vm *goja.Runtime) {
|
||||
resObj := hookResponseObject(vm, bw)
|
||||
hookErr = h.host.RunJob(vm, "plugin response hook "+h.short, func() error {
|
||||
_, err := h.fn(goja.Undefined(), hookRequestObject(vm, r, nil), resObj)
|
||||
return err
|
||||
})
|
||||
if hookErr != nil {
|
||||
return
|
||||
}
|
||||
status := bw.status()
|
||||
if value := resObj.Get("statusCode"); value != nil && !goja.IsUndefined(value) {
|
||||
status = int(value.ToInteger())
|
||||
}
|
||||
header := bw.Header().Clone()
|
||||
if value := resObj.Get("headers"); value != nil && !goja.IsUndefined(value) {
|
||||
var raw map[string]any
|
||||
if err := vm.ExportTo(value, &raw); err == nil {
|
||||
header = normalizeHeader(raw)
|
||||
}
|
||||
}
|
||||
responseBody := bw.bodyBytes()
|
||||
if value := resObj.Get("body"); value != nil && !goja.IsUndefined(value) {
|
||||
responseBody = []byte(value.String())
|
||||
}
|
||||
bw.apply(status, header, responseBody)
|
||||
})
|
||||
if !queued {
|
||||
return errors.New("plugin runtime is closed")
|
||||
}
|
||||
if timedOut {
|
||||
return errors.New("plugin response hook timed out")
|
||||
}
|
||||
if bw.passedThrough() {
|
||||
_, _ = m.logStore(h.short).Write([]byte(
|
||||
fmt.Sprintf("[plugin] response exceeded the %d-byte hook buffer limit and was sent to the client unmodified\n", maxHookBufferBytes)))
|
||||
}
|
||||
return hookErr
|
||||
}
|
||||
|
||||
// runHookTurn queues one hook callback on the plugin event loop and waits
|
||||
// for it with the runtime timeout. It reports whether the job was queued and
|
||||
// whether it timed out.
|
||||
func runHookTurn(host *jsruntime.Host, name string, job func(vm *goja.Runtime)) (queued, timedOut bool) {
|
||||
return runHookTurnTimeout(host, name, host.Timeout(), job)
|
||||
}
|
||||
|
||||
// runHookTurnTimeout is runHookTurn with an explicit wait budget. The
|
||||
// callback keeps running on the plugin loop after a timeout; the waiter gives
|
||||
// up and the caller falls back to passing the message through.
|
||||
func runHookTurnTimeout(host *jsruntime.Host, name string, timeout time.Duration, job func(vm *goja.Runtime)) (queued, timedOut bool) {
|
||||
done := make(chan struct{})
|
||||
if !host.RunOnLoop(func(vm *goja.Runtime) {
|
||||
defer close(done)
|
||||
job(vm)
|
||||
}) {
|
||||
return false, false
|
||||
}
|
||||
select {
|
||||
case <-done:
|
||||
return true, false
|
||||
case <-time.After(timeout):
|
||||
return true, true
|
||||
}
|
||||
}
|
||||
|
||||
func (m *Manager) logHookError(h *hookEntry, err error) {
|
||||
_, _ = m.logStore(h.short).Write([]byte("[plugin] hook error: " + err.Error() + "\n"))
|
||||
}
|
||||
|
||||
// hookRequestObject builds the mutable JS request object.
|
||||
func hookRequestObject(vm *goja.Runtime, r *http.Request, body []byte) *goja.Object {
|
||||
req := vm.NewObject()
|
||||
_ = req.Set("method", r.Method)
|
||||
_ = req.Set("url", r.URL.RequestURI())
|
||||
_ = req.Set("headers", headerToMap(r.Header))
|
||||
query := make(map[string]string, len(r.URL.Query()))
|
||||
for name, values := range r.URL.Query() {
|
||||
query[name] = strings.Join(values, ",")
|
||||
}
|
||||
_ = req.Set("query", query)
|
||||
_ = req.Set("body", string(body))
|
||||
_ = req.Set("context", hookRequestContext(vm, r))
|
||||
return req
|
||||
}
|
||||
|
||||
// hookResponseObject builds the mutable JS response object from the captured
|
||||
// response.
|
||||
func hookResponseObject(vm *goja.Runtime, bw *bufferedResponseWriter) *goja.Object {
|
||||
res := vm.NewObject()
|
||||
_ = res.Set("statusCode", bw.status())
|
||||
_ = res.Set("statusMessage", "")
|
||||
_ = res.Set("headers", headerToMap(bw.Header()))
|
||||
_ = res.Set("body", string(bw.bodyBytes()))
|
||||
return res
|
||||
}
|
||||
|
||||
func requestWithBody(r *http.Request, body []byte) *http.Request {
|
||||
clone := r.Clone(r.Context())
|
||||
clone.Body = io.NopCloser(bytes.NewReader(body))
|
||||
clone.ContentLength = int64(len(body))
|
||||
return clone
|
||||
}
|
||||
|
||||
func normalizeHeader(raw map[string]any) http.Header {
|
||||
header := make(http.Header, len(raw))
|
||||
for name, value := range raw {
|
||||
switch values := value.(type) {
|
||||
case string:
|
||||
header.Set(name, values)
|
||||
case []any:
|
||||
for _, item := range values {
|
||||
header.Add(name, fmt.Sprint(item))
|
||||
}
|
||||
case []string:
|
||||
for _, item := range values {
|
||||
header.Add(name, item)
|
||||
}
|
||||
default:
|
||||
header.Set(name, fmt.Sprint(value))
|
||||
}
|
||||
}
|
||||
return header
|
||||
}
|
||||
|
||||
func headerToMap(header http.Header) map[string]any {
|
||||
out := make(map[string]any, len(header))
|
||||
for name, values := range header {
|
||||
if len(values) == 1 {
|
||||
out[strings.ToLower(name)] = values[0]
|
||||
} else {
|
||||
out[strings.ToLower(name)] = values
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func isUpgradeRequest(r *http.Request) bool {
|
||||
return strings.EqualFold(strings.TrimSpace(r.Header.Get("Connection")), "upgrade") ||
|
||||
strings.EqualFold(r.Header.Get("Upgrade"), "websocket")
|
||||
}
|
||||
|
||||
// bufferedResponseWriter captures status/headers/body so response hooks can
|
||||
// modify them before the real writer receives the final bytes.
|
||||
//
|
||||
// Streaming responses (SSE / MJPEG / chunked feeds) must not be buffered:
|
||||
// their handlers never return, so buffering would hold the client empty
|
||||
// forever and grow memory without bound. The first Flush() switches the
|
||||
// writer to passthrough mode: buffered status/headers/body are forwarded to
|
||||
// the real writer immediately, every later Write/WriteHeader goes straight
|
||||
// through, and the response hooks can no longer rewrite the stream.
|
||||
type bufferedResponseWriter struct {
|
||||
mu sync.Mutex
|
||||
header http.Header
|
||||
statusCode int
|
||||
body bytes.Buffer
|
||||
wroteHeader bool
|
||||
streaming bool // switched to passthrough after the first Flush()
|
||||
forwarded bool // status/headers already sent to the underlying writer
|
||||
hijacked bool
|
||||
limit int // maximum buffered body bytes before passthrough
|
||||
passthrough bool // buffering abandoned because the body exceeded limit
|
||||
underlying http.ResponseWriter
|
||||
}
|
||||
|
||||
func newBufferedResponseWriter(w http.ResponseWriter) *bufferedResponseWriter {
|
||||
return &bufferedResponseWriter{header: make(http.Header), limit: maxHookBufferBytes, underlying: w}
|
||||
}
|
||||
|
||||
func (b *bufferedResponseWriter) passedThrough() bool {
|
||||
b.mu.Lock()
|
||||
defer b.mu.Unlock()
|
||||
return b.passthrough
|
||||
}
|
||||
|
||||
func (b *bufferedResponseWriter) Header() http.Header { return b.header }
|
||||
|
||||
func (b *bufferedResponseWriter) WriteHeader(code int) {
|
||||
b.mu.Lock()
|
||||
if b.streaming {
|
||||
if b.forwarded || b.hijacked {
|
||||
b.mu.Unlock()
|
||||
return
|
||||
}
|
||||
b.forwarded = true
|
||||
status, header := code, b.header.Clone()
|
||||
b.mu.Unlock()
|
||||
b.writeThrough(status, header)
|
||||
return
|
||||
}
|
||||
if !b.wroteHeader {
|
||||
b.statusCode = code
|
||||
b.wroteHeader = true
|
||||
}
|
||||
b.mu.Unlock()
|
||||
}
|
||||
|
||||
func (b *bufferedResponseWriter) Write(p []byte) (int, error) {
|
||||
b.mu.Lock()
|
||||
if b.streaming {
|
||||
b.mu.Unlock()
|
||||
return b.underlying.Write(p)
|
||||
}
|
||||
if !b.wroteHeader {
|
||||
b.statusCode = http.StatusOK
|
||||
b.wroteHeader = true
|
||||
}
|
||||
if b.body.Len()+len(p) > b.limit {
|
||||
// Response is too large to rewrite: switch to passthrough mode and
|
||||
// forward status/headers and the buffered body to the real writer
|
||||
// immediately. Response hooks can no longer modify this response.
|
||||
status, header, buffered := b.statusCode, b.header.Clone(), b.body.Bytes()
|
||||
b.streaming = true
|
||||
b.forwarded = true
|
||||
b.passthrough = true
|
||||
b.mu.Unlock()
|
||||
b.writeThrough(status, header)
|
||||
_, _ = b.underlying.Write(buffered)
|
||||
_, _ = b.underlying.Write(p)
|
||||
return len(p), nil
|
||||
}
|
||||
_, err := b.body.Write(p)
|
||||
b.mu.Unlock()
|
||||
return len(p), err
|
||||
}
|
||||
|
||||
func (b *bufferedResponseWriter) status() int {
|
||||
b.mu.Lock()
|
||||
defer b.mu.Unlock()
|
||||
if !b.wroteHeader {
|
||||
return http.StatusOK
|
||||
}
|
||||
return b.statusCode
|
||||
}
|
||||
|
||||
func (b *bufferedResponseWriter) bodyBytes() []byte {
|
||||
b.mu.Lock()
|
||||
defer b.mu.Unlock()
|
||||
return append([]byte(nil), b.body.Bytes()...)
|
||||
}
|
||||
|
||||
func (b *bufferedResponseWriter) apply(status int, header http.Header, body []byte) {
|
||||
b.mu.Lock()
|
||||
defer b.mu.Unlock()
|
||||
if b.streaming {
|
||||
return // already streamed to the client; hooks cannot rewrite it
|
||||
}
|
||||
bodyChanged := !bytes.Equal(body, b.body.Bytes())
|
||||
b.statusCode = status
|
||||
b.wroteHeader = true
|
||||
b.header = header
|
||||
b.body.Reset()
|
||||
_, _ = b.body.Write(body)
|
||||
if bodyChanged {
|
||||
// The rewritten body differs from the original response; a
|
||||
// Content-Length copied from it is now stale and would truncate or
|
||||
// hang the client. Drop it so Go recomputes the length (or falls
|
||||
// back to chunked encoding).
|
||||
b.header.Del("Content-Length")
|
||||
}
|
||||
}
|
||||
|
||||
func (b *bufferedResponseWriter) flushTo(w http.ResponseWriter) {
|
||||
b.mu.Lock()
|
||||
if b.streaming || b.hijacked {
|
||||
b.mu.Unlock()
|
||||
return // passthrough already forwarded everything
|
||||
}
|
||||
status, body, header := b.statusCode, append([]byte(nil), b.body.Bytes()...), b.header.Clone()
|
||||
b.mu.Unlock()
|
||||
for name, values := range header {
|
||||
for _, value := range values {
|
||||
w.Header().Add(name, value)
|
||||
}
|
||||
}
|
||||
w.WriteHeader(status)
|
||||
_, _ = w.Write(body)
|
||||
}
|
||||
|
||||
func (b *bufferedResponseWriter) Hijack() (net.Conn, *bufio.ReadWriter, error) {
|
||||
b.mu.Lock()
|
||||
b.hijacked = true
|
||||
b.mu.Unlock()
|
||||
hijacker, ok := b.underlying.(http.Hijacker)
|
||||
if !ok {
|
||||
return nil, nil, errors.New("plugin response buffer does not support hijacking")
|
||||
}
|
||||
return hijacker.Hijack()
|
||||
}
|
||||
|
||||
func (b *bufferedResponseWriter) Flush() {
|
||||
b.mu.Lock()
|
||||
if !b.streaming {
|
||||
// First Flush: switch to passthrough mode and forward the buffered
|
||||
// response to the real writer immediately.
|
||||
b.streaming = true
|
||||
status, body, header := b.statusCode, append([]byte(nil), b.body.Bytes()...), b.header.Clone()
|
||||
b.mu.Unlock()
|
||||
b.writeThrough(status, header)
|
||||
_, _ = b.underlying.Write(body)
|
||||
} else {
|
||||
b.mu.Unlock()
|
||||
}
|
||||
if flusher, ok := b.underlying.(http.Flusher); ok {
|
||||
flusher.Flush()
|
||||
}
|
||||
}
|
||||
|
||||
// writeThrough forwards status and headers to the underlying writer once.
|
||||
func (b *bufferedResponseWriter) writeThrough(status int, header http.Header) {
|
||||
for name, values := range header {
|
||||
for _, value := range values {
|
||||
b.underlying.Header().Add(name, value)
|
||||
}
|
||||
}
|
||||
b.underlying.WriteHeader(status)
|
||||
}
|
||||
|
||||
func (b *bufferedResponseWriter) ReadFrom(r io.Reader) (int64, error) {
|
||||
if !b.wroteHeader {
|
||||
b.WriteHeader(http.StatusOK)
|
||||
}
|
||||
return io.Copy(b, r)
|
||||
}
|
||||
|
||||
func (b *bufferedResponseWriter) CloseNotify() <-chan bool {
|
||||
if notifier, ok := b.underlying.(interface{ CloseNotify() <-chan bool }); ok {
|
||||
return notifier.CloseNotify()
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
var (
|
||||
_ http.ResponseWriter = (*bufferedResponseWriter)(nil)
|
||||
_ http.Hijacker = (*bufferedResponseWriter)(nil)
|
||||
_ http.Flusher = (*bufferedResponseWriter)(nil)
|
||||
)
|
||||
@@ -0,0 +1,277 @@
|
||||
package plugin
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"bytes"
|
||||
"errors"
|
||||
"io"
|
||||
"net"
|
||||
"net/http"
|
||||
"strings"
|
||||
"sync"
|
||||
)
|
||||
|
||||
// injectEntry is one plugin's registered HTML fragments. head is inserted
|
||||
// before </head>, body before </body>, in every text/html response.
|
||||
type injectEntry struct {
|
||||
short string
|
||||
head string
|
||||
body string
|
||||
}
|
||||
|
||||
// htmlInjectHandler installs the HTML injection wrapper around the
|
||||
// application handler. WebSocket upgrades pass through untouched because the
|
||||
// response cannot be buffered.
|
||||
func (m *Manager) htmlInjectHandler(next http.Handler) http.Handler {
|
||||
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if isUpgradeRequest(r) {
|
||||
next.ServeHTTP(w, r)
|
||||
return
|
||||
}
|
||||
fragments := m.injectsSnapshot()
|
||||
if len(fragments) == 0 {
|
||||
next.ServeHTTP(w, r)
|
||||
return
|
||||
}
|
||||
iw := newHTMLInjectWriter(w)
|
||||
next.ServeHTTP(iw, r)
|
||||
iw.flushTo(fragments)
|
||||
})
|
||||
}
|
||||
|
||||
// injectHTMLContent inserts the plugin head/body fragments into an HTML
|
||||
// document: head before </head>, body before </body> with fallbacks to
|
||||
// </html> and the document end. Tag matching is case-insensitive.
|
||||
func injectHTMLContent(body []byte, fragments []*injectEntry) []byte {
|
||||
if len(fragments) == 0 {
|
||||
return body
|
||||
}
|
||||
var head, tail strings.Builder
|
||||
for _, fragment := range fragments {
|
||||
head.WriteString(fragment.head)
|
||||
tail.WriteString(fragment.body)
|
||||
}
|
||||
if head.Len() == 0 && tail.Len() == 0 {
|
||||
return body
|
||||
}
|
||||
text := string(body)
|
||||
headHTML, tailHTML := head.String(), tail.String()
|
||||
if headHTML != "" {
|
||||
if idx := indexFold(text, "</head>"); idx >= 0 {
|
||||
text = text[:idx] + headHTML + text[idx:]
|
||||
} else {
|
||||
text = headHTML + text
|
||||
}
|
||||
}
|
||||
if tailHTML != "" {
|
||||
switch {
|
||||
case indexFold(text, "</body>") >= 0:
|
||||
text = text[:indexFold(text, "</body>")] + tailHTML + text[indexFold(text, "</body>"):]
|
||||
case indexFold(text, "</html>") >= 0:
|
||||
text = text[:indexFold(text, "</html>")] + tailHTML + text[indexFold(text, "</html>"):]
|
||||
default:
|
||||
text += tailHTML
|
||||
}
|
||||
}
|
||||
return []byte(text)
|
||||
}
|
||||
|
||||
func indexFold(text, needle string) int {
|
||||
return strings.Index(strings.ToLower(text), needle)
|
||||
}
|
||||
|
||||
// htmlInjectWriter captures only text/html responses so the registered
|
||||
// plugin fragments can be injected before flushTo. Every other response is
|
||||
// forwarded to the real writer unmodified as soon as its content type is
|
||||
// known, so JSON, assets and streams are never buffered.
|
||||
//
|
||||
// The content type is read from the response header when set; otherwise the
|
||||
// first bytes are sniffed with http.DetectContentType. Streaming responses
|
||||
// (SSE / MJPEG / chunked feeds) switch to passthrough on the first Flush(),
|
||||
// and responses larger than the buffer limit are forwarded unmodified.
|
||||
type htmlInjectWriter struct {
|
||||
mu sync.Mutex
|
||||
header http.Header
|
||||
statusCode int
|
||||
wroteHeader bool
|
||||
body bytes.Buffer
|
||||
html bool // decided: the response is text/html and buffered
|
||||
streaming bool // switched to passthrough
|
||||
hijacked bool
|
||||
limit int
|
||||
underlying http.ResponseWriter
|
||||
}
|
||||
|
||||
func newHTMLInjectWriter(w http.ResponseWriter) *htmlInjectWriter {
|
||||
return &htmlInjectWriter{header: make(http.Header), limit: maxHTMLInjectBufferBytes, underlying: w}
|
||||
}
|
||||
|
||||
func (w *htmlInjectWriter) Header() http.Header { return w.header }
|
||||
|
||||
func (w *htmlInjectWriter) WriteHeader(code int) {
|
||||
w.mu.Lock()
|
||||
if w.streaming {
|
||||
w.mu.Unlock()
|
||||
return
|
||||
}
|
||||
if !w.wroteHeader {
|
||||
w.statusCode = code
|
||||
w.wroteHeader = true
|
||||
}
|
||||
ct := w.header.Get("Content-Type")
|
||||
w.mu.Unlock()
|
||||
if ct != "" && !isHTMLContentType(ct) {
|
||||
w.passthrough()
|
||||
}
|
||||
}
|
||||
|
||||
func (w *htmlInjectWriter) Write(p []byte) (int, error) {
|
||||
w.mu.Lock()
|
||||
if w.streaming {
|
||||
w.mu.Unlock()
|
||||
return w.underlying.Write(p)
|
||||
}
|
||||
if !w.wroteHeader {
|
||||
w.statusCode = http.StatusOK
|
||||
w.wroteHeader = true
|
||||
}
|
||||
if !w.html {
|
||||
ct := w.header.Get("Content-Type")
|
||||
if ct == "" {
|
||||
ct = sniffContentType(w.body.Bytes(), p)
|
||||
if ct != "" {
|
||||
w.header.Set("Content-Type", ct)
|
||||
}
|
||||
}
|
||||
if ct != "" {
|
||||
if !isHTMLContentType(ct) {
|
||||
w.mu.Unlock()
|
||||
w.passthrough()
|
||||
return w.underlying.Write(p)
|
||||
}
|
||||
w.html = true
|
||||
}
|
||||
}
|
||||
if w.body.Len()+len(p) > w.limit {
|
||||
w.mu.Unlock()
|
||||
w.passthrough()
|
||||
return w.underlying.Write(p)
|
||||
}
|
||||
n, err := w.body.Write(p)
|
||||
w.mu.Unlock()
|
||||
return n, err
|
||||
}
|
||||
|
||||
// passthrough forwards the buffered status/headers/body to the real writer
|
||||
// and switches every later write to direct passthrough. The injection is
|
||||
// skipped for the response.
|
||||
func (w *htmlInjectWriter) passthrough() {
|
||||
w.mu.Lock()
|
||||
if w.streaming || w.hijacked {
|
||||
w.mu.Unlock()
|
||||
return
|
||||
}
|
||||
w.streaming = true
|
||||
status, header, body := w.statusCode, w.header.Clone(), append([]byte(nil), w.body.Bytes()...)
|
||||
w.mu.Unlock()
|
||||
w.writeThrough(status, header)
|
||||
_, _ = w.underlying.Write(body)
|
||||
}
|
||||
|
||||
// flushTo finalizes a fully buffered response. HTML responses get the
|
||||
// registered fragments injected, everything else is written through
|
||||
// unmodified.
|
||||
func (w *htmlInjectWriter) flushTo(fragments []*injectEntry) {
|
||||
w.mu.Lock()
|
||||
if w.streaming || w.hijacked {
|
||||
w.mu.Unlock()
|
||||
return
|
||||
}
|
||||
w.streaming = true
|
||||
status, header, body := w.statusCode, w.header.Clone(), append([]byte(nil), w.body.Bytes()...)
|
||||
w.mu.Unlock()
|
||||
|
||||
ct := header.Get("Content-Type")
|
||||
if ct == "" {
|
||||
ct = http.DetectContentType(body)
|
||||
if isHTMLContentType(ct) {
|
||||
header.Set("Content-Type", ct)
|
||||
}
|
||||
}
|
||||
if isHTMLContentType(ct) {
|
||||
body = injectHTMLContent(body, fragments)
|
||||
header.Del("Content-Length")
|
||||
}
|
||||
w.writeThrough(status, header)
|
||||
_, _ = w.underlying.Write(body)
|
||||
}
|
||||
|
||||
func (w *htmlInjectWriter) writeThrough(status int, header http.Header) {
|
||||
for name, values := range header {
|
||||
for _, value := range values {
|
||||
w.underlying.Header().Add(name, value)
|
||||
}
|
||||
}
|
||||
w.underlying.WriteHeader(status)
|
||||
}
|
||||
|
||||
// isHTMLContentType reports whether the MIME type is HTML, with or without
|
||||
// parameters such as charset.
|
||||
func isHTMLContentType(ct string) bool {
|
||||
return strings.Contains(strings.ToLower(ct), "text/html")
|
||||
}
|
||||
|
||||
// sniffContentType decides the content type of a response from the header
|
||||
// value when available, otherwise from the first bytes via
|
||||
// http.DetectContentType. An empty result means the type is not decidable
|
||||
// yet; buffering continues.
|
||||
func sniffContentType(buffered, next []byte) string {
|
||||
const sniffLength = 512
|
||||
total := buffered
|
||||
if len(total) < sniffLength {
|
||||
room := sniffLength - len(total)
|
||||
if len(next) > room {
|
||||
next = next[:room]
|
||||
}
|
||||
total = append(append([]byte(nil), total...), next...)
|
||||
}
|
||||
return http.DetectContentType(total)
|
||||
}
|
||||
|
||||
func (w *htmlInjectWriter) ReadFrom(r io.Reader) (int64, error) {
|
||||
if !w.wroteHeader {
|
||||
w.WriteHeader(http.StatusOK)
|
||||
}
|
||||
return io.Copy(w, r)
|
||||
}
|
||||
|
||||
func (w *htmlInjectWriter) Flush() {
|
||||
w.passthrough()
|
||||
if flusher, ok := w.underlying.(http.Flusher); ok {
|
||||
flusher.Flush()
|
||||
}
|
||||
}
|
||||
|
||||
func (w *htmlInjectWriter) Hijack() (net.Conn, *bufio.ReadWriter, error) {
|
||||
w.mu.Lock()
|
||||
w.hijacked = true
|
||||
w.mu.Unlock()
|
||||
hijacker, ok := w.underlying.(http.Hijacker)
|
||||
if !ok {
|
||||
return nil, nil, errors.New("plugin HTML injector does not support hijacking")
|
||||
}
|
||||
return hijacker.Hijack()
|
||||
}
|
||||
|
||||
func (w *htmlInjectWriter) CloseNotify() <-chan bool {
|
||||
if notifier, ok := w.underlying.(interface{ CloseNotify() <-chan bool }); ok {
|
||||
return notifier.CloseNotify()
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
var (
|
||||
_ http.ResponseWriter = (*htmlInjectWriter)(nil)
|
||||
_ http.Hijacker = (*htmlInjectWriter)(nil)
|
||||
_ http.Flusher = (*htmlInjectWriter)(nil)
|
||||
)
|
||||
@@ -0,0 +1,161 @@
|
||||
package plugin
|
||||
|
||||
import (
|
||||
"archive/zip"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
|
||||
"github.com/komari-monitor/komari/database/models"
|
||||
)
|
||||
|
||||
// InstallZip validates a plugin ZIP and extracts it into DataDir/<short>.
|
||||
// The archive must contain komari-plugin.json at its root. Archive limits
|
||||
// mirror the theme package format; path-traversal entries reject the whole
|
||||
// package instead of being skipped. Reinstalling over a running plugin
|
||||
// unloads it first and restores it to its persisted enabled state when the
|
||||
// extraction succeeds.
|
||||
func InstallZip(zipPath string) (models.Plugin, error) {
|
||||
var info models.Plugin
|
||||
r, err := zip.OpenReader(zipPath)
|
||||
if err != nil {
|
||||
return info, fmt.Errorf("failed to open ZIP file: %v", err)
|
||||
}
|
||||
defer r.Close()
|
||||
|
||||
if err := validatePluginArchive(r.File); err != nil {
|
||||
return info, err
|
||||
}
|
||||
|
||||
var manifest *zip.File
|
||||
for _, f := range r.File {
|
||||
if f.Name == manifestFile {
|
||||
manifest = f
|
||||
break
|
||||
}
|
||||
}
|
||||
if manifest == nil {
|
||||
return info, fmt.Errorf("plugin manifest %s not found, not a valid plugin package", manifestFile)
|
||||
}
|
||||
|
||||
rc, err := manifest.Open()
|
||||
if err != nil {
|
||||
return info, fmt.Errorf("failed to read plugin manifest: %v", err)
|
||||
}
|
||||
configData, readErr := io.ReadAll(io.LimitReader(rc, maxPluginManifestSize+1))
|
||||
_ = rc.Close()
|
||||
if readErr != nil {
|
||||
return info, fmt.Errorf("failed to read plugin manifest: %v", readErr)
|
||||
}
|
||||
if len(configData) > maxPluginManifestSize {
|
||||
return info, fmt.Errorf("plugin manifest exceeds the %d byte limit", maxPluginManifestSize)
|
||||
}
|
||||
if err := json.Unmarshal(configData, &info); err != nil {
|
||||
return info, fmt.Errorf("invalid plugin manifest: %v", err)
|
||||
}
|
||||
if err := validateManifest(&info); err != nil {
|
||||
return info, err
|
||||
}
|
||||
if err := CheckKomariVersion(info.Komari); err != nil {
|
||||
return info, err
|
||||
}
|
||||
|
||||
if err := global.unload(info.Short); err != nil && !errors.Is(err, errNotLoaded) {
|
||||
return info, fmt.Errorf("failed to unload running plugin %q before reinstall: %w", info.Short, err)
|
||||
}
|
||||
|
||||
dir := filepath.Join(DataDir, info.Short)
|
||||
if err := os.RemoveAll(dir); err != nil {
|
||||
return info, fmt.Errorf("failed to remove existing plugin directory: %v", err)
|
||||
}
|
||||
if err := os.MkdirAll(dir, 0755); err != nil {
|
||||
return info, fmt.Errorf("failed to create plugin directory: %v", err)
|
||||
}
|
||||
if err := extractPluginArchive(r.File, dir); err != nil {
|
||||
_ = os.RemoveAll(dir)
|
||||
return info, err
|
||||
}
|
||||
if _, err := os.Stat(filepath.Join(dir, info.Entry)); err != nil {
|
||||
_ = os.RemoveAll(dir)
|
||||
return info, fmt.Errorf("plugin entry %s does not exist", info.Entry)
|
||||
}
|
||||
for _, page := range info.Pages {
|
||||
if _, err := os.Stat(filepath.Join(dir, page.File)); err != nil {
|
||||
_ = os.RemoveAll(dir)
|
||||
return info, fmt.Errorf("plugin page %s does not exist", page.File)
|
||||
}
|
||||
}
|
||||
if global.stateStore().get(info.Short).Enabled {
|
||||
if err := global.restartPlugin(info.Short); err != nil {
|
||||
return info, fmt.Errorf("plugin reinstalled but reload failed: %w", err)
|
||||
}
|
||||
}
|
||||
return info, nil
|
||||
}
|
||||
|
||||
func validatePluginArchive(files []*zip.File) error {
|
||||
if len(files) > maxPluginArchiveFiles {
|
||||
return fmt.Errorf("plugin archive has more than %d files", maxPluginArchiveFiles)
|
||||
}
|
||||
var total uint64
|
||||
for _, file := range files {
|
||||
if file.FileInfo().IsDir() {
|
||||
continue
|
||||
}
|
||||
if file.UncompressedSize64 > maxPluginFileSize {
|
||||
return fmt.Errorf("plugin file %s exceeds the %d byte limit", file.Name, maxPluginFileSize)
|
||||
}
|
||||
total += file.UncompressedSize64
|
||||
if total > maxPluginExtractedSize {
|
||||
return fmt.Errorf("plugin archive exceeds the %d byte extraction limit", maxPluginExtractedSize)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func extractPluginArchive(files []*zip.File, dir string) error {
|
||||
for _, f := range files {
|
||||
path := filepath.Join(dir, f.Name)
|
||||
if !withinDir(path, dir) {
|
||||
return fmt.Errorf("plugin archive contains an invalid path %q", f.Name)
|
||||
}
|
||||
if f.FileInfo().IsDir() {
|
||||
if err := os.MkdirAll(path, 0755); err != nil {
|
||||
return fmt.Errorf("failed to create directory: %v", err)
|
||||
}
|
||||
continue
|
||||
}
|
||||
if err := os.MkdirAll(filepath.Dir(path), 0755); err != nil {
|
||||
return fmt.Errorf("failed to create directory: %v", err)
|
||||
}
|
||||
rc, err := f.Open()
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to open archive file: %v", err)
|
||||
}
|
||||
outFile, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0644)
|
||||
if err != nil {
|
||||
_ = rc.Close()
|
||||
return fmt.Errorf("failed to create file: %v", err)
|
||||
}
|
||||
_, copyErr := io.Copy(outFile, rc)
|
||||
_ = outFile.Close()
|
||||
_ = rc.Close()
|
||||
if copyErr != nil {
|
||||
return fmt.Errorf("failed to extract file: %v", copyErr)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// withinDir reports whether path stays inside dir after cleaning.
|
||||
func withinDir(path, dir string) bool {
|
||||
rel, err := filepath.Rel(dir, path)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
return rel != ".." && !strings.HasPrefix(rel, ".."+string(os.PathSeparator)) && !filepath.IsAbs(rel)
|
||||
}
|
||||
@@ -0,0 +1,191 @@
|
||||
package plugin
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"github.com/komari-monitor/komari/database/models"
|
||||
)
|
||||
|
||||
func TestInstallZipExtractsValidPlugin(t *testing.T) {
|
||||
withTempDataDir(t)
|
||||
zipPath := writePluginZip(t, map[string]string{
|
||||
"komari-plugin.json": `{"name":"Demo","short":"demo","version":"1.0.0","komari":">=0.0.1","permissions":{"node":true}}`,
|
||||
"script.js": `function load() {}`,
|
||||
})
|
||||
info, err := InstallZip(zipPath)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if info.Short != "demo" {
|
||||
t.Fatalf("short = %q", info.Short)
|
||||
}
|
||||
if !info.Permissions.Node {
|
||||
t.Fatal("permissions.node not parsed")
|
||||
}
|
||||
if _, err := os.Stat(filepath.Join(DataDir, "demo", "script.js")); err != nil {
|
||||
t.Fatalf("entry not extracted: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestInstallZipDefaultsEntryToScriptJS(t *testing.T) {
|
||||
withTempDataDir(t)
|
||||
zipPath := writePluginZip(t, map[string]string{
|
||||
"komari-plugin.json": `{"name":"Demo","short":"demo","version":"1.0.0"}`,
|
||||
"script.js": `function load() {}`,
|
||||
})
|
||||
info, err := InstallZip(zipPath)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if info.Entry != "script.js" {
|
||||
t.Fatalf("entry = %q, want script.js", info.Entry)
|
||||
}
|
||||
}
|
||||
|
||||
func TestInstallZipRejectsMissingManifest(t *testing.T) {
|
||||
withTempDataDir(t)
|
||||
zipPath := writePluginZip(t, map[string]string{"script.js": "function load() {}"})
|
||||
if _, err := InstallZip(zipPath); err == nil {
|
||||
t.Fatal("expected missing manifest error")
|
||||
}
|
||||
}
|
||||
|
||||
func TestInstallZipRejectsTraversal(t *testing.T) {
|
||||
withTempDataDir(t)
|
||||
zipPath := writePluginZip(t, map[string]string{
|
||||
"komari-plugin.json": `{"name":"Demo","short":"demo","version":"1.0.0"}`,
|
||||
"../evil.txt": "boom",
|
||||
})
|
||||
if _, err := InstallZip(zipPath); err == nil {
|
||||
t.Fatal("expected traversal rejection")
|
||||
}
|
||||
if _, err := os.Stat(filepath.Join(DataDir, "demo")); !os.IsNotExist(err) {
|
||||
t.Fatal("plugin directory must be removed after a failed install")
|
||||
}
|
||||
}
|
||||
|
||||
func TestInstallZipRejectsKomariVersionMismatch(t *testing.T) {
|
||||
withTempDataDir(t)
|
||||
zipPath := writePluginZip(t, map[string]string{
|
||||
"komari-plugin.json": `{"name":"Demo","short":"demo","version":"1.0.0","komari":">=99.0.0"}`,
|
||||
"script.js": `function load() {}`,
|
||||
})
|
||||
if _, err := InstallZip(zipPath); err == nil {
|
||||
t.Fatal("expected komari version mismatch error")
|
||||
}
|
||||
}
|
||||
|
||||
func TestInstallZipRejectsMissingEntry(t *testing.T) {
|
||||
withTempDataDir(t)
|
||||
zipPath := writePluginZip(t, map[string]string{
|
||||
"komari-plugin.json": `{"name":"Demo","short":"demo","version":"1.0.0"}`,
|
||||
})
|
||||
if _, err := InstallZip(zipPath); err == nil {
|
||||
t.Fatal("expected missing entry error")
|
||||
}
|
||||
}
|
||||
func TestInstallZipValidatesPageTypesAndVisibility(t *testing.T) {
|
||||
withTempDataDir(t)
|
||||
|
||||
// iframe page 默认 visibility=admin、type=iframe;合法安装。
|
||||
zipPath := writePluginZip(t, map[string]string{
|
||||
"komari-plugin.json": `{"name":"Pages","short":"pages","version":"1.0.0","pages":[{"file":"admin.html","title":"Admin"}]}`,
|
||||
"script.js": `function load() {}`,
|
||||
"admin.html": `<h1>admin</h1>`,
|
||||
})
|
||||
info, err := InstallZip(zipPath)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(info.Pages) != 1 || info.Pages[0].Type != models.PageTypeIframe || info.Pages[0].Visibility != models.PageVisibilityAdmin {
|
||||
t.Fatalf("page defaults = %+v", info.Pages)
|
||||
}
|
||||
|
||||
// redirect 页面需要站内路径。
|
||||
zipPath = writePluginZip(t, map[string]string{
|
||||
"komari-plugin.json": `{"name":"Pages","short":"pages2","version":"1.0.0","pages":[{"title":"Go","type":"redirect","url":"/admin/settings","visibility":"admin"}]}`,
|
||||
"script.js": `function load() {}`,
|
||||
})
|
||||
if _, err := InstallZip(zipPath); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// public iframe 页面合法。
|
||||
zipPath = writePluginZip(t, map[string]string{
|
||||
"komari-plugin.json": `{"name":"Pages","short":"pages3","version":"1.0.0","pages":[{"file":"pub.html","title":"Pub","visibility":"public"}]}`,
|
||||
"script.js": `function load() {}`,
|
||||
"pub.html": `<h1>pub</h1>`,
|
||||
})
|
||||
if _, err := InstallZip(zipPath); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestInstallZipRejectsInvalidPages(t *testing.T) {
|
||||
withTempDataDir(t)
|
||||
cases := []struct {
|
||||
name string
|
||||
manifest string
|
||||
}{
|
||||
{"bad type", `{"name":"P","short":"p1","version":"1.0.0","pages":[{"file":"a.html","title":"A","type":"popup"}]}`},
|
||||
{"redirect without url", `{"name":"P","short":"p2","version":"1.0.0","pages":[{"title":"A","type":"redirect"}]}`},
|
||||
{"redirect external url", `{"name":"P","short":"p3","version":"1.0.0","pages":[{"title":"A","type":"redirect","url":"https://evil.example/"}]}`},
|
||||
{"redirect protocol-relative", `{"name":"P","short":"p4","version":"1.0.0","pages":[{"title":"A","type":"redirect","url":"//evil.example/"}]}`},
|
||||
{"redirect traversal", `{"name":"P","short":"p5","version":"1.0.0","pages":[{"title":"A","type":"redirect","url":"/admin/../secret"}]}`},
|
||||
{"bad visibility", `{"name":"P","short":"p6","version":"1.0.0","pages":[{"file":"a.html","title":"A","visibility":"everyone"}]}`},
|
||||
{"iframe without file", `{"name":"P","short":"p7","version":"1.0.0","pages":[{"title":"A"}]}`},
|
||||
}
|
||||
for _, tt := range cases {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
zipPath := writePluginZip(t, map[string]string{
|
||||
"komari-plugin.json": tt.manifest,
|
||||
"script.js": `function load() {}`,
|
||||
})
|
||||
if _, err := InstallZip(zipPath); err == nil {
|
||||
t.Fatalf("expected rejection for %s", tt.name)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
func TestInstallZipRejectsInvalidIcon(t *testing.T) {
|
||||
withTempDataDir(t)
|
||||
cases := []struct {
|
||||
name string
|
||||
manifest string
|
||||
}{
|
||||
{"plugin icon traversal", `{"name":"I","short":"i1","version":"1.0.0","icon":"../evil.png"}`},
|
||||
{"plugin icon absolute", `{"name":"I","short":"i2","version":"1.0.0","icon":"/etc/passwd"}`},
|
||||
{"page icon traversal", `{"name":"I","short":"i3","version":"1.0.0","pages":[{"file":"a.html","title":"A","icon":"../evil.png"}]}`},
|
||||
}
|
||||
for _, tt := range cases {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
zipPath := writePluginZip(t, map[string]string{
|
||||
"komari-plugin.json": tt.manifest,
|
||||
"script.js": `function load() {}`,
|
||||
})
|
||||
if _, err := InstallZip(zipPath); err == nil {
|
||||
t.Fatalf("expected rejection for %s", tt.name)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestInstallZipAcceptsRelativeIcon(t *testing.T) {
|
||||
withTempDataDir(t)
|
||||
zipPath := writePluginZip(t, map[string]string{
|
||||
"komari-plugin.json": `{"name":"I","short":"ic","version":"1.0.0","icon":"icon.svg","pages":[{"file":"a.html","title":"A","icon":"page.svg"}]}`,
|
||||
"script.js": `function load() {}`,
|
||||
"a.html": `<h1>a</h1>`,
|
||||
"icon.svg": `<svg/>`,
|
||||
"page.svg": `<svg/>`,
|
||||
})
|
||||
info, err := InstallZip(zipPath)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if info.Icon != "icon.svg" || info.Pages[0].Icon != "page.svg" {
|
||||
t.Fatalf("icons = %+v", info)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,44 @@
|
||||
package plugin
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"sync"
|
||||
)
|
||||
|
||||
// LogBuffer is a bounded ring buffer holding one plugin's console output and
|
||||
// lifecycle markers. Old bytes are dropped from the front when the limit is
|
||||
// reached.
|
||||
type LogBuffer struct {
|
||||
mu sync.Mutex
|
||||
buf bytes.Buffer
|
||||
limit int
|
||||
}
|
||||
|
||||
func newLogBuffer(limit int) *LogBuffer {
|
||||
if limit <= 0 {
|
||||
limit = defaultLogBufferSize
|
||||
}
|
||||
return &LogBuffer{limit: limit}
|
||||
}
|
||||
|
||||
func (b *LogBuffer) Write(p []byte) (int, error) {
|
||||
b.mu.Lock()
|
||||
defer b.mu.Unlock()
|
||||
_, _ = b.buf.Write(p)
|
||||
if b.buf.Len() > b.limit {
|
||||
b.buf.Next(b.buf.Len() - b.limit)
|
||||
}
|
||||
return len(p), nil
|
||||
}
|
||||
|
||||
func (b *LogBuffer) String() string {
|
||||
b.mu.Lock()
|
||||
defer b.mu.Unlock()
|
||||
return b.buf.String()
|
||||
}
|
||||
|
||||
func (b *LogBuffer) Reset() {
|
||||
b.mu.Lock()
|
||||
defer b.mu.Unlock()
|
||||
b.buf.Reset()
|
||||
}
|
||||
@@ -0,0 +1,69 @@
|
||||
package plugin
|
||||
|
||||
import (
|
||||
"archive/zip"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"github.com/komari-monitor/komari/cmd/flags"
|
||||
"github.com/komari-monitor/komari/database/dbcore"
|
||||
)
|
||||
|
||||
func TestMain(m *testing.M) {
|
||||
flags.DatabaseType = flags.DatabaseTypeSQLite
|
||||
flags.DatabaseFile = "file:komari_plugin_test?mode=memory&cache=shared"
|
||||
db := dbcore.GetDBInstance()
|
||||
if sqlDB, err := db.DB(); err == nil {
|
||||
sqlDB.SetMaxOpenConns(1)
|
||||
}
|
||||
os.Exit(m.Run())
|
||||
}
|
||||
|
||||
// withTempDataDir redirects DataDir and StorageDir and resets the global
|
||||
// manager for the duration of a test.
|
||||
func withTempDataDir(t *testing.T) {
|
||||
t.Helper()
|
||||
oldDir := DataDir
|
||||
oldStorage := StorageDir
|
||||
oldGlobal := global
|
||||
DataDir = t.TempDir()
|
||||
StorageDir = t.TempDir()
|
||||
mgr := &Manager{
|
||||
instances: make(map[string]*Instance),
|
||||
routes: make(map[string]map[string]bool),
|
||||
logs: make(map[string]*LogBuffer),
|
||||
}
|
||||
global = mgr
|
||||
t.Cleanup(func() { _ = mgr.closeAll() }) // release runtimes before TempDir removal
|
||||
t.Cleanup(func() {
|
||||
DataDir = oldDir
|
||||
StorageDir = oldStorage
|
||||
global = oldGlobal
|
||||
})
|
||||
}
|
||||
|
||||
// writePluginZip creates a plugin ZIP from a name -> content map.
|
||||
func writePluginZip(t *testing.T, files map[string]string) string {
|
||||
t.Helper()
|
||||
path := filepath.Join(t.TempDir(), "plugin.zip")
|
||||
f, err := os.Create(path)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer f.Close()
|
||||
w := zip.NewWriter(f)
|
||||
for name, content := range files {
|
||||
entry, err := w.Create(name)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := entry.Write([]byte(content)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if err := w.Close(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return path
|
||||
}
|
||||
@@ -0,0 +1,244 @@
|
||||
package plugin
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/gin-gonic/gin"
|
||||
"github.com/komari-monitor/komari/database/models"
|
||||
"github.com/komari-monitor/komari/pkg/rpc"
|
||||
)
|
||||
|
||||
const demoManifest = `{"name":"Demo","short":"demo","version":"1.0.0","komari":">=0.0.1","permissions":{"node":true,"timeout":5,"allowRoutes":true,"allowSystemRPC":true}}`
|
||||
|
||||
func TestManagerSwitchRouteCallAndLogs(t *testing.T) {
|
||||
withTempDataDir(t)
|
||||
gin.SetMode(gin.TestMode)
|
||||
engine := gin.New()
|
||||
Init(engine)
|
||||
|
||||
zipPath := writePluginZip(t, map[string]string{
|
||||
"komari-plugin.json": demoManifest,
|
||||
"script.js": `
|
||||
const server = require("server");
|
||||
function load() {
|
||||
console.log("demo loaded");
|
||||
server.route("GET", "/plug", async (req, res) => {
|
||||
const result = await server.call("plugin:testEcho", { x: req.query.x });
|
||||
res.setHeader("Content-Type", "application/json");
|
||||
res.end(JSON.stringify(result));
|
||||
});
|
||||
}
|
||||
function unload() {
|
||||
console.log("demo unloaded");
|
||||
}
|
||||
`,
|
||||
})
|
||||
if _, err := InstallZip(zipPath); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// The switch must ask for permission approval on first enable.
|
||||
if err := SetEnabled("demo", true, false); err != ErrPermissionApprovalRequired {
|
||||
t.Fatalf("first enable: expected permission approval required, got %v", err)
|
||||
}
|
||||
// Approving permissions and enabling must load the plugin.
|
||||
if err := SetEnabled("demo", true, true); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// Register a test RPC method the plugin calls through server.call.
|
||||
if err := rpc.Register("plugin:testEcho", func(_ context.Context, req *rpc.JsonRpcRequest) (any, *rpc.JsonRpcError) {
|
||||
return req.Params, nil
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer rpc.Unregister("plugin:testEcho")
|
||||
|
||||
rec := httptest.NewRecorder()
|
||||
engine.ServeHTTP(rec, httptest.NewRequest(http.MethodGet, "/plug?x=7", nil))
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("route status = %d, body = %s", rec.Code, rec.Body.String())
|
||||
}
|
||||
var body map[string]any
|
||||
if err := json.Unmarshal(rec.Body.Bytes(), &body); err != nil {
|
||||
t.Fatalf("route body is not JSON: %v (%s)", err, rec.Body.String())
|
||||
}
|
||||
if body["x"] != "7" {
|
||||
t.Fatalf("server.call echo mismatch: %s", rec.Body.String())
|
||||
}
|
||||
|
||||
logs := GetLogs("demo")
|
||||
if !strings.Contains(logs, "demo loaded") {
|
||||
t.Fatalf("logs missing console output: %q", logs)
|
||||
}
|
||||
|
||||
// Disabling runs unload() and makes the route slot return 404.
|
||||
if err := SetEnabled("demo", false, false); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
rec2 := httptest.NewRecorder()
|
||||
engine.ServeHTTP(rec2, httptest.NewRequest(http.MethodGet, "/plug", nil))
|
||||
if rec2.Code != http.StatusNotFound {
|
||||
t.Fatalf("expected 404 after unload, got %d", rec2.Code)
|
||||
}
|
||||
if logs := GetLogs("demo"); !strings.Contains(logs, "demo unloaded") {
|
||||
t.Fatalf("logs missing unload marker: %q", logs)
|
||||
}
|
||||
|
||||
// Re-enabling reuses the route slot (approval already stored).
|
||||
if err := SetEnabled("demo", true, false); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
rec3 := httptest.NewRecorder()
|
||||
engine.ServeHTTP(rec3, httptest.NewRequest(http.MethodGet, "/plug?x=9", nil))
|
||||
if rec3.Code != http.StatusOK || !strings.Contains(rec3.Body.String(), "9") {
|
||||
t.Fatalf("reload route status = %d, body = %s", rec3.Code, rec3.Body.String())
|
||||
}
|
||||
|
||||
infos := List()
|
||||
if len(infos) != 1 || infos[0].Short != "demo" || !infos[0].Enabled || !infos[0].Running {
|
||||
t.Fatalf("List mismatch: %+v", infos)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSetEnabledFailureAutoDisablesAndRecordsError(t *testing.T) {
|
||||
withTempDataDir(t)
|
||||
gin.SetMode(gin.TestMode)
|
||||
Init(gin.New())
|
||||
|
||||
zipPath := writePluginZip(t, map[string]string{
|
||||
"komari-plugin.json": `{"name":"Bad","short":"bad","version":"1.0.0"}`,
|
||||
"script.js": `throw new Error("boom");`,
|
||||
})
|
||||
if _, err := InstallZip(zipPath); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
err := SetEnabled("bad", true, true)
|
||||
if err == nil || !strings.Contains(err.Error(), "boom") {
|
||||
t.Fatalf("expected boom error, got %v", err)
|
||||
}
|
||||
st := global.stateStore().get("bad")
|
||||
if st.Enabled {
|
||||
t.Fatal("failed plugin must be auto-disabled")
|
||||
}
|
||||
if !strings.Contains(st.LastError, "boom") {
|
||||
t.Fatalf("last_error = %q", st.LastError)
|
||||
}
|
||||
infos := List()
|
||||
if len(infos) != 1 || infos[0].Enabled || infos[0].Running || !strings.Contains(infos[0].LastError, "boom") {
|
||||
t.Fatalf("List mismatch: %+v", infos)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadHookFailureIsReported(t *testing.T) {
|
||||
withTempDataDir(t)
|
||||
gin.SetMode(gin.TestMode)
|
||||
Init(gin.New())
|
||||
|
||||
zipPath := writePluginZip(t, map[string]string{
|
||||
"komari-plugin.json": `{"name":"Hook","short":"hook","version":"1.0.0"}`,
|
||||
"script.js": `function load() {
|
||||
throw new Error("loadhook");
|
||||
}`,
|
||||
})
|
||||
if _, err := InstallZip(zipPath); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
err := SetEnabled("hook", true, true)
|
||||
if err == nil || !strings.Contains(err.Error(), "load() failed") || !strings.Contains(err.Error(), "loadhook") {
|
||||
t.Fatalf("unexpected load() error: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadAllSkipsDisabledAndApprovesEnabled(t *testing.T) {
|
||||
withTempDataDir(t)
|
||||
gin.SetMode(gin.TestMode)
|
||||
Init(gin.New())
|
||||
|
||||
zipPath := writePluginZip(t, map[string]string{
|
||||
"komari-plugin.json": demoManifest,
|
||||
"script.js": `function load() { console.log("auto loaded"); }`,
|
||||
})
|
||||
if _, err := InstallZip(zipPath); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// Not enabled yet: LoadAll must not load it.
|
||||
if err := LoadAll(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if inst := global.instanceFor("demo"); inst != nil {
|
||||
t.Fatal("disabled plugin was loaded")
|
||||
}
|
||||
|
||||
// Persist an enabled+approved state, then LoadAll must load it.
|
||||
hash := approvalPermissionsHash(models.PluginPermissions{AllowRoutes: true, AllowSystemRPC: true})
|
||||
global.stateStore().set("demo", PluginState{Enabled: true, ApprovedPermissionsHash: hash})
|
||||
if err := LoadAll(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
infos := List()
|
||||
if len(infos) != 1 || !infos[0].Running {
|
||||
t.Fatalf("enabled plugin not running after LoadAll: %+v", infos)
|
||||
}
|
||||
if logs := GetLogs("demo"); !strings.Contains(logs, "auto loaded") {
|
||||
t.Fatalf("logs missing output: %q", logs)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadAllAutoDisablesUnapprovedPlugin(t *testing.T) {
|
||||
withTempDataDir(t)
|
||||
gin.SetMode(gin.TestMode)
|
||||
Init(gin.New())
|
||||
|
||||
zipPath := writePluginZip(t, map[string]string{
|
||||
"komari-plugin.json": demoManifest,
|
||||
"script.js": `function load() {}`,
|
||||
})
|
||||
if _, err := InstallZip(zipPath); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// Simulate a previously enabled plugin whose approval was lost.
|
||||
global.stateStore().set("demo", PluginState{Enabled: true})
|
||||
err := LoadAll()
|
||||
if err == nil || !strings.Contains(err.Error(), "approval") {
|
||||
t.Fatalf("expected approval error, got %v", err)
|
||||
}
|
||||
st := global.stateStore().get("demo")
|
||||
if st.Enabled {
|
||||
t.Fatal("unapproved plugin must be auto-disabled")
|
||||
}
|
||||
if !strings.Contains(st.LastError, "approval") {
|
||||
t.Fatalf("last_error = %q", st.LastError)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCloseAllUnloadsEverything(t *testing.T) {
|
||||
withTempDataDir(t)
|
||||
gin.SetMode(gin.TestMode)
|
||||
Init(gin.New())
|
||||
|
||||
zipPath := writePluginZip(t, map[string]string{
|
||||
"komari-plugin.json": demoManifest,
|
||||
"script.js": `function unload() { console.log("bye"); }`,
|
||||
})
|
||||
if _, err := InstallZip(zipPath); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := SetEnabled("demo", true, true); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := CloseAll(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if inst := global.instanceFor("demo"); inst != nil {
|
||||
t.Fatal("plugin still loaded after CloseAll")
|
||||
}
|
||||
if logs := GetLogs("demo"); !strings.Contains(logs, "bye") {
|
||||
t.Fatalf("unload hook not called: %q", logs)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,135 @@
|
||||
package plugin
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"strings"
|
||||
|
||||
"github.com/komari-monitor/komari/database/models"
|
||||
)
|
||||
|
||||
// urlSchemeRE matches a leading URL scheme such as "http:" or "javascript:",
|
||||
// mirroring the theme redirect validation.
|
||||
var urlSchemeRE = regexp.MustCompile(`^[a-zA-Z][a-zA-Z0-9+\-.]*:`)
|
||||
|
||||
// readManifest loads and validates the manifest of an installed plugin
|
||||
// directory.
|
||||
func readManifest(dir string) (models.Plugin, error) {
|
||||
var info models.Plugin
|
||||
data, err := os.ReadFile(filepath.Join(dir, manifestFile))
|
||||
if err != nil {
|
||||
return info, fmt.Errorf("read plugin manifest: %w", err)
|
||||
}
|
||||
if err := json.Unmarshal(data, &info); err != nil {
|
||||
return info, fmt.Errorf("parse plugin manifest: %w", err)
|
||||
}
|
||||
if err := validateManifest(&info); err != nil {
|
||||
return info, err
|
||||
}
|
||||
return info, nil
|
||||
}
|
||||
|
||||
// validateManifest checks the fields the server relies on and fills in
|
||||
// defaults.
|
||||
func validateManifest(info *models.Plugin) error {
|
||||
if info.Short == "" {
|
||||
return fmt.Errorf("plugin short is required")
|
||||
}
|
||||
if !validShort(info.Short) {
|
||||
return fmt.Errorf("plugin short %q is invalid: only letters, digits, '_' and '-' are allowed", info.Short)
|
||||
}
|
||||
if !localizedText(info.Name) {
|
||||
return fmt.Errorf("plugin name is required")
|
||||
}
|
||||
if info.Entry == "" {
|
||||
info.Entry = defaultEntry
|
||||
}
|
||||
if !filepath.IsLocal(info.Entry) {
|
||||
return fmt.Errorf("plugin entry %q must be a relative path inside the plugin directory", info.Entry)
|
||||
}
|
||||
if info.Icon != "" && !filepath.IsLocal(info.Icon) {
|
||||
return fmt.Errorf("plugin icon %q must be a relative path inside the plugin directory", info.Icon)
|
||||
}
|
||||
for i := range info.Pages {
|
||||
if err := validatePage(&info.Pages[i]); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// validatePage checks one declared plugin page. Type and visibility default
|
||||
// to iframe/admin so existing manifests keep their behavior.
|
||||
func validatePage(page *models.PluginPage) error {
|
||||
if page.Type == "" {
|
||||
page.Type = models.PageTypeIframe
|
||||
}
|
||||
if page.Visibility == "" {
|
||||
page.Visibility = models.PageVisibilityAdmin
|
||||
}
|
||||
switch page.Visibility {
|
||||
case models.PageVisibilityAdmin, models.PageVisibilityPublic:
|
||||
default:
|
||||
return fmt.Errorf("plugin page visibility %q is invalid: use \"admin\" or \"public\"", page.Visibility)
|
||||
}
|
||||
if page.Icon != "" && !filepath.IsLocal(page.Icon) {
|
||||
return fmt.Errorf("plugin page icon %q must be a relative path inside the plugin directory", page.Icon)
|
||||
}
|
||||
switch page.Type {
|
||||
case models.PageTypeIframe:
|
||||
if page.File == "" || !filepath.IsLocal(page.File) {
|
||||
return fmt.Errorf("plugin iframe page requires a relative file path inside the plugin directory")
|
||||
}
|
||||
case models.PageTypeRedirect:
|
||||
if page.URL == "" || !isSafeInternalPath(page.URL) {
|
||||
return fmt.Errorf("plugin redirect page requires an internal site path starting with /")
|
||||
}
|
||||
default:
|
||||
return fmt.Errorf("plugin page type %q is invalid: use \"iframe\" or \"redirect\"", page.Type)
|
||||
}
|
||||
if !localizedText(page.Title) {
|
||||
return fmt.Errorf("plugin page %q requires a title", page.Title)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// isSafeInternalPath mirrors the theme redirect rule: only same-origin
|
||||
// relative paths are allowed, no scheme, no backslashes, no traversal.
|
||||
func isSafeInternalPath(target string) bool {
|
||||
target = strings.TrimSpace(target)
|
||||
if !strings.HasPrefix(target, "/") || strings.HasPrefix(target, "//") {
|
||||
return false
|
||||
}
|
||||
if strings.Contains(target, "\\") || urlSchemeRE.MatchString(target) {
|
||||
return false
|
||||
}
|
||||
for _, segment := range strings.Split(target, "/") {
|
||||
if segment == ".." {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// localizedText reports whether v is a usable name/description/author value:
|
||||
// a non-empty string, or an i18n object with at least one non-empty value.
|
||||
func localizedText(v any) bool {
|
||||
switch t := v.(type) {
|
||||
case nil:
|
||||
return false
|
||||
case string:
|
||||
return strings.TrimSpace(t) != ""
|
||||
case map[string]any:
|
||||
for _, value := range t {
|
||||
if s, ok := value.(string); ok && strings.TrimSpace(s) != "" {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,759 @@
|
||||
// Package plugin manages Komari plugins: ZIP packages with a
|
||||
// komari-plugin.json manifest, mirroring the theme package format. A plugin
|
||||
// runs in its own jsruntime instance confined to its data/plugin/<short>
|
||||
// directory, plus its long-term data directory data/plugin-data/<short>
|
||||
// exposed as __storageDir__ (which survives plugin updates), declares runtime
|
||||
// permissions in its manifest, and receives the host-injected "server" module
|
||||
// (server.route / server.call / server.hook, whose ws kinds can intercept
|
||||
// WebSocket connections and frames). Plugins run with admin authority inside
|
||||
// the system.
|
||||
//
|
||||
// Lifecycle: plugins are installed as directories under DataDir. The
|
||||
// persisted state file (state.json) records which plugins are enabled, which
|
||||
// permissions were approved, and the last load error. Enabled plugins are
|
||||
// loaded at startup; a failed load auto-disables the plugin and keeps the
|
||||
// error text visible through admin:listPlugins.
|
||||
package plugin
|
||||
|
||||
import (
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/dop251/goja"
|
||||
"github.com/dop251/goja_nodejs/require"
|
||||
"github.com/gin-gonic/gin"
|
||||
"github.com/komari-monitor/komari/database/models"
|
||||
"github.com/komari-monitor/komari/internal/scheduler"
|
||||
"github.com/komari-monitor/komari/pkg/jsruntime"
|
||||
"github.com/komari-monitor/komari/pkg/rpc"
|
||||
"github.com/komari-monitor/komari/web/connection"
|
||||
)
|
||||
|
||||
// DataDir is the on-disk root for installed plugins, mirroring the theme
|
||||
// directory layout. It is a variable so tests can redirect it.
|
||||
var DataDir = "./data/plugin"
|
||||
|
||||
// StorageDir is the on-disk root for long-term plugin data. Each plugin gets
|
||||
// StorageDir/<short>. The directory is confined into the plugin runtime as
|
||||
// __storageDir__ and survives plugin reinstalls: update removes only
|
||||
// DataDir/<short> (the code from the ZIP) while StorageDir/<short> persists.
|
||||
// Deleting a plugin removes both.
|
||||
var StorageDir = "./data/plugin-data"
|
||||
|
||||
const (
|
||||
manifestFile = "komari-plugin.json"
|
||||
defaultEntry = "script.js"
|
||||
|
||||
defaultMaxHTTPBodyBytes int64 = 32 << 20
|
||||
defaultLogBufferSize = 64 << 10
|
||||
|
||||
stateFileName = "state.json"
|
||||
)
|
||||
|
||||
// maxHookBufferBytes caps how much of a response the hook chain buffers
|
||||
// for rewriting. Larger responses pass through to the client unmodified;
|
||||
// it is a variable so tests can lower it.
|
||||
var maxHookBufferBytes = 32 << 20
|
||||
|
||||
// maxHTMLInjectBufferBytes caps how much of a text/html response the HTML
|
||||
// injector buffers before injecting plugin fragments. Larger responses pass
|
||||
// through unmodified; it is a variable so tests can lower it.
|
||||
var maxHTMLInjectBufferBytes = 32 << 20
|
||||
|
||||
const (
|
||||
maxPluginArchiveFiles = 10000
|
||||
maxPluginFileSize = 128 << 20
|
||||
maxPluginExtractedSize = 512 << 20
|
||||
maxPluginManifestSize = 1 << 20
|
||||
)
|
||||
|
||||
// ErrPermissionApprovalRequired is returned when enabling a plugin whose
|
||||
// declared permissions have not been approved, or changed since approval.
|
||||
var ErrPermissionApprovalRequired = errors.New("plugin permissions require approval")
|
||||
|
||||
var errNotLoaded = errors.New("plugin is not loaded")
|
||||
|
||||
// ErrNotInstalled is returned when deleting a plugin that is not installed.
|
||||
var ErrNotInstalled = errors.New("plugin is not installed")
|
||||
|
||||
// Info describes one installed plugin for the admin surface.
|
||||
type Info struct {
|
||||
models.Plugin
|
||||
Enabled bool `json:"enabled"`
|
||||
Running bool `json:"running"`
|
||||
LastError string `json:"last_error"`
|
||||
}
|
||||
|
||||
// Manager owns the installed plugin set: load/unload lifecycle, persisted
|
||||
// state, per-plugin logs, and the host engine used by server.route.
|
||||
type Manager struct {
|
||||
mu sync.RWMutex
|
||||
engine *gin.Engine
|
||||
instances map[string]*Instance
|
||||
routes map[string]map[string]bool // short -> "METHOD path" -> gin slot registered
|
||||
hooks []*hookEntry // http request/response hooks in registration order
|
||||
injects []*injectEntry // html head/body fragments in registration order
|
||||
|
||||
stateMu sync.Mutex
|
||||
state *State
|
||||
|
||||
logsMu sync.Mutex
|
||||
logs map[string]*LogBuffer
|
||||
}
|
||||
|
||||
// Instance is one loaded plugin runtime plus its host bindings.
|
||||
type Instance struct {
|
||||
mu sync.RWMutex
|
||||
info models.Plugin
|
||||
dir string
|
||||
runtime *jsruntime.Runtime
|
||||
host *jsruntime.Host
|
||||
handlers map[string]goja.Callable // "METHOD path" -> current route handler
|
||||
statics map[string]*staticConfig // mount path -> static folder config
|
||||
rpcMethods map[string]goja.Callable // registered RPC method -> JS handler
|
||||
cronJobs []string // scheduler job names, removed on unload
|
||||
}
|
||||
|
||||
// global is the process-wide plugin manager.
|
||||
var global = &Manager{
|
||||
instances: make(map[string]*Instance),
|
||||
routes: make(map[string]map[string]bool),
|
||||
logs: make(map[string]*LogBuffer),
|
||||
}
|
||||
|
||||
// Init wires the manager to the HTTP engine and into every WebSocket
|
||||
// connection. It must be called before any plugin is loaded.
|
||||
func Init(engine *gin.Engine) {
|
||||
global.mu.Lock()
|
||||
global.engine = engine
|
||||
global.mu.Unlock()
|
||||
connection.SetFrameInterceptor(global)
|
||||
}
|
||||
|
||||
// LoadAll loads every installed plugin whose persisted state is enabled and
|
||||
// approved. A failed load auto-disables the plugin, persists last_error and
|
||||
// is returned joined so callers can log it without stopping startup.
|
||||
func LoadAll() error {
|
||||
return global.loadAll()
|
||||
}
|
||||
|
||||
// SetEnabled is the single switch entry point. Enabling a plugin whose
|
||||
// declared permissions differ from the approved hash requires approved=true.
|
||||
func SetEnabled(short string, enabled, approved bool) error {
|
||||
return global.setEnabled(short, enabled, approved)
|
||||
}
|
||||
|
||||
// CloseAll unloads every loaded plugin. Used during server shutdown.
|
||||
func CloseAll() error {
|
||||
return global.closeAll()
|
||||
}
|
||||
|
||||
// List returns the installed plugins with their enabled/running state.
|
||||
func List() []Info {
|
||||
return global.list()
|
||||
}
|
||||
|
||||
// GetLogs returns the bounded log buffer of one plugin.
|
||||
func GetLogs(short string) string {
|
||||
return global.getLogs(short)
|
||||
}
|
||||
|
||||
// Delete removes an installed plugin and its persisted state.
|
||||
func Delete(short string) error {
|
||||
return global.delete(short)
|
||||
}
|
||||
|
||||
// WrapHandler installs the plugin HTTP hook chain around the application
|
||||
// handler. Request hooks run before the handler and may modify the request;
|
||||
// response hooks run after the handler and may modify status/headers/body.
|
||||
func WrapHandler(next http.Handler) http.Handler {
|
||||
return global.wrapHandler(next)
|
||||
}
|
||||
|
||||
// HTMLInjectHandler installs plugin HTML injection around the application
|
||||
// handler. Every text/html response is buffered and gets the fragments
|
||||
// registered through server.injectHTML inserted before </head> and </body>;
|
||||
// all other responses (JSON, assets, streams, downloads...) pass through to
|
||||
// the client unmodified.
|
||||
func HTMLInjectHandler(next http.Handler) http.Handler {
|
||||
return global.htmlInjectHandler(next)
|
||||
}
|
||||
|
||||
func (m *Manager) stateStore() *State {
|
||||
m.stateMu.Lock()
|
||||
defer m.stateMu.Unlock()
|
||||
if m.state == nil || m.state.path != filepath.Join(DataDir, stateFileName) {
|
||||
m.state = openState(filepath.Join(DataDir, stateFileName))
|
||||
}
|
||||
return m.state
|
||||
}
|
||||
|
||||
func (m *Manager) logStore(short string) *LogBuffer {
|
||||
m.logsMu.Lock()
|
||||
defer m.logsMu.Unlock()
|
||||
if m.logs == nil {
|
||||
m.logs = make(map[string]*LogBuffer)
|
||||
}
|
||||
buf := m.logs[short]
|
||||
if buf == nil {
|
||||
buf = newLogBuffer(defaultLogBufferSize)
|
||||
m.logs[short] = buf
|
||||
}
|
||||
return buf
|
||||
}
|
||||
|
||||
func (m *Manager) getLogs(short string) string {
|
||||
m.logsMu.Lock()
|
||||
defer m.logsMu.Unlock()
|
||||
if m.logs == nil {
|
||||
return ""
|
||||
}
|
||||
buf := m.logs[short]
|
||||
if buf == nil {
|
||||
return ""
|
||||
}
|
||||
return buf.String()
|
||||
}
|
||||
|
||||
func (m *Manager) instanceFor(short string) *Instance {
|
||||
m.mu.RLock()
|
||||
defer m.mu.RUnlock()
|
||||
return m.instances[short]
|
||||
}
|
||||
|
||||
// load runs a plugin inside its own jsruntime instance. The manager lock is
|
||||
// only held while the instance map is mutated; script evaluation and the
|
||||
// load() hook run without it so the plugin can call server.call (which takes
|
||||
// a manager read lock through rpcHandler) without deadlocking. On failure the
|
||||
// instance is dropped and its hooks and RPC methods are cleaned up.
|
||||
func (m *Manager) load(short string) error {
|
||||
dir := filepath.Join(DataDir, short)
|
||||
info, err := readManifest(dir)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := CheckKomariVersion(info.Komari); err != nil {
|
||||
return err
|
||||
}
|
||||
script, err := os.ReadFile(filepath.Join(dir, info.Entry))
|
||||
if err != nil {
|
||||
return fmt.Errorf("read plugin entry %s: %w", info.Entry, err)
|
||||
}
|
||||
|
||||
logs := m.logStore(short)
|
||||
logs.Reset()
|
||||
_, _ = logs.Write([]byte("[plugin] loading " + short + "\n"))
|
||||
|
||||
inst := &Instance{info: info, dir: dir, handlers: make(map[string]goja.Callable), statics: make(map[string]*staticConfig)}
|
||||
m.mu.Lock()
|
||||
if _, ok := m.instances[short]; ok {
|
||||
m.mu.Unlock()
|
||||
return fmt.Errorf("plugin %q is already loaded", short)
|
||||
}
|
||||
m.instances[short] = inst
|
||||
m.mu.Unlock()
|
||||
|
||||
storageDir := filepath.Join(StorageDir, short)
|
||||
if err := os.MkdirAll(storageDir, 0755); err != nil {
|
||||
m.dropInstance(short, inst)
|
||||
return fmt.Errorf("create plugin storage dir: %w", err)
|
||||
}
|
||||
|
||||
opts := jsruntime.Options{
|
||||
BaseDir: dir,
|
||||
StorageDir: storageDir,
|
||||
NodeJS: info.Permissions.Node,
|
||||
AllowExec: info.Permissions.AllowExec,
|
||||
AllowListen: info.Permissions.AllowListen,
|
||||
AllowAllFileAccess: info.Permissions.AllowAllFileAccess,
|
||||
MaxHTTPBodyBytes: info.Permissions.MaxHTTPBodyBytes,
|
||||
MaxChildOutputBytes: info.Permissions.MaxChildOutputBytes,
|
||||
Timeout: time.Duration(info.Permissions.TimeoutSeconds) * time.Second,
|
||||
Console: logs,
|
||||
ConfigureHost: func(host *jsruntime.Host, registry *require.Registry) {
|
||||
inst.mu.Lock()
|
||||
inst.host = host
|
||||
inst.mu.Unlock()
|
||||
m.registerServerModule(host, registry, inst)
|
||||
},
|
||||
}
|
||||
rt, err := jsruntime.New(string(script), opts)
|
||||
if err != nil {
|
||||
m.dropInstance(short, inst)
|
||||
return fmt.Errorf("initialize plugin %q: %w", short, err)
|
||||
}
|
||||
inst.mu.Lock()
|
||||
inst.runtime = rt
|
||||
inst.mu.Unlock()
|
||||
|
||||
if rt.HasFunction("load") {
|
||||
if err := rt.CallVoid("load"); err != nil {
|
||||
rt.Close()
|
||||
m.dropInstance(short, inst)
|
||||
return fmt.Errorf("plugin %q load() failed: %w", short, err)
|
||||
}
|
||||
}
|
||||
m.mu.Lock()
|
||||
stillLoaded := m.instances[short] == inst
|
||||
m.mu.Unlock()
|
||||
if !stillLoaded {
|
||||
// A concurrent unload claimed this instance while it was loading.
|
||||
rt.Close()
|
||||
return fmt.Errorf("plugin %q was unloaded while loading", short)
|
||||
}
|
||||
_, _ = logs.Write([]byte("[plugin] loaded " + short + "\n"))
|
||||
return nil
|
||||
}
|
||||
|
||||
// dropInstance removes a failed load from the manager: it detaches the
|
||||
// instance, unregisters its RPC methods, and clears its route slots and
|
||||
// hooks. Gin route slots from the failed load stay inert, mirroring the
|
||||
// unload behavior. The caller must not hold any Manager lock.
|
||||
func (m *Manager) dropInstance(short string, inst *Instance) {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
if m.instances[short] != inst {
|
||||
return // replaced by a newer load or already unloaded
|
||||
}
|
||||
delete(m.instances, short)
|
||||
inst.mu.Lock()
|
||||
for method := range inst.rpcMethods {
|
||||
rpc.Unregister(method)
|
||||
}
|
||||
removeCronJobs(inst.cronJobs)
|
||||
inst.runtime = nil
|
||||
inst.host = nil
|
||||
clear(inst.handlers)
|
||||
clear(inst.statics)
|
||||
clear(inst.rpcMethods)
|
||||
inst.cronJobs = nil
|
||||
inst.mu.Unlock()
|
||||
m.removeHooksLocked(short)
|
||||
m.removeInjectsLocked(short)
|
||||
}
|
||||
|
||||
// removeCronJobs cancels and forgets the scheduler jobs of one plugin load.
|
||||
// It runs with the instance lock held; scheduler.Remove only takes the
|
||||
// scheduler's own lock, so no lock cycle is possible.
|
||||
func removeCronJobs(names []string) {
|
||||
for _, name := range names {
|
||||
scheduler.Remove(name)
|
||||
}
|
||||
}
|
||||
|
||||
// unload stops one plugin. The instance is claimed (removed from the map)
|
||||
// under the manager lock, then the unload() hook runs without any lock so
|
||||
// the script can still call other registered RPC methods. After the hook the
|
||||
// remaining state is detached under the lock and the runtime is closed.
|
||||
// Calls to server.call made from unload() itself are rejected with a clean
|
||||
// "not loaded" error instead of deadlocking.
|
||||
func (m *Manager) unload(short string) error {
|
||||
m.mu.Lock()
|
||||
inst, ok := m.instances[short]
|
||||
if !ok {
|
||||
m.mu.Unlock()
|
||||
return fmt.Errorf("%w: %q", errNotLoaded, short)
|
||||
}
|
||||
delete(m.instances, short)
|
||||
m.mu.Unlock()
|
||||
|
||||
inst.mu.RLock()
|
||||
rt := inst.runtime
|
||||
inst.mu.RUnlock()
|
||||
var unloadErr error
|
||||
if rt != nil {
|
||||
if rt.HasFunction("unload") {
|
||||
if err := rt.CallVoid("unload"); err != nil {
|
||||
unloadErr = fmt.Errorf("plugin %q unload() failed: %w", short, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
m.mu.Lock()
|
||||
inst.mu.Lock()
|
||||
for method := range inst.rpcMethods {
|
||||
rpc.Unregister(method)
|
||||
}
|
||||
removeCronJobs(inst.cronJobs)
|
||||
inst.runtime = nil
|
||||
inst.host = nil
|
||||
clear(inst.handlers)
|
||||
clear(inst.statics)
|
||||
clear(inst.rpcMethods)
|
||||
inst.cronJobs = nil
|
||||
inst.mu.Unlock()
|
||||
m.removeHooksLocked(short)
|
||||
m.removeInjectsLocked(short)
|
||||
m.mu.Unlock()
|
||||
if rt != nil {
|
||||
rt.Close()
|
||||
}
|
||||
_, _ = m.logStore(short).Write([]byte("[plugin] unloaded " + short + "\n"))
|
||||
return unloadErr
|
||||
}
|
||||
|
||||
func (m *Manager) closeAll() error {
|
||||
m.mu.RLock()
|
||||
shorts := make([]string, 0, len(m.instances))
|
||||
for short := range m.instances {
|
||||
shorts = append(shorts, short)
|
||||
}
|
||||
m.mu.RUnlock()
|
||||
var errs []error
|
||||
for _, short := range shorts {
|
||||
if err := m.unload(short); err != nil {
|
||||
errs = append(errs, err)
|
||||
}
|
||||
}
|
||||
return errors.Join(errs...)
|
||||
}
|
||||
|
||||
func (m *Manager) setEnabled(short string, enabled, approved bool) error {
|
||||
dir := filepath.Join(DataDir, short)
|
||||
info, err := readManifest(dir)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !enabled {
|
||||
st := m.stateStore().get(short)
|
||||
st.Enabled = false
|
||||
if err := m.unload(short); err != nil && !errors.Is(err, errNotLoaded) {
|
||||
st.LastError = err.Error()
|
||||
}
|
||||
m.stateStore().set(short, st)
|
||||
return nil
|
||||
}
|
||||
if m.instanceFor(short) != nil {
|
||||
// Already running: enabling is idempotent. Re-running load() would
|
||||
// fail with "already loaded" and auto-disable the plugin.
|
||||
st := m.stateStore().get(short)
|
||||
st.Enabled = true
|
||||
st.LastError = ""
|
||||
m.stateStore().set(short, st)
|
||||
return nil
|
||||
}
|
||||
|
||||
hash := approvalPermissionsHash(info.Permissions)
|
||||
st := m.stateStore().get(short)
|
||||
if permissionsRequireApproval(info.Permissions) && st.ApprovedPermissionsHash != hash {
|
||||
if !approved {
|
||||
return ErrPermissionApprovalRequired
|
||||
}
|
||||
st.ApprovedPermissionsHash = hash
|
||||
m.stateStore().set(short, st)
|
||||
}
|
||||
if err := m.load(short); err != nil {
|
||||
st.Enabled = false
|
||||
st.LastError = err.Error()
|
||||
m.stateStore().set(short, st)
|
||||
return err
|
||||
}
|
||||
st.Enabled = true
|
||||
st.LastError = ""
|
||||
m.stateStore().set(short, st)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (m *Manager) loadAll() error {
|
||||
entries, err := os.ReadDir(DataDir)
|
||||
if err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
return nil
|
||||
}
|
||||
return err
|
||||
}
|
||||
var errs []error
|
||||
for _, entry := range entries {
|
||||
if !entry.IsDir() {
|
||||
continue
|
||||
}
|
||||
short := entry.Name()
|
||||
st := m.stateStore().get(short)
|
||||
if !st.Enabled || m.instanceFor(short) != nil {
|
||||
continue
|
||||
}
|
||||
info, err := readManifest(filepath.Join(DataDir, short))
|
||||
if err != nil {
|
||||
errs = append(errs, m.disableWithError(short, st, err))
|
||||
continue
|
||||
}
|
||||
if permissionsRequireApproval(info.Permissions) &&
|
||||
st.ApprovedPermissionsHash != approvalPermissionsHash(info.Permissions) {
|
||||
errs = append(errs, m.disableWithError(short, st, ErrPermissionApprovalRequired))
|
||||
continue
|
||||
}
|
||||
if err := m.load(short); err != nil {
|
||||
errs = append(errs, m.disableWithError(short, st, err))
|
||||
continue
|
||||
}
|
||||
}
|
||||
return errors.Join(errs...)
|
||||
}
|
||||
|
||||
// restartPlugin brings a plugin back to its persisted enabled state after a
|
||||
// reinstall. A failed reload or a changed approval hash auto-disables the
|
||||
// plugin and keeps the error visible, mirroring the startup load path.
|
||||
func (m *Manager) restartPlugin(short string) error {
|
||||
st := m.stateStore().get(short)
|
||||
info, err := readManifest(filepath.Join(DataDir, short))
|
||||
if err != nil {
|
||||
return m.disableWithError(short, st, err)
|
||||
}
|
||||
if permissionsRequireApproval(info.Permissions) &&
|
||||
st.ApprovedPermissionsHash != approvalPermissionsHash(info.Permissions) {
|
||||
return m.disableWithError(short, st, ErrPermissionApprovalRequired)
|
||||
}
|
||||
if err := m.load(short); err != nil {
|
||||
return m.disableWithError(short, st, err)
|
||||
}
|
||||
st.Enabled = true
|
||||
st.LastError = ""
|
||||
m.stateStore().set(short, st)
|
||||
return nil
|
||||
}
|
||||
|
||||
// disableWithError persists the auto-disable state for one plugin and wraps
|
||||
// the load failure for the startup error summary.
|
||||
func (m *Manager) disableWithError(short string, st PluginState, err error) error {
|
||||
st.Enabled = false
|
||||
st.LastError = err.Error()
|
||||
m.stateStore().set(short, st)
|
||||
return fmt.Errorf("plugin %q: %w", short, err)
|
||||
}
|
||||
|
||||
func (m *Manager) list() []Info {
|
||||
entries, err := os.ReadDir(DataDir)
|
||||
if err != nil {
|
||||
return []Info{}
|
||||
}
|
||||
infos := make([]Info, 0, len(entries))
|
||||
for _, entry := range entries {
|
||||
if !entry.IsDir() {
|
||||
continue
|
||||
}
|
||||
short := entry.Name()
|
||||
info, err := readManifest(filepath.Join(DataDir, short))
|
||||
if err != nil {
|
||||
continue // mirror the theme list: skip unreadable entries
|
||||
}
|
||||
st := m.stateStore().get(short)
|
||||
infos = append(infos, Info{
|
||||
Plugin: info,
|
||||
Enabled: st.Enabled,
|
||||
Running: m.instanceFor(short) != nil,
|
||||
LastError: st.LastError,
|
||||
})
|
||||
}
|
||||
sort.Slice(infos, func(i, j int) bool { return infos[i].Short < infos[j].Short })
|
||||
return infos
|
||||
}
|
||||
|
||||
// Delete removes an installed plugin: it unloads the runtime, deletes the
|
||||
// plugin directory and clears its persisted state. Registered gin route
|
||||
// slots stay inert (404) until the server restarts.
|
||||
func (m *Manager) delete(short string) error {
|
||||
if err := m.unload(short); err != nil && !errors.Is(err, errNotLoaded) {
|
||||
return err
|
||||
}
|
||||
if !validShort(short) {
|
||||
return fmt.Errorf("invalid plugin short %q", short)
|
||||
}
|
||||
dir := filepath.Join(DataDir, short)
|
||||
if _, err := os.Stat(dir); err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
return fmt.Errorf("%w: %q", ErrNotInstalled, short)
|
||||
}
|
||||
return err
|
||||
}
|
||||
if err := os.RemoveAll(dir); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := os.RemoveAll(filepath.Join(StorageDir, short)); err != nil {
|
||||
return err
|
||||
}
|
||||
m.stateStore().delete(short)
|
||||
return nil
|
||||
}
|
||||
|
||||
// Manifest returns the parsed manifest of an installed plugin.
|
||||
func Manifest(short string) (models.Plugin, error) {
|
||||
if !validShort(short) {
|
||||
return models.Plugin{}, fmt.Errorf("invalid plugin short %q", short)
|
||||
}
|
||||
return readManifest(filepath.Join(DataDir, short))
|
||||
}
|
||||
|
||||
// ResolvePublicFile returns the absolute path of a file that belongs to a
|
||||
// public iframe page of an installed plugin. Public pages are served without
|
||||
// authentication: the plugin must be enabled and the requested file must live
|
||||
// inside the directory of a page declared with visibility=public, so the page
|
||||
// HTML and its relative assets are reachable without exposing the whole
|
||||
// plugin directory.
|
||||
func ResolvePublicFile(short, name string) (string, error) {
|
||||
if !validShort(short) {
|
||||
return "", fmt.Errorf("invalid plugin short %q", short)
|
||||
}
|
||||
if !global.stateStore().get(short).Enabled {
|
||||
return "", fmt.Errorf("plugin %q is not enabled", short)
|
||||
}
|
||||
info, err := readManifest(filepath.Join(DataDir, short))
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
name = filepath.Clean(strings.TrimPrefix(name, "/"))
|
||||
for _, page := range info.Pages {
|
||||
if page.Visibility != models.PageVisibilityPublic || page.Type != models.PageTypeIframe {
|
||||
continue
|
||||
}
|
||||
dir := filepath.Dir(page.File)
|
||||
if dir == "." || name == dir || strings.HasPrefix(name, dir+string(os.PathSeparator)) {
|
||||
return ResolveFile(short, name)
|
||||
}
|
||||
}
|
||||
return "", fmt.Errorf("plugin page %q is not public", name)
|
||||
}
|
||||
|
||||
// ResolveFile returns the absolute path of a file inside an installed plugin
|
||||
// directory, rejecting traversal. Used to serve injected plugin pages.
|
||||
func ResolveFile(short, name string) (string, error) {
|
||||
if !validShort(short) {
|
||||
return "", fmt.Errorf("invalid plugin short %q", short)
|
||||
}
|
||||
if !filepath.IsLocal(name) {
|
||||
return "", fmt.Errorf("invalid plugin file path %q", name)
|
||||
}
|
||||
dir := filepath.Join(DataDir, short)
|
||||
full := filepath.Join(dir, name)
|
||||
if !withinDir(full, dir) {
|
||||
return "", fmt.Errorf("invalid plugin file path %q", name)
|
||||
}
|
||||
if _, err := os.Stat(full); err != nil {
|
||||
return "", err
|
||||
}
|
||||
return full, nil
|
||||
}
|
||||
|
||||
// hooksOf returns a snapshot of the hooks with the given kind.
|
||||
func (m *Manager) hooksOf(kind hookKind) []*hookEntry {
|
||||
m.mu.RLock()
|
||||
defer m.mu.RUnlock()
|
||||
snapshot := make([]*hookEntry, 0, len(m.hooks))
|
||||
for _, hook := range m.hooks {
|
||||
if hook.kind == kind {
|
||||
snapshot = append(snapshot, hook)
|
||||
}
|
||||
}
|
||||
return snapshot
|
||||
}
|
||||
|
||||
// registerHook appends a hook. Called from the plugin's own event loop
|
||||
// during script evaluation; it takes the manager lock itself. matcher is
|
||||
// nil for unfiltered hooks; the plugin-declared body limit is captured at
|
||||
// registration so unloaded plugins never contribute a stale limit.
|
||||
func (m *Manager) registerHook(short string, kind hookKind, fn goja.Callable, matcher *hookMatcher) {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
inst := m.instances[short]
|
||||
if inst == nil {
|
||||
return
|
||||
}
|
||||
inst.mu.RLock()
|
||||
host := inst.host
|
||||
bodyLimit := inst.info.Permissions.MaxHTTPBodyBytes
|
||||
inst.mu.RUnlock()
|
||||
m.hooks = append(m.hooks, &hookEntry{short: short, kind: kind, fn: fn, host: host, matcher: matcher, bodyLimit: bodyLimit})
|
||||
}
|
||||
|
||||
// removeHooksLocked drops all hooks of one plugin. The caller must hold the
|
||||
// manager write lock.
|
||||
func (m *Manager) removeHooksLocked(short string) {
|
||||
kept := m.hooks[:0]
|
||||
for _, hook := range m.hooks {
|
||||
if hook.short != short {
|
||||
kept = append(kept, hook)
|
||||
}
|
||||
}
|
||||
m.hooks = kept
|
||||
}
|
||||
|
||||
// injectsSnapshot returns the registered HTML injection fragments in
|
||||
// registration order.
|
||||
func (m *Manager) injectsSnapshot() []*injectEntry {
|
||||
m.mu.RLock()
|
||||
defer m.mu.RUnlock()
|
||||
return append([]*injectEntry(nil), m.injects...)
|
||||
}
|
||||
|
||||
// registerInject appends one plugin's HTML head/body fragments. Called from
|
||||
// the plugin's own event loop during script evaluation; it takes the manager
|
||||
// lock itself.
|
||||
func (m *Manager) registerInject(short, head, body string) {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
if m.instances[short] == nil {
|
||||
return
|
||||
}
|
||||
m.injects = append(m.injects, &injectEntry{short: short, head: head, body: body})
|
||||
}
|
||||
|
||||
// removeInjectsLocked drops all injection fragments of one plugin. The
|
||||
// caller must hold the manager write lock.
|
||||
func (m *Manager) removeInjectsLocked(short string) {
|
||||
kept := m.injects[:0]
|
||||
for _, inject := range m.injects {
|
||||
if inject.short != short {
|
||||
kept = append(kept, inject)
|
||||
}
|
||||
}
|
||||
m.injects = kept
|
||||
}
|
||||
|
||||
// validShort mirrors the theme short-name rule.
|
||||
func validShort(short string) bool {
|
||||
if short == "" || short == "default" {
|
||||
return false
|
||||
}
|
||||
for _, r := range short {
|
||||
if !((r >= 'a' && r <= 'z') || (r >= 'A' && r <= 'Z') ||
|
||||
(r >= '0' && r <= '9') || r == '_' || r == '-') {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// permissionsRequireApproval reports whether the manifest declares at least
|
||||
// one approval-relevant capability. Plugins without dangerous capabilities
|
||||
// (for example only declaring node modules, limits or the timeout) enable
|
||||
// without an approval step.
|
||||
func permissionsRequireApproval(p models.PluginPermissions) bool {
|
||||
return p.AllowSystemRPC || p.AllowRoutes || p.AllowHooks || p.AllowHTMLInject ||
|
||||
p.AllowExec || p.AllowListen || p.AllowAllFileAccess
|
||||
}
|
||||
|
||||
// approvalPermissionsHash canonicalizes the approval-relevant permissions so
|
||||
// approval can be compared across plugin upgrades. Runtime settings such as
|
||||
// node modules, resource limits and the execution timeout are granted by
|
||||
// default and do not participate in the approval hash.
|
||||
func approvalPermissionsHash(p models.PluginPermissions) string {
|
||||
data, _ := json.Marshal(models.PluginPermissions{
|
||||
AllowSystemRPC: p.AllowSystemRPC,
|
||||
AllowRoutes: p.AllowRoutes,
|
||||
AllowHooks: p.AllowHooks,
|
||||
AllowHTMLInject: p.AllowHTMLInject,
|
||||
AllowExec: p.AllowExec,
|
||||
AllowListen: p.AllowListen,
|
||||
AllowAllFileAccess: p.AllowAllFileAccess,
|
||||
})
|
||||
sum := sha256.Sum256(data)
|
||||
return "sha256:" + hex.EncodeToString(sum[:])
|
||||
}
|
||||
@@ -0,0 +1,140 @@
|
||||
package plugin
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/dop251/goja"
|
||||
"github.com/komari-monitor/komari/pkg/rpc"
|
||||
)
|
||||
|
||||
// registerRPC registers a plugin-owned RPC method. The handler is a
|
||||
// JavaScript function receiving the JSON-RPC params and returning the result
|
||||
// (or throwing an Error with optional code/data). Called from the plugin's
|
||||
// own event loop during script evaluation; it takes the manager lock itself.
|
||||
func (m *Manager) registerRPC(short, method string, handler goja.Callable) error {
|
||||
method = strings.TrimSpace(method)
|
||||
if method == "" || strings.HasPrefix(method, "rpc.") {
|
||||
return fmt.Errorf("invalid RPC method name %q", method)
|
||||
}
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
inst, ok := m.instances[short]
|
||||
if !ok {
|
||||
return fmt.Errorf("plugin %q is not loaded", short)
|
||||
}
|
||||
inst.mu.Lock()
|
||||
if inst.rpcMethods == nil {
|
||||
inst.rpcMethods = make(map[string]goja.Callable)
|
||||
}
|
||||
if _, exists := inst.rpcMethods[method]; exists {
|
||||
inst.mu.Unlock()
|
||||
return nil // already registered in this load
|
||||
}
|
||||
inst.rpcMethods[method] = handler
|
||||
inst.mu.Unlock()
|
||||
if err := rpc.Register(method, m.rpcHandler(short, method)); err != nil {
|
||||
inst.mu.Lock()
|
||||
delete(inst.rpcMethods, method)
|
||||
inst.mu.Unlock()
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// rpcHandler bridges one registered RPC call into the plugin event loop.
|
||||
func (m *Manager) rpcHandler(short, method string) rpc.Handler {
|
||||
return func(ctx context.Context, req *rpc.JsonRpcRequest) (any, *rpc.JsonRpcError) {
|
||||
inst := m.instanceFor(short)
|
||||
if inst == nil {
|
||||
return nil, &rpc.JsonRpcError{Code: rpc.Unavailable, Message: fmt.Sprintf("plugin %q is not loaded", short)}
|
||||
}
|
||||
inst.mu.RLock()
|
||||
handler := inst.rpcMethods[method]
|
||||
host := inst.host
|
||||
alive := inst.runtime != nil
|
||||
inst.mu.RUnlock()
|
||||
if !alive || handler == nil || host == nil {
|
||||
return nil, &rpc.JsonRpcError{Code: rpc.MethodNotFound, Message: "method not found", Data: method}
|
||||
}
|
||||
|
||||
resultCh := make(chan any, 1)
|
||||
errCh := make(chan *rpc.JsonRpcError, 1)
|
||||
queued := host.RunOnLoop(func(vm *goja.Runtime) {
|
||||
_ = host.RunJob(vm, "plugin rpc "+method, func() error {
|
||||
params := vm.ToValue(req.Params)
|
||||
if params == nil {
|
||||
params = goja.Null()
|
||||
}
|
||||
value, callErr := handler(goja.Undefined(), params)
|
||||
if callErr != nil {
|
||||
errCh <- jsErrorToRPCError(vm, callErr)
|
||||
return nil // error is captured, do not double-report
|
||||
}
|
||||
exported, exportErr := exportJSValue(value)
|
||||
if exportErr != nil {
|
||||
errCh <- &rpc.JsonRpcError{Code: rpc.InternalError, Message: exportErr.Error()}
|
||||
return nil
|
||||
}
|
||||
resultCh <- exported
|
||||
return nil
|
||||
})
|
||||
})
|
||||
if !queued {
|
||||
return nil, &rpc.JsonRpcError{Code: rpc.Unavailable, Message: "plugin runtime is closed"}
|
||||
}
|
||||
timer := time.NewTimer(host.Timeout())
|
||||
defer timer.Stop()
|
||||
select {
|
||||
case result := <-resultCh:
|
||||
return result, nil
|
||||
case rpcErr := <-errCh:
|
||||
return nil, rpcErr
|
||||
case <-timer.C:
|
||||
return nil, &rpc.JsonRpcError{Code: rpc.DeadlineExceeded, Message: "plugin RPC handler timed out"}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// exportJSValue exports a goja value into a JSON-RPC result. Circular
|
||||
// structures and other export failures become an error instead of panicking
|
||||
// inside the event loop.
|
||||
func exportJSValue(value goja.Value) (result any, err error) {
|
||||
defer func() {
|
||||
if recovered := recover(); recovered != nil {
|
||||
result = nil
|
||||
err = fmt.Errorf("export JavaScript value: %v", recovered)
|
||||
}
|
||||
}()
|
||||
return value.Export(), nil
|
||||
}
|
||||
|
||||
// jsErrorToRPCError converts a thrown JavaScript error into a JSON-RPC error,
|
||||
// honoring optional code/data properties.
|
||||
func jsErrorToRPCError(vm *goja.Runtime, err error) *rpc.JsonRpcError {
|
||||
code := rpc.InternalError
|
||||
message := err.Error()
|
||||
var data any
|
||||
var exception *goja.Exception
|
||||
if errors.As(err, &exception) {
|
||||
if obj, ok := exception.Value().(*goja.Object); ok {
|
||||
if value := obj.Get("code"); value != nil && !goja.IsUndefined(value) {
|
||||
if n := value.ToInteger(); n != 0 {
|
||||
code = int(n)
|
||||
}
|
||||
}
|
||||
if value := obj.Get("message"); value != nil && !goja.IsUndefined(value) && value.String() != "" {
|
||||
message = value.String()
|
||||
}
|
||||
if value := obj.Get("data"); value != nil && !goja.IsUndefined(value) {
|
||||
if exported, exportErr := exportJSValue(value); exportErr == nil {
|
||||
data = exported
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return &rpc.JsonRpcError{Code: code, Message: message, Data: data}
|
||||
}
|
||||
@@ -0,0 +1,681 @@
|
||||
package plugin
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"github.com/dop251/goja"
|
||||
"github.com/dop251/goja_nodejs/buffer"
|
||||
"github.com/dop251/goja_nodejs/require"
|
||||
"github.com/gin-gonic/gin"
|
||||
"github.com/komari-monitor/komari/pkg/jsruntime"
|
||||
"github.com/komari-monitor/komari/pkg/jsruntime/httpbody"
|
||||
"github.com/komari-monitor/komari/pkg/rpc"
|
||||
)
|
||||
|
||||
// registerServerModule registers the "server" native module for one plugin
|
||||
// instance. The module exposes:
|
||||
//
|
||||
// server.route(method, path, handler) register an HTTP route on the host
|
||||
// engine; handler receives (req, res)
|
||||
// and must call res.end() to finish
|
||||
// server.static(path, dir, opts) serve a folder from the plugin
|
||||
// directory at a mount path; opts.spa
|
||||
// makes unmatched paths fall back to
|
||||
// index.html
|
||||
// server.hook(kind, fn) register a request or response hook;
|
||||
// an optional "METHOD /path", "/path"
|
||||
// or "/path/*" filter limits the hook
|
||||
// to matching requests. The ws kinds
|
||||
// (wsConnect / wsMessage / wsSend /
|
||||
// wsClose) intercept WebSocket
|
||||
// connections and frames on matching
|
||||
// endpoints; they share the
|
||||
// allowHooks permission and accept
|
||||
// only path matchers
|
||||
// server.injectHTML(head, body) register HTML fragments embedded
|
||||
// into every text/html response:
|
||||
// head before </head>, body before
|
||||
// </body> (all pages, including the
|
||||
// admin and terminal pages)
|
||||
// server.call(method, params...) call a registered RPC method with
|
||||
// admin authority; resolves to the RPC
|
||||
// result or rejects with an Error
|
||||
// carrying code/message/data
|
||||
// server.registerRPC(method, handler) register a plugin-owned RPC method
|
||||
// server.getConfig() resolve the saved plugin configuration
|
||||
// server.cron(expr, fn) run fn on the plugin event loop each
|
||||
// time the cron expression fires
|
||||
//
|
||||
// server.registerRPC, server.getConfig, server.cron and filesystem access
|
||||
// confined to the plugin directory are always granted without a manifest
|
||||
// declaration.
|
||||
// server.route, server.hook, server.injectHTML and server.call require the
|
||||
// allowRoutes, allowHooks, allowHTMLInject and allowSystemRPC permissions
|
||||
// respectively; a missing permission fails at load time (route/hook/inject)
|
||||
// or rejects the call.
|
||||
//
|
||||
// The host engine keeps a registered route slot after unload; requests then
|
||||
// receive 404 until the plugin is loaded again.
|
||||
func (m *Manager) registerServerModule(host *jsruntime.Host, registry *require.Registry, inst *Instance) {
|
||||
registry.RegisterNativeModule("server", func(vm *goja.Runtime, module *goja.Object) {
|
||||
exports := vm.NewObject()
|
||||
_ = exports.Set("route", func(call goja.FunctionCall) goja.Value {
|
||||
method := strings.ToUpper(strings.TrimSpace(call.Argument(0).String()))
|
||||
path := call.Argument(1).String()
|
||||
handler, ok := goja.AssertFunction(call.Argument(2))
|
||||
if method == "" {
|
||||
panic(vm.NewTypeError("server.route requires an HTTP method"))
|
||||
}
|
||||
if path == "" || !strings.HasPrefix(path, "/") {
|
||||
panic(vm.NewTypeError("server.route requires a path starting with /"))
|
||||
}
|
||||
if !ok {
|
||||
panic(vm.NewTypeError("server.route requires a function handler"))
|
||||
}
|
||||
if !inst.info.Permissions.AllowRoutes {
|
||||
panic(vm.NewTypeError("server.route requires the \"route\" permission (allowRoutes)"))
|
||||
}
|
||||
if err := m.registerRoute(inst.info.Short, method, path, handler); err != nil {
|
||||
panic(vm.NewGoError(err))
|
||||
}
|
||||
return goja.Undefined()
|
||||
})
|
||||
_ = exports.Set("static", func(call goja.FunctionCall) goja.Value {
|
||||
mount := strings.TrimSpace(call.Argument(0).String())
|
||||
mount = strings.TrimRight(mount, "/")
|
||||
dir := strings.TrimSpace(call.Argument(1).String())
|
||||
if mount == "" || mount == "/" || !strings.HasPrefix(mount, "/") {
|
||||
panic(vm.NewTypeError("server.static requires a mount path starting with / (and not \"/\")"))
|
||||
}
|
||||
if dir == "" {
|
||||
panic(vm.NewTypeError("server.static requires a folder name"))
|
||||
}
|
||||
if !filepath.IsLocal(dir) {
|
||||
panic(vm.NewTypeError("server.static folder must be a relative path inside the plugin directory"))
|
||||
}
|
||||
if !inst.info.Permissions.AllowRoutes {
|
||||
panic(vm.NewTypeError("server.static requires the \"route\" permission (allowRoutes)"))
|
||||
}
|
||||
var opts map[string]any
|
||||
if !goja.IsUndefined(call.Argument(2)) && !goja.IsNull(call.Argument(2)) {
|
||||
if err := vm.ExportTo(call.Argument(2), &opts); err != nil {
|
||||
panic(vm.NewTypeError("server.static options must be an object like { spa: true }"))
|
||||
}
|
||||
}
|
||||
spa := false
|
||||
if v, ok := opts["spa"]; ok {
|
||||
spa, _ = v.(bool)
|
||||
}
|
||||
if err := m.registerStatic(inst.info.Short, mount, dir, spa); err != nil {
|
||||
panic(vm.NewGoError(err))
|
||||
}
|
||||
return goja.Undefined()
|
||||
})
|
||||
_ = exports.Set("call", func(call goja.FunctionCall) goja.Value {
|
||||
method := strings.TrimSpace(call.Argument(0).String())
|
||||
if method == "" {
|
||||
panic(vm.NewTypeError("server.call requires an RPC method name"))
|
||||
}
|
||||
params := callParams(call)
|
||||
promise, resolve, reject := vm.NewPromise()
|
||||
go func() {
|
||||
if !inst.info.Permissions.AllowSystemRPC {
|
||||
host.RunOnLoop(func(vm *goja.Runtime) {
|
||||
_ = reject(vm.NewGoError(errors.New("server.call requires the \"system RPC\" permission (allowSystemRPC)")))
|
||||
})
|
||||
return
|
||||
}
|
||||
meta := &rpc.ContextMeta{Permission: rpc.RoleAdmin, Principal: rpc.PrincipalFromRole(rpc.RoleAdmin)}
|
||||
ctx := rpc.NewContextWithMeta(context.Background(), meta)
|
||||
resp := rpc.CallWithContext(ctx, nil, method, params)
|
||||
host.RunOnLoop(func(vm *goja.Runtime) {
|
||||
if resp.Error != nil {
|
||||
_ = reject(jsRPCError(vm, resp.Error))
|
||||
return
|
||||
}
|
||||
_ = resolve(resp.Result)
|
||||
})
|
||||
}()
|
||||
return vm.ToValue(promise)
|
||||
})
|
||||
_ = exports.Set("hook", func(call goja.FunctionCall) goja.Value {
|
||||
kind := strings.ToLower(strings.TrimSpace(call.Argument(0).String()))
|
||||
switch kind {
|
||||
case "request", "response", string(hookWSConnect), string(hookWSMessage), string(hookWSSend), string(hookWSClose):
|
||||
if !inst.info.Permissions.AllowHooks {
|
||||
panic(vm.NewTypeError("server.hook requires the \"hook\" permission (allowHooks)"))
|
||||
}
|
||||
default:
|
||||
panic(vm.NewTypeError("server.hook kind must be \"request\", \"response\", \"wsConnect\", \"wsMessage\", \"wsSend\" or \"wsClose\""))
|
||||
}
|
||||
// server.hook(kind, fn) or server.hook(kind, matcher, fn) where
|
||||
// matcher is "METHOD /path", "/path" or "/path/*". The ws kinds
|
||||
// only accept a path matcher: every upgrade is a GET request.
|
||||
fnValue := call.Argument(1)
|
||||
var matcher *hookMatcher
|
||||
if goja.IsString(fnValue) {
|
||||
parsed, err := parseHookMatcher(fnValue.String())
|
||||
if err != nil {
|
||||
panic(vm.NewTypeError(err.Error()))
|
||||
}
|
||||
if isWSHookKind(kind) && parsed.method != "" {
|
||||
panic(vm.NewTypeError("server.hook \"" + kind + "\" only accepts a path matcher like \"/api/*\""))
|
||||
}
|
||||
matcher = parsed
|
||||
fnValue = call.Argument(2)
|
||||
}
|
||||
fn, ok := goja.AssertFunction(fnValue)
|
||||
if !ok {
|
||||
panic(vm.NewTypeError("server.hook requires a function handler"))
|
||||
}
|
||||
m.registerHook(inst.info.Short, hookKind(kind), fn, matcher)
|
||||
return goja.Undefined()
|
||||
})
|
||||
_ = exports.Set("injectHTML", func(call goja.FunctionCall) goja.Value {
|
||||
if !inst.info.Permissions.AllowHTMLInject {
|
||||
panic(vm.NewTypeError("server.injectHTML requires the \"HTML inject\" permission (allowHTMLInject)"))
|
||||
}
|
||||
m.registerInject(inst.info.Short, call.Argument(0).String(), call.Argument(1).String())
|
||||
return goja.Undefined()
|
||||
})
|
||||
_ = exports.Set("cron", func(call goja.FunctionCall) goja.Value {
|
||||
spec := strings.TrimSpace(call.Argument(0).String())
|
||||
fn, ok := goja.AssertFunction(call.Argument(1))
|
||||
if spec == "" {
|
||||
panic(vm.NewTypeError("server.cron requires a cron expression"))
|
||||
}
|
||||
if !ok {
|
||||
panic(vm.NewTypeError("server.cron requires a function handler"))
|
||||
}
|
||||
if err := m.registerCron(inst.info.Short, spec, fn); err != nil {
|
||||
panic(vm.NewGoError(err))
|
||||
}
|
||||
return goja.Undefined()
|
||||
})
|
||||
_ = exports.Set("registerRPC", func(call goja.FunctionCall) goja.Value {
|
||||
method := strings.TrimSpace(call.Argument(0).String())
|
||||
fn, ok := goja.AssertFunction(call.Argument(1))
|
||||
if method == "" {
|
||||
panic(vm.NewTypeError("server.registerRPC requires an RPC method name"))
|
||||
}
|
||||
if !ok {
|
||||
panic(vm.NewTypeError("server.registerRPC requires a function handler"))
|
||||
}
|
||||
if err := m.registerRPC(inst.info.Short, method, fn); err != nil {
|
||||
panic(vm.NewGoError(err))
|
||||
}
|
||||
return goja.Undefined()
|
||||
})
|
||||
_ = exports.Set("getConfig", func(call goja.FunctionCall) goja.Value {
|
||||
promise, resolve, reject := vm.NewPromise()
|
||||
go func() {
|
||||
data, err := GetConfiguration(inst.info.Short)
|
||||
host.RunOnLoop(func(vm *goja.Runtime) {
|
||||
if err != nil {
|
||||
_ = reject(vm.NewGoError(err))
|
||||
return
|
||||
}
|
||||
_ = resolve(data)
|
||||
})
|
||||
}()
|
||||
return vm.ToValue(promise)
|
||||
})
|
||||
_ = module.Set("exports", exports)
|
||||
})
|
||||
}
|
||||
|
||||
// callParams converts JS arguments into JSON-RPC params: no params, a single
|
||||
// value, or a positional array. Values that cannot be exported (for example
|
||||
// circular objects) are skipped so the call rejects instead of panicking.
|
||||
func callParams(call goja.FunctionCall) any {
|
||||
switch n := len(call.Arguments); {
|
||||
case n <= 1:
|
||||
return nil
|
||||
case n == 2:
|
||||
exported, _ := exportJSValue(call.Argument(1))
|
||||
return exported
|
||||
default:
|
||||
params := make([]any, 0, n-1)
|
||||
for i := 1; i < n; i++ {
|
||||
exported, err := exportJSValue(call.Argument(i))
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
params = append(params, exported)
|
||||
}
|
||||
return params
|
||||
}
|
||||
}
|
||||
|
||||
func jsRPCError(vm *goja.Runtime, e *rpc.JsonRpcError) *goja.Object {
|
||||
obj := vm.NewGoError(fmt.Errorf("%s", e.Message))
|
||||
_ = obj.Set("code", e.Code)
|
||||
if e.Data != nil {
|
||||
_ = obj.Set("data", e.Data)
|
||||
}
|
||||
return obj
|
||||
}
|
||||
|
||||
// registerRoute registers (or reuses) a gin route slot for one plugin. It is
|
||||
// called from the plugin's own event loop during script evaluation and takes
|
||||
// the manager lock itself. The critical section is short and never waits on
|
||||
// the event loop, so no lock cycle is possible.
|
||||
func (m *Manager) registerRoute(short, method, path string, handler goja.Callable) (err error) {
|
||||
key := method + " " + path
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
inst, ok := m.instances[short]
|
||||
if !ok {
|
||||
return fmt.Errorf("plugin %q is not loaded", short)
|
||||
}
|
||||
if m.engine == nil {
|
||||
return fmt.Errorf("plugin manager has no HTTP engine")
|
||||
}
|
||||
if m.routes[short] == nil {
|
||||
m.routes[short] = make(map[string]bool)
|
||||
}
|
||||
slotExists := m.routes[short][key]
|
||||
inst.mu.Lock()
|
||||
if !slotExists && inst.handlers[key] != nil {
|
||||
inst.mu.Unlock()
|
||||
return nil // already registered in this load
|
||||
}
|
||||
inst.handlers[key] = handler
|
||||
inst.mu.Unlock()
|
||||
if slotExists {
|
||||
return nil // gin slot from an earlier load stays; the handler slot is refreshed
|
||||
}
|
||||
defer func() {
|
||||
if r := recover(); r != nil {
|
||||
inst.mu.Lock()
|
||||
delete(inst.handlers, key)
|
||||
inst.mu.Unlock()
|
||||
err = fmt.Errorf("register route %s %s: %v", method, path, r)
|
||||
}
|
||||
}()
|
||||
m.engine.Handle(method, path, m.routeHandler(short))
|
||||
m.routes[short][key] = true
|
||||
return nil
|
||||
}
|
||||
|
||||
// routeHandler bridges one gin request into the plugin event loop. The
|
||||
// handler receives (req, res); the response is written once res.end() runs
|
||||
// or the runtime timeout elapses.
|
||||
func (m *Manager) routeHandler(short string) gin.HandlerFunc {
|
||||
return func(c *gin.Context) {
|
||||
inst := m.instanceFor(short)
|
||||
if inst == nil {
|
||||
c.Status(http.StatusNotFound)
|
||||
return
|
||||
}
|
||||
key := c.Request.Method + " " + c.FullPath()
|
||||
inst.mu.RLock()
|
||||
handler := inst.handlers[key]
|
||||
host := inst.host
|
||||
alive := inst.runtime != nil
|
||||
limit := inst.info.Permissions.MaxHTTPBodyBytes
|
||||
inst.mu.RUnlock()
|
||||
if !alive || handler == nil || host == nil {
|
||||
c.Status(http.StatusNotFound)
|
||||
return
|
||||
}
|
||||
if limit < 1 {
|
||||
limit = defaultMaxHTTPBodyBytes
|
||||
}
|
||||
body, err := httpbody.ReadAll(c.Request.Body, limit)
|
||||
if err != nil {
|
||||
c.Status(http.StatusRequestEntityTooLarge)
|
||||
_, _ = c.Writer.WriteString(err.Error())
|
||||
return
|
||||
}
|
||||
|
||||
state := &routeResponse{
|
||||
headers: make(http.Header),
|
||||
statusCode: http.StatusOK,
|
||||
done: make(chan struct{}),
|
||||
chunks: make(chan []byte, 4),
|
||||
abortCh: make(chan struct{}),
|
||||
}
|
||||
queued := host.RunOnLoop(func(vm *goja.Runtime) {
|
||||
req := routeRequest(vm, c, body)
|
||||
res := routeResponseObject(vm, state)
|
||||
runErr := host.RunJob(vm, "plugin route "+short, func() error {
|
||||
_, callErr := handler(goja.Undefined(), req, res)
|
||||
return callErr
|
||||
})
|
||||
if runErr != nil {
|
||||
state.finish(http.StatusInternalServerError, nil)
|
||||
}
|
||||
})
|
||||
if !queued {
|
||||
c.Status(http.StatusServiceUnavailable)
|
||||
return
|
||||
}
|
||||
|
||||
writeFinal := func(status int, body []byte) {
|
||||
state.mu.Lock()
|
||||
if status != 0 {
|
||||
state.statusCode = status
|
||||
}
|
||||
if body != nil {
|
||||
_, _ = state.body.Write(body)
|
||||
}
|
||||
for name, values := range state.headers {
|
||||
for _, value := range values {
|
||||
c.Header(name, value)
|
||||
}
|
||||
}
|
||||
statusCode := state.statusCode
|
||||
responseBody := append([]byte(nil), state.body.Bytes()...)
|
||||
state.mu.Unlock()
|
||||
c.Status(statusCode)
|
||||
_, _ = c.Writer.Write(responseBody)
|
||||
}
|
||||
|
||||
// Streaming pump: with res.streaming = true every res.write is sent back
|
||||
// to the client and flushed immediately; normal mode buffers and writes
|
||||
// once at res.end(). When the client disconnects, aborted is set so the
|
||||
// plugin script can stop producing via res.isAborted().
|
||||
idle := time.NewTimer(host.Timeout())
|
||||
defer idle.Stop()
|
||||
committed := false
|
||||
for {
|
||||
select {
|
||||
case chunk := <-state.chunks:
|
||||
if !committed {
|
||||
state.mu.Lock()
|
||||
for name, values := range state.headers {
|
||||
for _, value := range values {
|
||||
c.Header(name, value)
|
||||
}
|
||||
}
|
||||
statusCode := state.statusCode
|
||||
state.mu.Unlock()
|
||||
c.Status(statusCode)
|
||||
committed = true
|
||||
}
|
||||
_, _ = c.Writer.Write(chunk)
|
||||
if flusher, ok := c.Writer.(http.Flusher); ok {
|
||||
flusher.Flush()
|
||||
}
|
||||
if !idle.Stop() {
|
||||
select {
|
||||
case <-idle.C:
|
||||
default:
|
||||
}
|
||||
}
|
||||
idle.Reset(host.Timeout())
|
||||
case <-state.done:
|
||||
if !committed {
|
||||
writeFinal(0, nil)
|
||||
}
|
||||
return
|
||||
case <-c.Request.Context().Done():
|
||||
state.aborted.Store(true)
|
||||
close(state.abortCh)
|
||||
return
|
||||
case <-idle.C:
|
||||
if state.isStreaming() {
|
||||
// Streaming idle timeout: end the stream and notify JS
|
||||
state.aborted.Store(true)
|
||||
close(state.abortCh)
|
||||
return
|
||||
}
|
||||
writeFinal(http.StatusGatewayTimeout, []byte("plugin route handler timed out"))
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// staticConfig is the current static folder config of one mount path. It is
|
||||
// stored on the Instance so the gin slot (which survives unload) always reads
|
||||
// the config of the latest load.
|
||||
type staticConfig struct {
|
||||
dir string // absolute folder path inside the plugin directory
|
||||
spa bool // fall back to index.html for unmatched paths
|
||||
}
|
||||
|
||||
// registerStatic serves a folder from the plugin directory at a mount path.
|
||||
// It registers GET and HEAD slots for both the mount path and its wildcard
|
||||
// so /ui and /ui/anything both reach the folder. Like registerRoute, the gin
|
||||
// slots survive unload (they return 404) and are reused on reload; the
|
||||
// handler always resolves the current config from the instance.
|
||||
func (m *Manager) registerStatic(short, mount, dir string, spa bool) (err error) {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
inst, ok := m.instances[short]
|
||||
if !ok {
|
||||
return fmt.Errorf("plugin %q is not loaded", short)
|
||||
}
|
||||
if m.engine == nil {
|
||||
return fmt.Errorf("plugin manager has no HTTP engine")
|
||||
}
|
||||
absDir := filepath.Join(inst.dir, dir)
|
||||
if !withinDir(absDir, inst.dir) {
|
||||
return fmt.Errorf("static folder %q must be inside the plugin directory", dir)
|
||||
}
|
||||
fi, statErr := os.Stat(absDir)
|
||||
if statErr != nil || !fi.IsDir() {
|
||||
return fmt.Errorf("static folder %q does not exist", dir)
|
||||
}
|
||||
if m.routes[short] == nil {
|
||||
m.routes[short] = make(map[string]bool)
|
||||
}
|
||||
inst.mu.Lock()
|
||||
inst.statics[mount] = &staticConfig{dir: absDir, spa: spa}
|
||||
inst.mu.Unlock()
|
||||
|
||||
key := "STATIC " + mount
|
||||
if m.routes[short][key] {
|
||||
return nil // gin slots from an earlier load stay; the config was refreshed above
|
||||
}
|
||||
defer func() {
|
||||
if r := recover(); r != nil {
|
||||
inst.mu.Lock()
|
||||
delete(inst.statics, mount)
|
||||
inst.mu.Unlock()
|
||||
err = fmt.Errorf("register static %s: %v", mount, r)
|
||||
}
|
||||
}()
|
||||
handler := m.staticHandler(short, mount)
|
||||
for _, method := range []string{http.MethodGet, http.MethodHead} {
|
||||
m.engine.Handle(method, mount, handler)
|
||||
m.engine.Handle(method, mount+"/*filepath", handler)
|
||||
}
|
||||
m.routes[short][key] = true
|
||||
return nil
|
||||
}
|
||||
|
||||
// staticHandler serves files from a static folder. Requests to the mount
|
||||
// path resolve to index.html; requests into subpaths resolve to the matching
|
||||
// file. With spa enabled, any unmatched path (including directories without
|
||||
// an index.html) falls back to index.html so client-side routers work.
|
||||
func (m *Manager) staticHandler(short, mount string) gin.HandlerFunc {
|
||||
return func(c *gin.Context) {
|
||||
inst := m.instanceFor(short)
|
||||
if inst == nil {
|
||||
c.Status(http.StatusNotFound)
|
||||
return
|
||||
}
|
||||
inst.mu.RLock()
|
||||
cfg := inst.statics[mount]
|
||||
inst.mu.RUnlock()
|
||||
if cfg == nil {
|
||||
c.Status(http.StatusNotFound)
|
||||
return
|
||||
}
|
||||
name := strings.TrimPrefix(c.Request.URL.Path, mount)
|
||||
name = strings.TrimPrefix(name, "/")
|
||||
target, err := resolveStaticFile(cfg.dir, name)
|
||||
if err != nil {
|
||||
if !cfg.spa {
|
||||
c.Status(http.StatusNotFound)
|
||||
return
|
||||
}
|
||||
target = filepath.Join(cfg.dir, "index.html")
|
||||
}
|
||||
c.File(target)
|
||||
}
|
||||
}
|
||||
|
||||
// resolveStaticFile maps a request path to a file inside a static folder,
|
||||
// rejecting traversal. A directory resolves to its index.html.
|
||||
func resolveStaticFile(dir, name string) (string, error) {
|
||||
if name == "" {
|
||||
name = "index.html"
|
||||
}
|
||||
if !filepath.IsLocal(name) {
|
||||
return "", fmt.Errorf("invalid static path %q", name)
|
||||
}
|
||||
full := filepath.Join(dir, name)
|
||||
if !withinDir(full, dir) {
|
||||
return "", fmt.Errorf("invalid static path %q", name)
|
||||
}
|
||||
fi, err := os.Stat(full)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if fi.IsDir() {
|
||||
full = filepath.Join(full, "index.html")
|
||||
if _, err := os.Stat(full); err != nil {
|
||||
return "", err
|
||||
}
|
||||
}
|
||||
return full, nil
|
||||
}
|
||||
|
||||
// routeResponse is the Go side of the JS response object. It mirrors the
|
||||
// minimal nodeHTTPResponse pattern from the jsruntime http module. When
|
||||
// res.streaming is enabled, chunks are pushed through a channel and written
|
||||
// back to the client immediately instead of being buffered.
|
||||
type routeResponse struct {
|
||||
mu sync.Mutex
|
||||
headers http.Header
|
||||
body bytes.Buffer
|
||||
statusCode int
|
||||
ended bool
|
||||
done chan struct{}
|
||||
|
||||
streaming bool
|
||||
chunks chan []byte
|
||||
aborted atomic.Bool
|
||||
abortCh chan struct{}
|
||||
}
|
||||
|
||||
func (state *routeResponse) markStreaming() {
|
||||
state.mu.Lock()
|
||||
state.streaming = true
|
||||
state.mu.Unlock()
|
||||
}
|
||||
|
||||
func (state *routeResponse) isStreaming() bool {
|
||||
state.mu.Lock()
|
||||
defer state.mu.Unlock()
|
||||
return state.streaming
|
||||
}
|
||||
|
||||
// pushChunk delivers a streamed chunk to the HTTP pump. It returns false
|
||||
// when the client connection is gone so the script can stop producing.
|
||||
func (state *routeResponse) pushChunk(p []byte) bool {
|
||||
select {
|
||||
case state.chunks <- p:
|
||||
return true
|
||||
case <-state.abortCh:
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
func (state *routeResponse) finish(status int, body []byte) {
|
||||
state.mu.Lock()
|
||||
defer state.mu.Unlock()
|
||||
if state.ended {
|
||||
return
|
||||
}
|
||||
if status != 0 {
|
||||
state.statusCode = status
|
||||
}
|
||||
if body != nil {
|
||||
_, _ = state.body.Write(body)
|
||||
}
|
||||
state.ended = true
|
||||
close(state.done)
|
||||
}
|
||||
|
||||
func routeRequest(vm *goja.Runtime, c *gin.Context, body []byte) *goja.Object {
|
||||
req := hookRequestObject(vm, c.Request, body)
|
||||
_ = req.Set("context", routeRequestContext(vm, c))
|
||||
return req
|
||||
}
|
||||
|
||||
func routeResponseObject(vm *goja.Runtime, state *routeResponse) *goja.Object {
|
||||
res := vm.NewObject()
|
||||
_ = res.Set("statusCode", http.StatusOK)
|
||||
_ = res.Set("statusMessage", "")
|
||||
_ = res.Set("streaming", false)
|
||||
_ = res.Set("isAborted", func() bool { return state.aborted.Load() })
|
||||
_ = res.Set("setHeader", func(name string, value goja.Value) *goja.Object {
|
||||
state.mu.Lock()
|
||||
defer state.mu.Unlock()
|
||||
var values []string
|
||||
if err := vm.ExportTo(value, &values); err == nil {
|
||||
state.headers[name] = values
|
||||
} else {
|
||||
state.headers.Set(name, value.String())
|
||||
}
|
||||
return res
|
||||
})
|
||||
_ = res.Set("getHeader", func(name string) goja.Value {
|
||||
state.mu.Lock()
|
||||
defer state.mu.Unlock()
|
||||
values := state.headers.Values(name)
|
||||
if len(values) == 0 {
|
||||
return goja.Undefined()
|
||||
}
|
||||
if len(values) == 1 {
|
||||
return vm.ToValue(values[0])
|
||||
}
|
||||
return vm.ToValue(values)
|
||||
})
|
||||
_ = res.Set("removeHeader", func(name string) {
|
||||
state.mu.Lock()
|
||||
state.headers.Del(name)
|
||||
state.mu.Unlock()
|
||||
})
|
||||
_ = res.Set("write", func(call goja.FunctionCall) goja.Value {
|
||||
data := buffer.Bytes(vm, call.Argument(0))
|
||||
if value := res.Get("streaming"); value != nil && value.ToBoolean() {
|
||||
state.markStreaming()
|
||||
_ = state.pushChunk(data)
|
||||
return vm.ToValue(true)
|
||||
}
|
||||
state.mu.Lock()
|
||||
_, _ = state.body.Write(data)
|
||||
state.mu.Unlock()
|
||||
return vm.ToValue(true)
|
||||
})
|
||||
_ = res.Set("end", func(call goja.FunctionCall) goja.Value {
|
||||
state.mu.Lock()
|
||||
if len(call.Arguments) > 0 && !goja.IsUndefined(call.Argument(0)) && !goja.IsNull(call.Argument(0)) {
|
||||
_, _ = state.body.WriteString(call.Argument(0).String())
|
||||
}
|
||||
state.statusCode = int(res.Get("statusCode").ToInteger())
|
||||
if !state.ended {
|
||||
state.ended = true
|
||||
close(state.done)
|
||||
}
|
||||
state.mu.Unlock()
|
||||
return res
|
||||
})
|
||||
return res
|
||||
}
|
||||
@@ -0,0 +1,83 @@
|
||||
package plugin
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"os"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
// State persists the enabled flag, the approved permissions hash and the
|
||||
// last error for each installed plugin in DataDir/state.json. Keeping it
|
||||
// next to the plugin directories keeps plugin management self-contained.
|
||||
type State struct {
|
||||
mu sync.Mutex
|
||||
path string
|
||||
current map[string]PluginState
|
||||
}
|
||||
|
||||
// PluginState is the persisted state of one plugin.
|
||||
type PluginState struct {
|
||||
Enabled bool `json:"enabled"`
|
||||
ApprovedPermissionsHash string `json:"approved_permissions_hash"`
|
||||
LastError string `json:"last_error"`
|
||||
UpdatedAt time.Time `json:"updated_at"`
|
||||
}
|
||||
|
||||
func openState(path string) *State {
|
||||
return &State{path: path}
|
||||
}
|
||||
|
||||
func (s *State) get(short string) PluginState {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
s.ensure()
|
||||
return s.current[short]
|
||||
}
|
||||
|
||||
func (s *State) set(short string, st PluginState) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
s.ensure()
|
||||
st.UpdatedAt = time.Now().UTC()
|
||||
s.current[short] = st
|
||||
s.saveLocked()
|
||||
}
|
||||
|
||||
func (s *State) delete(short string) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
s.ensure()
|
||||
if _, ok := s.current[short]; ok {
|
||||
delete(s.current, short)
|
||||
s.saveLocked()
|
||||
}
|
||||
}
|
||||
|
||||
func (s *State) ensure() {
|
||||
if s.current != nil {
|
||||
return
|
||||
}
|
||||
s.current = map[string]PluginState{}
|
||||
data, err := os.ReadFile(s.path)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
var file struct {
|
||||
Plugins map[string]PluginState `json:"plugins"`
|
||||
}
|
||||
if json.Unmarshal(data, &file) == nil && file.Plugins != nil {
|
||||
s.current = file.Plugins
|
||||
}
|
||||
}
|
||||
|
||||
func (s *State) saveLocked() {
|
||||
file := struct {
|
||||
Plugins map[string]PluginState `json:"plugins"`
|
||||
}{Plugins: s.current}
|
||||
data, err := json.MarshalIndent(file, "", " ")
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
_ = os.WriteFile(s.path, data, 0644)
|
||||
}
|
||||
@@ -0,0 +1,201 @@
|
||||
package plugin
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"bytes"
|
||||
"context"
|
||||
"fmt"
|
||||
"image"
|
||||
"image/jpeg"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strconv"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/gin-gonic/gin"
|
||||
)
|
||||
|
||||
func encodeTestJPEG(t *testing.T) []byte {
|
||||
t.Helper()
|
||||
img := image.NewRGBA(image.Rect(0, 0, 32, 32))
|
||||
var buf bytes.Buffer
|
||||
if err := jpeg.Encode(&buf, img, nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return buf.Bytes()
|
||||
}
|
||||
|
||||
// TestBuiltinStyleStreamingWithResponseHooks 模拟普通 Gin handler 风格的流式
|
||||
// 路由(直接写 c.Writer + Flush)在存在 response hook 时仍然能把数据推给客户端。
|
||||
// 回归:bufferedResponseWriter 此前会吞掉所有 Write,直到 handler 返回(流式
|
||||
// handler 永不返回),客户端因此收不到任何数据(黑屏)。
|
||||
func TestBuiltinStyleStreamingWithResponseHooks(t *testing.T) {
|
||||
withTempDataDir(t)
|
||||
gin.SetMode(gin.TestMode)
|
||||
engine := gin.New()
|
||||
Init(engine)
|
||||
|
||||
hookZip := writePluginZip(t, map[string]string{
|
||||
"komari-plugin.json": `{"name":"Hook","short":"hook","version":"1.0.0","permissions":{"timeout":5,"allowHooks":true}}`,
|
||||
"script.js": `const server = require("server");
|
||||
function load() {
|
||||
server.hook("response", (req, res) => { res.headers["x-hooked"] = "1"; });
|
||||
}
|
||||
`,
|
||||
})
|
||||
if _, err := InstallZip(hookZip); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := SetEnabled("hook", true, true); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer func() { _ = SetEnabled("hook", false, false) }()
|
||||
|
||||
frame := encodeTestJPEG(t)
|
||||
engine.GET("/live", func(c *gin.Context) {
|
||||
c.Header("Content-Type", "multipart/x-mixed-replace; boundary=frame")
|
||||
for i := 0; i < 5; i++ {
|
||||
_, _ = c.Writer.WriteString("--frame\r\nContent-Type: image/jpeg\r\n\r\n")
|
||||
_, _ = c.Writer.Write(frame)
|
||||
_, _ = c.Writer.WriteString("\r\n")
|
||||
c.Writer.Flush()
|
||||
time.Sleep(20 * time.Millisecond)
|
||||
}
|
||||
})
|
||||
|
||||
wrapped := WrapHandler(engine)
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
req, _ := http.NewRequestWithContext(ctx, http.MethodGet, "http://127.0.0.1/live", nil)
|
||||
rec := httptest.NewRecorder()
|
||||
wrapped.ServeHTTP(rec, req)
|
||||
body := rec.Body.Bytes()
|
||||
|
||||
if ct := rec.Header().Get("Content-Type"); !strings.Contains(ct, "multipart/x-mixed-replace") {
|
||||
t.Fatalf("content-type = %q, want multipart stream", ct)
|
||||
}
|
||||
if !bytes.Contains(body, []byte("--frame")) || !bytes.Contains(body, []byte{0xFF, 0xD8}) {
|
||||
t.Fatalf("streamed body missing frames: len=%d", len(body))
|
||||
}
|
||||
// 流式响应不能被 response hook 改写,也不应带上 hook 头
|
||||
if got := rec.Header().Get("X-Hooked"); got != "" {
|
||||
t.Fatalf("streaming response unexpectedly rewritten by response hook: x-hooked=%q", got)
|
||||
}
|
||||
}
|
||||
|
||||
// TestRouteStreamingMJPEG 用本地“摄像头”(单张 JPEG)验证流式路由:
|
||||
// multipart 帧即时写出、客户端断开后脚本通过 isAborted 退出循环。
|
||||
func TestRouteStreamingMJPEG(t *testing.T) {
|
||||
withTempDataDir(t)
|
||||
gin.SetMode(gin.TestMode)
|
||||
engine := gin.New()
|
||||
Init(engine)
|
||||
|
||||
frame := encodeTestJPEG(t)
|
||||
camera := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
||||
w.Header().Set("Content-Type", "image/jpeg")
|
||||
_, _ = w.Write(frame)
|
||||
}))
|
||||
defer camera.Close()
|
||||
|
||||
script := fmt.Sprintf(`
|
||||
const server = require("server");
|
||||
function load() {
|
||||
server.route("GET", "/stream", async (req, res) => {
|
||||
res.streaming = true;
|
||||
res.setHeader("Content-Type", "multipart/x-mixed-replace; boundary=frame");
|
||||
res.setHeader("Cache-Control", "no-cache");
|
||||
while (!res.isAborted()) {
|
||||
try {
|
||||
const resp = await fetch("%s");
|
||||
const data = await resp.arrayBuffer();
|
||||
res.write("--frame\r\nContent-Type: image/jpeg\r\nContent-Length: " + data.byteLength + "\r\n\r\n");
|
||||
res.write(data);
|
||||
res.write("\r\n");
|
||||
} catch (e) {
|
||||
console.log("frame error: " + e.message);
|
||||
}
|
||||
await new Promise((resolve) => setTimeout(resolve, 50));
|
||||
}
|
||||
console.log("stream ended");
|
||||
});
|
||||
}
|
||||
`, camera.URL)
|
||||
zipPath := writePluginZip(t, map[string]string{
|
||||
"komari-plugin.json": `{"name":"Mjpeg","short":"mjpeg","version":"1.0.0","permissions":{"node":true,"timeout":5,"allowRoutes":true}}`,
|
||||
"script.js": script,
|
||||
})
|
||||
if _, err := InstallZip(zipPath); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := SetEnabled("mjpeg", true, true); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer func() { _ = SetEnabled("mjpeg", false, false) }()
|
||||
|
||||
srv := httptest.NewServer(engine)
|
||||
defer srv.Close()
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
req, _ := http.NewRequestWithContext(ctx, http.MethodGet, srv.URL+"/stream", nil)
|
||||
resp, err := http.DefaultClient.Do(req)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
if ct := resp.Header.Get("Content-Type"); !strings.Contains(ct, "multipart/x-mixed-replace") {
|
||||
t.Fatalf("content-type = %q", ct)
|
||||
}
|
||||
|
||||
// 读取第一个完整帧并校验 JPEG magic 与长度
|
||||
reader := bufio.NewReader(resp.Body)
|
||||
deadline := time.Now().Add(5 * time.Second)
|
||||
found := false
|
||||
for time.Now().Before(deadline) {
|
||||
line, err := reader.ReadBytes(10)
|
||||
if err != nil {
|
||||
t.Fatalf("read stream: %v", err)
|
||||
}
|
||||
if !bytes.Contains(line, []byte("--frame")) {
|
||||
continue
|
||||
}
|
||||
_, _ = reader.ReadBytes(10) // Content-Type
|
||||
lengthLine, err := reader.ReadBytes(10)
|
||||
if err != nil {
|
||||
t.Fatalf("read length line: %v", err)
|
||||
}
|
||||
length, _ := strconv.Atoi(strings.TrimSpace(strings.TrimPrefix(string(lengthLine), "Content-Length:")))
|
||||
_, _ = reader.ReadBytes(10) // 空行
|
||||
head := make([]byte, 2)
|
||||
if _, err := io.ReadFull(reader, head); err != nil {
|
||||
t.Fatalf("read jpeg head: %v", err)
|
||||
}
|
||||
if head[0] != 0xFF || head[1] != 0xD8 {
|
||||
t.Fatalf("jpeg magic = % X", head)
|
||||
}
|
||||
rest := make([]byte, length-2)
|
||||
if _, err := io.ReadFull(reader, rest); err != nil {
|
||||
t.Fatalf("read jpeg body: %v", err)
|
||||
}
|
||||
found = true
|
||||
break
|
||||
}
|
||||
if !found {
|
||||
t.Fatal("no frame received within 5s")
|
||||
}
|
||||
|
||||
// 客户端断开 → 脚本 isAborted 退出循环
|
||||
cancel()
|
||||
deadline = time.Now().Add(5 * time.Second)
|
||||
for time.Now().Before(deadline) {
|
||||
if logs := GetLogs("mjpeg"); strings.Contains(logs, "stream ended") {
|
||||
return
|
||||
}
|
||||
time.Sleep(100 * time.Millisecond)
|
||||
}
|
||||
t.Fatalf("stream did not stop after disconnect; logs = %q", GetLogs("mjpeg"))
|
||||
}
|
||||
@@ -0,0 +1,89 @@
|
||||
package plugin
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"github.com/komari-monitor/komari/utils"
|
||||
)
|
||||
|
||||
// CheckKomariVersion validates a manifest komari constraint against the
|
||||
// running server version. Supported constraints: empty (any version),
|
||||
// "x.y.z" (exact), and ">=x.y.z", ">x.y.z", "<=x.y.z", "<x.y.z". An optional
|
||||
// leading "v" is accepted.
|
||||
func CheckKomariVersion(constraint string) error {
|
||||
constraint = strings.TrimSpace(constraint)
|
||||
if constraint == "" {
|
||||
return nil
|
||||
}
|
||||
rest := constraint
|
||||
op := ""
|
||||
for _, candidate := range []string{">=", "<=", ">", "<", "="} {
|
||||
if strings.HasPrefix(rest, candidate) {
|
||||
op = candidate
|
||||
rest = strings.TrimSpace(strings.TrimPrefix(rest, candidate))
|
||||
break
|
||||
}
|
||||
}
|
||||
want, err := parseSemver(rest)
|
||||
if err != nil {
|
||||
return fmt.Errorf("invalid komari version constraint %q: %w", constraint, err)
|
||||
}
|
||||
have, err := parseSemver(utils.CurrentVersion)
|
||||
if err != nil {
|
||||
// A malformed server version must not block plugin loading.
|
||||
return nil
|
||||
}
|
||||
if !satisfies(compareSemver(have, want), op) {
|
||||
return fmt.Errorf("plugin requires komari %s, running %s", constraint, utils.CurrentVersion)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func satisfies(cmp int, op string) bool {
|
||||
switch op {
|
||||
case ">=":
|
||||
return cmp >= 0
|
||||
case ">":
|
||||
return cmp > 0
|
||||
case "<=":
|
||||
return cmp <= 0
|
||||
case "<":
|
||||
return cmp < 0
|
||||
default: // exact
|
||||
return cmp == 0
|
||||
}
|
||||
}
|
||||
|
||||
func parseSemver(s string) ([3]int, error) {
|
||||
s = strings.TrimSpace(strings.TrimPrefix(s, "v"))
|
||||
if s == "" {
|
||||
return [3]int{}, fmt.Errorf("empty version")
|
||||
}
|
||||
parts := strings.Split(s, ".")
|
||||
if len(parts) > 3 {
|
||||
return [3]int{}, fmt.Errorf("too many version parts")
|
||||
}
|
||||
var v [3]int
|
||||
for i, part := range parts {
|
||||
n, err := strconv.Atoi(part)
|
||||
if err != nil || n < 0 {
|
||||
return [3]int{}, fmt.Errorf("invalid version part %q", part)
|
||||
}
|
||||
v[i] = n
|
||||
}
|
||||
return v, nil
|
||||
}
|
||||
|
||||
func compareSemver(a, b [3]int) int {
|
||||
for i := 0; i < 3; i++ {
|
||||
if a[i] < b[i] {
|
||||
return -1
|
||||
}
|
||||
if a[i] > b[i] {
|
||||
return 1
|
||||
}
|
||||
}
|
||||
return 0
|
||||
}
|
||||
@@ -0,0 +1,30 @@
|
||||
package plugin
|
||||
|
||||
import "testing"
|
||||
|
||||
func TestCheckKomariVersion(t *testing.T) {
|
||||
tests := []struct {
|
||||
constraint string
|
||||
wantErr bool
|
||||
}{
|
||||
{"", false},
|
||||
{"0.0.1", false},
|
||||
{"v0.0.1", false},
|
||||
{">=0.0.1", false},
|
||||
{">0.0.1", true},
|
||||
{"<1.0.0", false},
|
||||
{"<=0.0.1", false},
|
||||
{">=99.0.0", true},
|
||||
{"0.1", true}, // 0.1.0 > 0.0.1
|
||||
{"1", true}, // 1.0.0 > 0.0.1
|
||||
{"1.2.3.4", true},
|
||||
{"abc", true},
|
||||
{">", true},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
err := CheckKomariVersion(tt.constraint)
|
||||
if (err != nil) != tt.wantErr {
|
||||
t.Errorf("CheckKomariVersion(%q) error = %v, wantErr %v", tt.constraint, err, tt.wantErr)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,233 @@
|
||||
package plugin
|
||||
|
||||
import (
|
||||
"time"
|
||||
|
||||
"github.com/dop251/goja"
|
||||
"github.com/gorilla/websocket"
|
||||
"github.com/komari-monitor/komari/web/connection"
|
||||
)
|
||||
|
||||
// WebSocket hook kinds. wsConnect runs right after the upgrade and may deny
|
||||
// the connection; wsMessage/wsSend see every inbound/outbound frame;
|
||||
// wsClose fires once when the connection ends. Path filtering reuses the
|
||||
// HTTP matcher syntax without the method prefix, since every upgrade is a
|
||||
// GET. Like the HTTP kinds, the input is lowercased, so any casing works
|
||||
// from JS.
|
||||
const (
|
||||
hookWSConnect hookKind = "wsconnect"
|
||||
hookWSMessage hookKind = "wsmessage"
|
||||
hookWSSend hookKind = "wssend"
|
||||
hookWSClose hookKind = "wsclose"
|
||||
)
|
||||
|
||||
// wsFrameHookTimeout bounds one frame-level callback. The agent read pumps
|
||||
// run on deadlines (v1: 11s, v2: readWait); a frame hook must never stall
|
||||
// them, so the wait is cut short and the frame passes through untouched.
|
||||
const wsFrameHookTimeout = time.Second
|
||||
|
||||
// maxWSFrameBytes caps the payload copied into the plugin event loop; larger
|
||||
// frames pass through without invoking the hooks.
|
||||
const maxWSFrameBytes = 8 << 20
|
||||
|
||||
// isWSHookKind reports whether kind is one of the WebSocket hook kinds.
|
||||
func isWSHookKind(kind string) bool {
|
||||
switch kind {
|
||||
case string(hookWSConnect), string(hookWSMessage), string(hookWSSend), string(hookWSClose):
|
||||
return true
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
// OnConnect implements connection.FrameInterceptor. It runs the wsConnect
|
||||
// hooks registered for the connection path; the first denial rejects the
|
||||
// connection.
|
||||
func (m *Manager) OnConnect(info *connection.ConnInfo) (deny bool, reason string) {
|
||||
for _, h := range m.wsHooksFor(hookWSConnect, info.Path) {
|
||||
var denied bool
|
||||
var why string
|
||||
queued, timedOut := runHookTurn(h.host, "plugin wsConnect hook "+h.short, func(vm *goja.Runtime) {
|
||||
_ = h.host.RunJob(vm, "plugin wsConnect hook "+h.short, func() error {
|
||||
res, err := h.fn(goja.Undefined(), wsHookContext(vm, info))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if goja.IsUndefined(res) || goja.IsNull(res) {
|
||||
return nil
|
||||
}
|
||||
obj := res.ToObject(vm)
|
||||
if v := obj.Get("deny"); v != nil && v.ToBoolean() {
|
||||
denied = true
|
||||
}
|
||||
if v := obj.Get("reason"); v != nil && !goja.IsUndefined(v) {
|
||||
why = v.String()
|
||||
}
|
||||
return nil
|
||||
})
|
||||
})
|
||||
if !queued {
|
||||
continue // runtime closed; later plugins still get a chance
|
||||
}
|
||||
if timedOut {
|
||||
m.logWSHookTimeout(h)
|
||||
continue
|
||||
}
|
||||
if denied {
|
||||
return true, why
|
||||
}
|
||||
}
|
||||
return false, ""
|
||||
}
|
||||
|
||||
// OnMessage implements connection.FrameInterceptor for inbound frames.
|
||||
func (m *Manager) OnMessage(info *connection.ConnInfo, frameType int, data []byte) (int, []byte, bool) {
|
||||
return m.runFrameHooks(hookWSMessage, info, frameType, data)
|
||||
}
|
||||
|
||||
// OnSend implements connection.FrameInterceptor for outbound frames.
|
||||
func (m *Manager) OnSend(info *connection.ConnInfo, frameType int, data []byte) (int, []byte, bool) {
|
||||
return m.runFrameHooks(hookWSSend, info, frameType, data)
|
||||
}
|
||||
|
||||
// runFrameHooks runs the wsMessage/wsSend chain for one frame in
|
||||
// registration order; each hook sees the previous hook's replacement. A drop
|
||||
// wins over later hooks. Timeouts, errors and oversized frames pass the
|
||||
// frame through.
|
||||
func (m *Manager) runFrameHooks(kind hookKind, info *connection.ConnInfo, frameType int, data []byte) (int, []byte, bool) {
|
||||
hooks := m.wsHooksFor(kind, info.Path)
|
||||
if len(hooks) == 0 || len(data) > maxWSFrameBytes {
|
||||
return frameType, data, false
|
||||
}
|
||||
next := data
|
||||
for _, h := range hooks {
|
||||
var drop bool
|
||||
var hookErr error
|
||||
queued, timedOut := runHookTurnTimeout(h.host, "plugin "+string(kind)+" hook "+h.short, wsFrameHookTimeout, func(vm *goja.Runtime) {
|
||||
msg := wsFrameObject(vm, info, frameType, next)
|
||||
hookErr = h.host.RunJob(vm, "plugin "+string(kind)+" hook "+h.short, func() error {
|
||||
res, err := h.fn(goja.Undefined(), wsHookContext(vm, info), msg)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if goja.IsUndefined(res) || goja.IsNull(res) {
|
||||
return nil
|
||||
}
|
||||
obj := res.ToObject(vm)
|
||||
if v := obj.Get("drop"); v != nil && v.ToBoolean() {
|
||||
drop = true
|
||||
return nil
|
||||
}
|
||||
if v := obj.Get("type"); v != nil && !goja.IsUndefined(v) {
|
||||
frameType = int(v.ToInteger())
|
||||
}
|
||||
if v := obj.Get("data"); v != nil && !goja.IsUndefined(v) && !goja.IsNull(v) {
|
||||
if data := wsFrameBytes(vm, v); data != nil {
|
||||
next = data
|
||||
}
|
||||
}
|
||||
return nil
|
||||
})
|
||||
})
|
||||
if !queued {
|
||||
return frameType, next, false // runtime closed: pass the frame on
|
||||
}
|
||||
if timedOut {
|
||||
m.logWSHookTimeout(h)
|
||||
continue
|
||||
}
|
||||
if hookErr != nil {
|
||||
m.logHookError(h, hookErr)
|
||||
continue
|
||||
}
|
||||
if drop {
|
||||
return frameType, nil, true
|
||||
}
|
||||
}
|
||||
return frameType, next, false
|
||||
}
|
||||
|
||||
// OnClose implements connection.FrameInterceptor for connection teardown.
|
||||
func (m *Manager) OnClose(info *connection.ConnInfo) {
|
||||
for _, h := range m.wsHooksFor(hookWSClose, info.Path) {
|
||||
var hookErr error
|
||||
queued, timedOut := runHookTurn(h.host, "plugin wsClose hook "+h.short, func(vm *goja.Runtime) {
|
||||
hookErr = h.host.RunJob(vm, "plugin wsClose hook "+h.short, func() error {
|
||||
_, err := h.fn(goja.Undefined(), wsHookContext(vm, info))
|
||||
return err
|
||||
})
|
||||
})
|
||||
if !queued {
|
||||
return
|
||||
}
|
||||
if timedOut {
|
||||
m.logWSHookTimeout(h)
|
||||
continue
|
||||
}
|
||||
if hookErr != nil {
|
||||
m.logHookError(h, hookErr)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// wsHooksFor returns the hooks of one ws kind whose matcher matches the
|
||||
// connection path, in registration order.
|
||||
func (m *Manager) wsHooksFor(kind hookKind, path string) []*hookEntry {
|
||||
all := m.hooksOf(kind)
|
||||
kept := all[:0]
|
||||
for _, h := range all {
|
||||
if h.matcher == nil || h.matcher.matchesPath(path) {
|
||||
kept = append(kept, h)
|
||||
}
|
||||
}
|
||||
return kept
|
||||
}
|
||||
|
||||
// wsHookContext builds the immutable per-connection context object.
|
||||
func wsHookContext(vm *goja.Runtime, info *connection.ConnInfo) *goja.Object {
|
||||
ctx := vm.NewObject()
|
||||
_ = ctx.Set("path", info.Path)
|
||||
_ = ctx.Set("connId", info.ID)
|
||||
_ = ctx.Set("remoteIp", info.RemoteIP)
|
||||
_ = ctx.Set("userAgent", info.UserAgent)
|
||||
if info.ClientUUID != "" {
|
||||
_ = ctx.Set("clientUuid", info.ClientUUID)
|
||||
}
|
||||
return ctx
|
||||
}
|
||||
|
||||
// wsFrameObject builds the frame object handed to wsMessage/wsSend hooks.
|
||||
// Text frames arrive as strings, binary frames as ArrayBuffers.
|
||||
func wsFrameObject(vm *goja.Runtime, info *connection.ConnInfo, frameType int, data []byte) *goja.Object {
|
||||
msg := vm.NewObject()
|
||||
_ = msg.Set("type", frameType)
|
||||
if frameType == websocket.BinaryMessage {
|
||||
_ = msg.Set("data", vm.NewArrayBuffer(append([]byte(nil), data...)))
|
||||
} else {
|
||||
_ = msg.Set("data", string(data))
|
||||
}
|
||||
_ = msg.Set("connId", info.ID)
|
||||
_ = msg.Set("path", info.Path)
|
||||
return msg
|
||||
}
|
||||
|
||||
// wsFrameBytes converts a hook-returned payload (string or ArrayBuffer) back
|
||||
// to bytes; anything else reports nil and leaves the frame untouched.
|
||||
func wsFrameBytes(vm *goja.Runtime, v goja.Value) []byte {
|
||||
if goja.IsString(v) {
|
||||
return []byte(v.String())
|
||||
}
|
||||
var data []byte
|
||||
if err := vm.ExportTo(v, &data); err == nil {
|
||||
return data
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// logWSHookTimeout records a timed-out ws hook for the plugin console.
|
||||
func (m *Manager) logWSHookTimeout(h *hookEntry) {
|
||||
_, _ = m.logStore(h.short).Write([]byte("[plugin] " + string(h.kind) + " hook timed out; the message passed through unmodified\n"))
|
||||
}
|
||||
|
||||
// The Manager is the process-wide WebSocket frame interceptor.
|
||||
var _ connection.FrameInterceptor = (*Manager)(nil)
|
||||
@@ -0,0 +1,319 @@
|
||||
package plugin
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/gin-gonic/gin"
|
||||
"github.com/komari-monitor/komari/web/connection"
|
||||
)
|
||||
|
||||
// wsManifest is a plugin manifest that declares the hooks permission (the
|
||||
// ws kinds reuse allowHooks).
|
||||
const wsManifest = `{"name":"Ws","short":"ws","version":"1.0.0","permissions":{"allowHooks":true,"timeout":5}}`
|
||||
|
||||
func wsInfo(path string) *connection.ConnInfo {
|
||||
return &connection.ConnInfo{ID: 1, Path: path, RemoteIP: "203.0.113.9", UserAgent: "test"}
|
||||
}
|
||||
|
||||
// TestWSHookConnectDenyAndAllow 验证 wsConnect 钩子:返回 {deny:true,reason}
|
||||
// 拒绝连接,返回 undefined 放行。
|
||||
func TestWSHookConnectDenyAndAllow(t *testing.T) {
|
||||
withTempDataDir(t)
|
||||
gin.SetMode(gin.TestMode)
|
||||
Init(gin.New())
|
||||
|
||||
zipPath := writePluginZip(t, map[string]string{
|
||||
"komari-plugin.json": wsManifest,
|
||||
"script.js": `
|
||||
const server = require("server");
|
||||
function load() {
|
||||
server.hook("wsConnect", (ctx) => {
|
||||
if (ctx.path === "/deny") return { deny: true, reason: "no agents here" };
|
||||
});
|
||||
}
|
||||
`,
|
||||
})
|
||||
if _, err := InstallZip(zipPath); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := SetEnabled("ws", true, true); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer func() { _ = SetEnabled("ws", false, false) }()
|
||||
|
||||
if deny, reason := global.OnConnect(wsInfo("/deny")); !deny || reason != "no agents here" {
|
||||
t.Fatalf("deny = %v, reason = %q", deny, reason)
|
||||
}
|
||||
if deny, reason := global.OnConnect(wsInfo("/api/clients/v2/rpc")); deny || reason != "" {
|
||||
t.Fatalf("unmatched path must be allowed, deny = %v, reason = %q", deny, reason)
|
||||
}
|
||||
}
|
||||
|
||||
// TestWSHookMessageChainReplaceAndDrop 验证 wsMessage 钩子链:按注册顺序链式
|
||||
// 传递替换结果,drop 优先于后续钩子。
|
||||
func TestWSHookMessageChainReplaceAndDrop(t *testing.T) {
|
||||
withTempDataDir(t)
|
||||
gin.SetMode(gin.TestMode)
|
||||
Init(gin.New())
|
||||
|
||||
zipPath := writePluginZip(t, map[string]string{
|
||||
"komari-plugin.json": wsManifest,
|
||||
"script.js": `
|
||||
const server = require("server");
|
||||
function load() {
|
||||
server.hook("wsMessage", (ctx, msg) => {
|
||||
if (msg.data === "hello") return { data: "hi" };
|
||||
});
|
||||
server.hook("wsMessage", (ctx, msg) => {
|
||||
if (msg.data === "dropme" || msg.data === "hi") return { drop: true };
|
||||
});
|
||||
}
|
||||
`,
|
||||
})
|
||||
if _, err := InstallZip(zipPath); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := SetEnabled("ws", true, true); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer func() { _ = SetEnabled("ws", false, false) }()
|
||||
|
||||
info := wsInfo("/api/clients/v2/rpc")
|
||||
// 第一个钩子改 hello->hi,第二个钩子丢弃 hi:结果应为 drop。
|
||||
if _, _, drop := global.OnMessage(info, 1, []byte("hello")); !drop {
|
||||
t.Fatal("chained replace followed by drop must drop")
|
||||
}
|
||||
// 第一个钩子不匹配,第二个钩子丢弃 dropme。
|
||||
if _, _, drop := global.OnMessage(info, 1, []byte("dropme")); !drop {
|
||||
t.Fatal("direct drop must drop")
|
||||
}
|
||||
// 两个钩子都不匹配:原样放行。
|
||||
typ, data, drop := global.OnMessage(info, 1, []byte("other"))
|
||||
if drop || typ != 1 || string(data) != "other" {
|
||||
t.Fatalf("pass-through = type %d data %q drop %v", typ, data, drop)
|
||||
}
|
||||
}
|
||||
|
||||
// TestWSHookSendReplacesType 验证 wsSend 钩子可以同时替换帧类型与载荷。
|
||||
func TestWSHookSendReplacesType(t *testing.T) {
|
||||
withTempDataDir(t)
|
||||
gin.SetMode(gin.TestMode)
|
||||
Init(gin.New())
|
||||
|
||||
zipPath := writePluginZip(t, map[string]string{
|
||||
"komari-plugin.json": wsManifest,
|
||||
"script.js": `
|
||||
const server = require("server");
|
||||
function load() {
|
||||
server.hook("wsSend", (ctx, msg) => {
|
||||
if (msg.data === "text") return { type: 2, data: "bin" };
|
||||
});
|
||||
}
|
||||
`,
|
||||
})
|
||||
if _, err := InstallZip(zipPath); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := SetEnabled("ws", true, true); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer func() { _ = SetEnabled("ws", false, false) }()
|
||||
|
||||
typ, data, drop := global.OnSend(wsInfo("/api/rpc2"), 1, []byte("text"))
|
||||
if drop || typ != 2 || string(data) != "bin" {
|
||||
t.Fatalf("send rewrite = type %d data %q drop %v", typ, data, drop)
|
||||
}
|
||||
}
|
||||
|
||||
// TestWSHookClose 验证 wsClose 钩子在连接结束时触发。
|
||||
func TestWSHookClose(t *testing.T) {
|
||||
withTempDataDir(t)
|
||||
gin.SetMode(gin.TestMode)
|
||||
Init(gin.New())
|
||||
|
||||
zipPath := writePluginZip(t, map[string]string{
|
||||
"komari-plugin.json": `{"name":"Ws","short":"ws","version":"1.0.0","permissions":{"node":true,"allowHooks":true,"timeout":5}}`,
|
||||
"script.js": `
|
||||
const fs = require("fs");
|
||||
const server = require("server");
|
||||
function load() {
|
||||
server.hook("wsClose", (ctx) => {
|
||||
fs.appendFileSync("closes.txt", ctx.path + "\n");
|
||||
});
|
||||
}
|
||||
`,
|
||||
})
|
||||
if _, err := InstallZip(zipPath); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := SetEnabled("ws", true, true); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer func() { _ = SetEnabled("ws", false, false) }()
|
||||
|
||||
global.OnClose(wsInfo("/api/clients/v2/rpc"))
|
||||
global.OnClose(wsInfo("/api/clients/v2/rpc"))
|
||||
|
||||
data, err := os.ReadFile(filepath.Join(DataDir, "ws", "closes.txt"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got := string(data); strings.Count(got, "/api/clients/v2/rpc") != 2 {
|
||||
t.Fatalf("wsClose fired %d times: %q", strings.Count(got, "/api/clients/v2/rpc"), got)
|
||||
}
|
||||
}
|
||||
|
||||
// TestWSHookPathFiltering 验证 ws kind 的路径 matcher 只拦截匹配端点的连接。
|
||||
func TestWSHookPathFiltering(t *testing.T) {
|
||||
withTempDataDir(t)
|
||||
gin.SetMode(gin.TestMode)
|
||||
Init(gin.New())
|
||||
|
||||
zipPath := writePluginZip(t, map[string]string{
|
||||
"komari-plugin.json": wsManifest,
|
||||
"script.js": `
|
||||
const server = require("server");
|
||||
function load() {
|
||||
server.hook("wsMessage", "/api/clients/v2/rpc", (ctx, msg) => {
|
||||
return { data: "filtered" };
|
||||
});
|
||||
}
|
||||
`,
|
||||
})
|
||||
if _, err := InstallZip(zipPath); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := SetEnabled("ws", true, true); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer func() { _ = SetEnabled("ws", false, false) }()
|
||||
|
||||
typ, data, _ := global.OnMessage(wsInfo("/api/clients/v2/rpc"), 1, []byte("x"))
|
||||
if string(data) != "filtered" {
|
||||
t.Fatalf("matching path not intercepted: %q", data)
|
||||
}
|
||||
typ, data, _ = global.OnMessage(wsInfo("/api/rpc2"), 1, []byte("x"))
|
||||
if typ != 1 || string(data) != "x" {
|
||||
t.Fatalf("non-matching path intercepted: type %d data %q", typ, data)
|
||||
}
|
||||
}
|
||||
|
||||
// TestWSHookMethodPrefixRejected 验证 ws kind 的 matcher 不接受方法前缀。
|
||||
func TestWSHookMethodPrefixRejected(t *testing.T) {
|
||||
withTempDataDir(t)
|
||||
gin.SetMode(gin.TestMode)
|
||||
Init(gin.New())
|
||||
|
||||
zipPath := writePluginZip(t, map[string]string{
|
||||
"komari-plugin.json": wsManifest,
|
||||
"script.js": `
|
||||
const server = require("server");
|
||||
function load() {
|
||||
server.hook("wsMessage", "POST /api/x", (ctx, msg) => {});
|
||||
}
|
||||
`,
|
||||
})
|
||||
if _, err := InstallZip(zipPath); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := SetEnabled("ws", true, true); err == nil || !strings.Contains(err.Error(), "path matcher") {
|
||||
t.Fatalf("method prefix must fail loading: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// TestWSHookRequiresPermission 验证 ws kind 缺 allowHooks 时加载失败(与
|
||||
// 请求/响应钩子共用同一权限)。
|
||||
func TestWSHookRequiresPermission(t *testing.T) {
|
||||
withTempDataDir(t)
|
||||
gin.SetMode(gin.TestMode)
|
||||
Init(gin.New())
|
||||
|
||||
zipPath := writePluginZip(t, map[string]string{
|
||||
"komari-plugin.json": `{"name":"Ws","short":"ws","version":"1.0.0","permissions":{"timeout":5}}`,
|
||||
"script.js": `
|
||||
const server = require("server");
|
||||
function load() {
|
||||
server.hook("wsConnect", (ctx) => {});
|
||||
}
|
||||
`,
|
||||
})
|
||||
if _, err := InstallZip(zipPath); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := SetEnabled("ws", true, true); err == nil || !strings.Contains(err.Error(), "allowHooks") {
|
||||
t.Fatalf("missing allowHooks must fail loading: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// TestWSHookUnloadRemovesHooks 验证卸载后 ws 钩子全部移除。
|
||||
func TestWSHookUnloadRemovesHooks(t *testing.T) {
|
||||
withTempDataDir(t)
|
||||
gin.SetMode(gin.TestMode)
|
||||
Init(gin.New())
|
||||
|
||||
zipPath := writePluginZip(t, map[string]string{
|
||||
"komari-plugin.json": wsManifest,
|
||||
"script.js": `
|
||||
const server = require("server");
|
||||
function load() {
|
||||
server.hook("wsMessage", (ctx, msg) => { return { data: "hooked" }; });
|
||||
server.hook("wsConnect", (ctx) => { return { deny: true }; });
|
||||
}
|
||||
`,
|
||||
})
|
||||
if _, err := InstallZip(zipPath); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := SetEnabled("ws", true, true); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if deny, _ := global.OnConnect(wsInfo("/x")); !deny {
|
||||
t.Fatal("wsConnect must deny while loaded")
|
||||
}
|
||||
if err := SetEnabled("ws", false, false); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if deny, _ := global.OnConnect(wsInfo("/x")); deny {
|
||||
t.Fatal("wsConnect must allow after unload")
|
||||
}
|
||||
typ, data, _ := global.OnMessage(wsInfo("/x"), 1, []byte("plain"))
|
||||
if typ != 1 || string(data) != "plain" {
|
||||
t.Fatalf("frame hook still active after unload: %q", data)
|
||||
}
|
||||
}
|
||||
|
||||
// TestWSHookOversizedFramePassesThrough 验证超过帧上限的载荷不经过钩子。
|
||||
func TestWSHookOversizedFramePassesThrough(t *testing.T) {
|
||||
withTempDataDir(t)
|
||||
gin.SetMode(gin.TestMode)
|
||||
Init(gin.New())
|
||||
|
||||
zipPath := writePluginZip(t, map[string]string{
|
||||
"komari-plugin.json": wsManifest,
|
||||
"script.js": `
|
||||
const server = require("server");
|
||||
function load() {
|
||||
server.hook("wsMessage", (ctx, msg) => { return { data: "replaced" }; });
|
||||
}
|
||||
`,
|
||||
})
|
||||
if _, err := InstallZip(zipPath); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := SetEnabled("ws", true, true); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer func() { _ = SetEnabled("ws", false, false) }()
|
||||
|
||||
big := make([]byte, maxWSFrameBytes+1)
|
||||
for i := range big {
|
||||
big[i] = 'x'
|
||||
}
|
||||
typ, data, _ := global.OnMessage(wsInfo("/x"), 1, big)
|
||||
if typ != 1 || len(data) != len(big) {
|
||||
t.Fatalf("oversized frame must pass through untouched, got %d bytes", len(data))
|
||||
}
|
||||
}
|
||||
@@ -1,9 +1,9 @@
|
||||
package corn
|
||||
package scheduler
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"log"
|
||||
logger "github.com/komari-monitor/komari/utils/log"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
@@ -172,7 +172,7 @@ func (m *Manager) run(ctx context.Context, name string, s schedule, runImmediate
|
||||
|
||||
nextTick := s.Next(time.Now())
|
||||
if nextTick.IsZero() {
|
||||
log.Printf("corn job %s has no next run time", name)
|
||||
logger.Warnf("scheduler", "corn job %s has no next run time", name)
|
||||
return
|
||||
}
|
||||
timer := time.NewTimer(time.Until(nextTick))
|
||||
@@ -203,7 +203,7 @@ func resetTimer(timer *time.Timer, duration time.Duration) {
|
||||
func safeRun(ctx context.Context, name string, fn Func) {
|
||||
defer func() {
|
||||
if r := recover(); r != nil {
|
||||
log.Printf("corn job %s panic: %v", name, r)
|
||||
logger.Errorf("scheduler", "corn job %s panic: %v", name, r)
|
||||
}
|
||||
}()
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
package corn
|
||||
package scheduler
|
||||
|
||||
import (
|
||||
"testing"
|
||||
@@ -0,0 +1,47 @@
|
||||
package server
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net/http"
|
||||
|
||||
"github.com/gin-gonic/gin"
|
||||
"github.com/komari-monitor/komari/internal/config"
|
||||
)
|
||||
|
||||
// cleanupFunc is a cleanup action run during shutdown.
|
||||
type cleanupFunc struct {
|
||||
name string
|
||||
fn func(ctx context.Context) error
|
||||
}
|
||||
|
||||
// App owns the server lifecycle. Individual phases live in focused files:
|
||||
// bootstrap, metric store, providers, guides, router, and runtime.
|
||||
type App struct {
|
||||
listenAddr string
|
||||
settings *config.Settings
|
||||
engine *gin.Engine
|
||||
server *http.Server
|
||||
reload *ReloadManager
|
||||
dbReady bool
|
||||
oauthReady bool
|
||||
metricStoreCleanupAdded bool
|
||||
|
||||
cleanups []cleanupFunc
|
||||
}
|
||||
|
||||
// Options configures the process-wide application runtime.
|
||||
type Options struct {
|
||||
ListenAddr string
|
||||
}
|
||||
|
||||
// New constructs an empty application. Initialization happens in the
|
||||
// explicit lifecycle phases called by the command entrypoint.
|
||||
func New(options Options) *App {
|
||||
return &App{listenAddr: options.ListenAddr, reload: NewReloadManager()}
|
||||
}
|
||||
|
||||
// addCleanup registers a cleanup action. Shutdown executes actions in LIFO
|
||||
// order, mirroring resource initialization order.
|
||||
func (a *App) addCleanup(name string, fn func(ctx context.Context) error) {
|
||||
a.cleanups = append(a.cleanups, cleanupFunc{name: name, fn: fn})
|
||||
}
|
||||
@@ -0,0 +1,82 @@
|
||||
package server
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
|
||||
"github.com/gin-gonic/gin"
|
||||
)
|
||||
|
||||
func TestRetryMetricStoreConnectionStopsAfterRecovery(t *testing.T) {
|
||||
wantErr := errors.New("temporary connection failure")
|
||||
attempts := 0
|
||||
err := retryMetricStoreConnection(3, 0, func() error {
|
||||
attempts++
|
||||
if attempts < 3 {
|
||||
return wantErr
|
||||
}
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("retry returned error after recovery: %v", err)
|
||||
}
|
||||
if attempts != 3 {
|
||||
t.Fatalf("attempts = %d, want 3", attempts)
|
||||
}
|
||||
}
|
||||
|
||||
func TestGuideNoRouteRejectsUnknownAPIAndRedirectsPages(t *testing.T) {
|
||||
gin.SetMode(gin.TestMode)
|
||||
router := gin.New()
|
||||
router.Use(noStoreAPIResponses())
|
||||
router.NoRoute(guideNoRoute("/install", "Not found in install mode", nil))
|
||||
|
||||
apiResponse := httptest.NewRecorder()
|
||||
router.ServeHTTP(apiResponse, httptest.NewRequest(http.MethodGet, "/api/unknown", nil))
|
||||
if apiResponse.Code != http.StatusNotFound {
|
||||
t.Fatalf("API status = %d, want %d", apiResponse.Code, http.StatusNotFound)
|
||||
}
|
||||
if got := apiResponse.Header().Get("Cache-Control"); got != "no-store" {
|
||||
t.Fatalf("API cache control = %q, want no-store", got)
|
||||
}
|
||||
|
||||
pageResponse := httptest.NewRecorder()
|
||||
router.ServeHTTP(pageResponse, httptest.NewRequest(http.MethodGet, "/stale-guide", nil))
|
||||
if pageResponse.Code != http.StatusTemporaryRedirect {
|
||||
t.Fatalf("page status = %d, want %d", pageResponse.Code, http.StatusTemporaryRedirect)
|
||||
}
|
||||
if got := pageResponse.Header().Get("Location"); got != "/install" {
|
||||
t.Fatalf("redirect location = %q, want /install", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRetryMetricStoreConnectionReturnsLastError(t *testing.T) {
|
||||
wantErr := errors.New("connection unavailable")
|
||||
attempts := 0
|
||||
err := retryMetricStoreConnection(3, 0, func() error {
|
||||
attempts++
|
||||
return wantErr
|
||||
})
|
||||
if !errors.Is(err, wantErr) {
|
||||
t.Fatalf("error = %v, want %v", err, wantErr)
|
||||
}
|
||||
if attempts != 3 {
|
||||
t.Fatalf("attempts = %d, want 3", attempts)
|
||||
}
|
||||
}
|
||||
|
||||
func TestShutdownReturnsCleanupError(t *testing.T) {
|
||||
wantErr := errors.New("queued reports were not flushed")
|
||||
app := New(Options{})
|
||||
app.addCleanup("metric-report-batcher", func(_ context.Context) error {
|
||||
return wantErr
|
||||
})
|
||||
|
||||
err := app.Shutdown()
|
||||
if !errors.Is(err, wantErr) {
|
||||
t.Fatalf("shutdown error = %v, want %v", err, wantErr)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,40 @@
|
||||
package server
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
|
||||
"github.com/gin-gonic/gin"
|
||||
"github.com/komari-monitor/komari/database/dbcore"
|
||||
"github.com/komari-monitor/komari/internal/config"
|
||||
"github.com/komari-monitor/komari/utils"
|
||||
)
|
||||
|
||||
// Bootstrap initializes the data directory, primary database, and settings.
|
||||
func (a *App) Bootstrap() error {
|
||||
if err := os.MkdirAll("./data/theme", os.ModePerm); err != nil {
|
||||
return fmt.Errorf("failed to create theme directory: %w", err)
|
||||
}
|
||||
if err := os.MkdirAll("./data/plugin", os.ModePerm); err != nil {
|
||||
return fmt.Errorf("failed to create plugin directory: %w", err)
|
||||
}
|
||||
if err := os.MkdirAll("./data/plugin-data", os.ModePerm); err != nil {
|
||||
return fmt.Errorf("failed to create plugin storage directory: %w", err)
|
||||
}
|
||||
|
||||
dbcore.SetVersionID(utils.CurrentVersion + "-" + utils.VersionHash)
|
||||
if err := dbcore.Initialize(); err != nil {
|
||||
return fmt.Errorf("failed to initialize database: %w", err)
|
||||
}
|
||||
a.dbReady = true
|
||||
a.addCleanup("database", func(context.Context) error { return dbcore.Close() })
|
||||
|
||||
gin.SetMode(gin.ReleaseMode)
|
||||
settings, err := config.GetManyAs[config.Settings]()
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to load settings: %w", err)
|
||||
}
|
||||
a.settings = settings
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,118 @@
|
||||
package server
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"os"
|
||||
"os/signal"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
"github.com/gin-gonic/gin"
|
||||
logger "github.com/komari-monitor/komari/utils/log"
|
||||
"github.com/komari-monitor/komari/web/api"
|
||||
frontendpublic "github.com/komari-monitor/komari/web/public"
|
||||
"github.com/komari-monitor/komari/web/security"
|
||||
)
|
||||
|
||||
type guideController interface {
|
||||
Activate()
|
||||
Deactivate()
|
||||
Register(*gin.Engine)
|
||||
Done() <-chan struct{}
|
||||
}
|
||||
|
||||
type guideServerConfig struct {
|
||||
pagePath string
|
||||
missingAPI string
|
||||
logMessage string
|
||||
requireIdentity bool
|
||||
restrictedStatic bool
|
||||
}
|
||||
|
||||
// runGuideServer hosts one temporary, narrowly scoped guide. It deliberately
|
||||
// constructs a new Gin engine for each mode so normal routes never leak into
|
||||
// installation, upgrade, or recovery flows.
|
||||
func (a *App) runGuideServer(controller guideController, cfg guideServerConfig) (bool, error) {
|
||||
controller.Activate()
|
||||
defer controller.Deactivate()
|
||||
|
||||
r := gin.New()
|
||||
r.Use(logger.GinLogger(), logger.GinRecovery(), noStoreAPIResponses())
|
||||
if cfg.requireIdentity {
|
||||
cors := security.NewCorsController(a.settings.CorsOriginCheckEnabled, a.settings.CorsAllowedOrigins)
|
||||
r.Use(cors.Middleware(), api.IdentityMiddleware())
|
||||
}
|
||||
controller.Register(r)
|
||||
|
||||
serveStatic := frontendpublic.Static
|
||||
if cfg.restrictedStatic {
|
||||
serveStatic = frontendpublic.StaticRestricted
|
||||
}
|
||||
serveStatic(r.Group("/"), func(handlers ...gin.HandlerFunc) {
|
||||
r.NoRoute(guideNoRoute(cfg.pagePath, cfg.missingAPI, handlers))
|
||||
})
|
||||
|
||||
server := &http.Server{Addr: a.listenAddr, Handler: r}
|
||||
a.engine = r
|
||||
a.server = server
|
||||
serverErr := make(chan error, 1)
|
||||
logger.Infof("server", cfg.logMessage, a.listenAddr)
|
||||
go func() {
|
||||
if err := server.ListenAndServe(); err != nil && err != http.ErrServerClosed {
|
||||
serverErr <- err
|
||||
}
|
||||
}()
|
||||
|
||||
quit := make(chan os.Signal, 1)
|
||||
signal.Notify(quit, os.Interrupt, syscall.SIGTERM)
|
||||
defer signal.Stop(quit)
|
||||
|
||||
select {
|
||||
case err := <-serverErr:
|
||||
return false, fmt.Errorf("listen in guide mode: %w", err)
|
||||
case <-quit:
|
||||
return false, a.Shutdown()
|
||||
case <-controller.Done():
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||
defer cancel()
|
||||
if err := server.Shutdown(ctx); err != nil {
|
||||
return false, fmt.Errorf("stop guide server: %w", err)
|
||||
}
|
||||
a.server = nil
|
||||
a.engine = nil
|
||||
return true, nil
|
||||
}
|
||||
}
|
||||
|
||||
func noStoreAPIResponses() gin.HandlerFunc {
|
||||
return func(c *gin.Context) {
|
||||
if strings.HasPrefix(c.Request.URL.Path, "/api") {
|
||||
c.Header("Cache-Control", "no-store")
|
||||
}
|
||||
c.Next()
|
||||
}
|
||||
}
|
||||
|
||||
func guideNoRoute(pagePath, missingAPI string, handlers []gin.HandlerFunc) gin.HandlerFunc {
|
||||
return func(c *gin.Context) {
|
||||
requestPath := c.Request.URL.Path
|
||||
if strings.HasPrefix(requestPath, "/api") {
|
||||
api.RespondError(c, http.StatusNotFound, missingAPI)
|
||||
return
|
||||
}
|
||||
if c.Request.Method == http.MethodGet && requestPath != pagePath && filepath.Ext(requestPath) == "" {
|
||||
c.Redirect(http.StatusTemporaryRedirect, pagePath)
|
||||
return
|
||||
}
|
||||
for _, handler := range handlers {
|
||||
handler(c)
|
||||
if c.IsAborted() {
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,96 @@
|
||||
package server
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"github.com/komari-monitor/komari/database/dbcore"
|
||||
"github.com/komari-monitor/komari/internal/metricstore"
|
||||
"github.com/komari-monitor/komari/database/models"
|
||||
"github.com/komari-monitor/komari/internal/migrations"
|
||||
installweb "github.com/komari-monitor/komari/web/install"
|
||||
migrationweb "github.com/komari-monitor/komari/web/migration"
|
||||
recoveryweb "github.com/komari-monitor/komari/web/recovery"
|
||||
)
|
||||
|
||||
// InstallRequired reports whether the instance still needs the first-run guide.
|
||||
func (a *App) InstallRequired() (bool, error) {
|
||||
var count int64
|
||||
if err := dbcore.GetDBInstance().Model(&models.User{}).Count(&count).Error; err != nil {
|
||||
return false, err
|
||||
}
|
||||
return count == 0, nil
|
||||
}
|
||||
|
||||
type DatabaseMigrationRequirement struct {
|
||||
mode migrationweb.Mode
|
||||
summary migrations.LegacyMonitoringSummary
|
||||
}
|
||||
|
||||
func (r DatabaseMigrationRequirement) Required() bool { return r.mode != "" }
|
||||
|
||||
// DatabaseMigrationRequired checks both supported migration inputs before the
|
||||
// normal Metric Store connection is initialized. Structure upgrades run first
|
||||
// when both inputs exist, then the startup loop detects the legacy tables on
|
||||
// its next pass.
|
||||
func (a *App) DatabaseMigrationRequired() (DatabaseMigrationRequirement, error) {
|
||||
structureRequired, err := metricstore.StructureUpgradeRequired(context.Background())
|
||||
if err != nil {
|
||||
return DatabaseMigrationRequirement{}, err
|
||||
}
|
||||
if structureRequired {
|
||||
return DatabaseMigrationRequirement{mode: migrationweb.ModeMetricStructure}, nil
|
||||
}
|
||||
legacyRequired, summary, err := migrations.LegacyMonitoringMigrationRequired(dbcore.GetDBInstance())
|
||||
if err != nil {
|
||||
return DatabaseMigrationRequirement{}, err
|
||||
}
|
||||
if legacyRequired {
|
||||
return DatabaseMigrationRequirement{mode: migrationweb.ModeLegacyMonitoring, summary: summary}, nil
|
||||
}
|
||||
return DatabaseMigrationRequirement{}, nil
|
||||
}
|
||||
|
||||
// RunInstallGuide exposes only first-run installation APIs. It intentionally
|
||||
// does not mount authentication or normal application routes.
|
||||
func (a *App) RunInstallGuide() (bool, error) {
|
||||
return a.runGuideServer(installweb.NewController(dbcore.GetDBInstance()), guideServerConfig{
|
||||
pagePath: installweb.PagePath,
|
||||
missingAPI: "Not found in install mode",
|
||||
logMessage: "First-run installation guide is available on %s",
|
||||
})
|
||||
}
|
||||
|
||||
// RunMetricStoreRecovery keeps login available while exposing only the
|
||||
// administrator-protected metric-store recovery API.
|
||||
func (a *App) RunMetricStoreRecovery(initialErr error) (bool, error) {
|
||||
a.initOAuth()
|
||||
return a.runGuideServer(recoveryweb.NewController(initialErr, metricStoreReconnectAttempts), guideServerConfig{
|
||||
pagePath: recoveryweb.PagePath,
|
||||
missingAPI: "Not found in database recovery mode",
|
||||
logMessage: "Metric store recovery is available on %s",
|
||||
requireIdentity: true,
|
||||
restrictedStatic: true,
|
||||
})
|
||||
}
|
||||
|
||||
// RunDatabaseMigration serves the same authenticated guide and status model
|
||||
// for either migration input while leaving each conversion engine independent.
|
||||
func (a *App) RunDatabaseMigration(requirement DatabaseMigrationRequirement) (bool, error) {
|
||||
a.initOAuth()
|
||||
var controller guideController
|
||||
switch requirement.mode {
|
||||
case migrationweb.ModeMetricStructure:
|
||||
controller = migrationweb.NewStructureController()
|
||||
case migrationweb.ModeLegacyMonitoring:
|
||||
controller = migrationweb.NewLegacyController(dbcore.GetDBInstance(), requirement.summary)
|
||||
default:
|
||||
return false, nil
|
||||
}
|
||||
return a.runGuideServer(controller, guideServerConfig{
|
||||
pagePath: migrationweb.PagePath,
|
||||
missingAPI: "Not found in database migration mode",
|
||||
logMessage: "Database migration guide is available on %s",
|
||||
requireIdentity: true,
|
||||
restrictedStatic: true,
|
||||
})
|
||||
}
|
||||
@@ -0,0 +1,90 @@
|
||||
package server
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"github.com/komari-monitor/komari/database/auditlog"
|
||||
"github.com/komari-monitor/komari/internal/metricstore"
|
||||
"github.com/komari-monitor/komari/internal/config"
|
||||
logger "github.com/komari-monitor/komari/utils/log"
|
||||
)
|
||||
|
||||
const (
|
||||
metricStoreReconnectAttempts = 3
|
||||
metricStoreReconnectInterval = 5 * time.Second
|
||||
)
|
||||
|
||||
// ConnectMetricStore performs one connection attempt and registers cleanup
|
||||
// only after a store has actually opened.
|
||||
func (a *App) ConnectMetricStore() error {
|
||||
if err := metricstore.InitializeStore(); err != nil {
|
||||
return fmt.Errorf("failed to initialize metric store: %s", redactMetricStoreError(err))
|
||||
}
|
||||
if !a.metricStoreCleanupAdded {
|
||||
a.addCleanup("metric-store", metricstore.CloseStoreContext)
|
||||
a.metricStoreCleanupAdded = true
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func redactMetricStoreError(err error) string {
|
||||
if err == nil {
|
||||
return ""
|
||||
}
|
||||
dsn := ""
|
||||
if cfg, cfgErr := config.GetManyAs[metricstore.MetricStoreConfig](); cfgErr == nil {
|
||||
dsn = cfg.DSN
|
||||
}
|
||||
return metricstore.RedactConnectionError(err.Error(), dsn)
|
||||
}
|
||||
|
||||
// ConnectMetricStoreWithRetry retries the monitoring database connection.
|
||||
func (a *App) ConnectMetricStoreWithRetry() error {
|
||||
attempt := 0
|
||||
err := retryMetricStoreConnection(metricStoreReconnectAttempts, metricStoreReconnectInterval, func() error {
|
||||
attempt++
|
||||
err := a.ConnectMetricStore()
|
||||
if err != nil {
|
||||
logger.Warn("server", "Metric store connection attempt failed", "attempt", attempt, "max_attempts", metricStoreReconnectAttempts, "error", err)
|
||||
}
|
||||
return err
|
||||
})
|
||||
if err == nil && attempt > 1 {
|
||||
logger.Infof("server", "Metric store connection recovered on attempt %d/%d", attempt, metricStoreReconnectAttempts)
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
func retryMetricStoreConnection(attempts int, interval time.Duration, connect func() error) error {
|
||||
if attempts < 1 {
|
||||
return fmt.Errorf("metric store retry attempts must be positive")
|
||||
}
|
||||
var lastErr error
|
||||
for attempt := 1; attempt <= attempts; attempt++ {
|
||||
if attempt > 1 {
|
||||
time.Sleep(interval)
|
||||
}
|
||||
if err := connect(); err == nil {
|
||||
return nil
|
||||
} else {
|
||||
lastErr = err
|
||||
}
|
||||
}
|
||||
return lastErr
|
||||
}
|
||||
|
||||
// InitStores opens the metric store and starts its report batcher. Historical
|
||||
// data migrations are started only through an explicit administrator action.
|
||||
func (a *App) InitStores() error {
|
||||
if err := a.ConnectMetricStore(); err != nil {
|
||||
auditlog.EventLog("error", fmt.Sprintf("Failed to initialize metric store: %v", err))
|
||||
return err
|
||||
}
|
||||
metricstore.StartReportBatcher()
|
||||
a.addCleanup("metric-report-batcher", metricstore.StopReportBatcher)
|
||||
// A store-to-store migration holds the exclusive operation lease. Stop it
|
||||
// before flushing queued reports, which need the shared lease to write.
|
||||
a.addCleanup("metric-store-migration", metricstore.StopStoreMigrationForShutdown)
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,38 @@
|
||||
package server
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
|
||||
"github.com/komari-monitor/komari/database/auditlog"
|
||||
"github.com/komari-monitor/komari/utils/geoip"
|
||||
logger "github.com/komari-monitor/komari/utils/log"
|
||||
"github.com/komari-monitor/komari/utils/messageSender"
|
||||
"github.com/komari-monitor/komari/web/oauth"
|
||||
)
|
||||
|
||||
// InitProviders initializes providers needed by the normal application.
|
||||
func (a *App) InitProviders() error {
|
||||
a.initOAuth()
|
||||
|
||||
go geoip.InitGeoIp()
|
||||
a.addCleanup("geoip", func(context.Context) error { return geoip.Shutdown() })
|
||||
|
||||
messageSender.Initialize()
|
||||
a.addCleanup("message-sender", func(context.Context) error { return messageSender.Shutdown() })
|
||||
return nil
|
||||
}
|
||||
|
||||
// initOAuth initializes OAuth once. Restricted authenticated guides need it
|
||||
// before the normal provider initialization phase.
|
||||
func (a *App) initOAuth() {
|
||||
if a.oauthReady {
|
||||
return
|
||||
}
|
||||
if err := oauth.Initialize(); err != nil {
|
||||
logger.Errorf("server", "Failed to initialize OAuth provider: %v", err)
|
||||
auditlog.EventLog("error", fmt.Sprintf("Failed to initialize OAuth provider: %v", err))
|
||||
}
|
||||
a.oauthReady = true
|
||||
a.addCleanup("oauth", func(context.Context) error { return oauth.Shutdown() })
|
||||
}
|
||||
@@ -1,9 +1,9 @@
|
||||
package cmd
|
||||
package server
|
||||
|
||||
import (
|
||||
"log"
|
||||
logger "github.com/komari-monitor/komari/utils/log"
|
||||
|
||||
"github.com/komari-monitor/komari/pkg/config"
|
||||
"github.com/komari-monitor/komari/internal/config"
|
||||
)
|
||||
|
||||
// reloadHandler 是单个配置热重载处理器。
|
||||
@@ -60,7 +60,7 @@ func (m *ReloadManager) Start() {
|
||||
func (m *ReloadManager) dispatch(h reloadHandler, event config.ConfigEvent) {
|
||||
defer func() {
|
||||
if r := recover(); r != nil {
|
||||
log.Printf("config reload handler %q panicked: %v", h.name, r)
|
||||
logger.Errorf("reload", "config reload handler %q panicked: %v", h.name, r)
|
||||
}
|
||||
}()
|
||||
h.handler(event)
|
||||
@@ -0,0 +1,253 @@
|
||||
package server
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"os"
|
||||
"os/signal"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
"github.com/gin-gonic/gin"
|
||||
"github.com/komari-monitor/komari/database"
|
||||
"github.com/komari-monitor/komari/database/accounts"
|
||||
"github.com/komari-monitor/komari/database/auditlog"
|
||||
d_notification "github.com/komari-monitor/komari/database/notification"
|
||||
"github.com/komari-monitor/komari/database/tasks"
|
||||
"github.com/komari-monitor/komari/internal/config"
|
||||
"github.com/komari-monitor/komari/internal/lifecycle"
|
||||
"github.com/komari-monitor/komari/internal/metricstore"
|
||||
"github.com/komari-monitor/komari/internal/plugin"
|
||||
"github.com/komari-monitor/komari/internal/scheduler"
|
||||
"github.com/komari-monitor/komari/utils/geoip"
|
||||
logger "github.com/komari-monitor/komari/utils/log"
|
||||
"github.com/komari-monitor/komari/utils/messageSender"
|
||||
"github.com/komari-monitor/komari/utils/notifier"
|
||||
"github.com/komari-monitor/komari/web/api"
|
||||
"github.com/komari-monitor/komari/web/oauth"
|
||||
recoveryweb "github.com/komari-monitor/komari/web/recovery"
|
||||
"github.com/komari-monitor/komari/web/router"
|
||||
"github.com/komari-monitor/komari/web/security"
|
||||
)
|
||||
|
||||
// ErrRestartRequested is returned after a clean shutdown when a configuration
|
||||
// change requires the next startup to enter a restricted guide.
|
||||
var ErrRestartRequested = errors.New("server restart requested")
|
||||
|
||||
const (
|
||||
// Give in-flight HTTP requests time to finish before the listener closes.
|
||||
httpShutdownTimeout = 10 * time.Second
|
||||
// Keep an independent budget for report flushing and store teardown. Reusing
|
||||
// the HTTP deadline here can skip queued metric writes after a slow request.
|
||||
resourceCleanupTimeout = 30 * time.Second
|
||||
)
|
||||
|
||||
// StartBackground starts scheduled work after all stores are ready.
|
||||
func (a *App) StartBackground() error {
|
||||
registerScheduledWork()
|
||||
a.addCleanup("scheduler", func(context.Context) error {
|
||||
scheduler.StopAll()
|
||||
return nil
|
||||
})
|
||||
return nil
|
||||
}
|
||||
|
||||
func (a *App) registerReloadHandlers(cors *security.CorsController) {
|
||||
a.reload.Register("oauth-provider", func(event config.ConfigEvent) {
|
||||
if ok, providerName := config.IsChangedT[string](event, config.OAuthProviderKey); ok {
|
||||
if providerName == "" || providerName == "none" {
|
||||
providerName = "github"
|
||||
}
|
||||
oidcProvider, err := database.GetOidcConfigByName(providerName)
|
||||
if err != nil {
|
||||
logger.Errorf("server", "Failed to get OIDC provider config: %v", err)
|
||||
return
|
||||
}
|
||||
logger.Infof("server", "Using %s as OIDC provider", oidcProvider.Name)
|
||||
if err := oauth.LoadProvider(oidcProvider.Name, oidcProvider.Addition); err != nil {
|
||||
auditlog.EventLog("error", fmt.Sprintf("Failed to load OIDC provider: %v", err))
|
||||
}
|
||||
}
|
||||
})
|
||||
a.reload.Register("geoip-provider", func(event config.ConfigEvent) {
|
||||
if event.IsChanged(config.GeoIpProviderKey) {
|
||||
go geoip.InitGeoIp()
|
||||
}
|
||||
})
|
||||
a.reload.Register("message-sender", func(event config.ConfigEvent) {
|
||||
if event.IsChanged(config.NotificationMethodKey) {
|
||||
go messageSender.Initialize()
|
||||
}
|
||||
})
|
||||
a.reload.Register("cors", func(event config.ConfigEvent) { cors.Update(event) })
|
||||
}
|
||||
|
||||
// BuildRouter constructs the normal application router and starts reloads.
|
||||
func (a *App) BuildRouter() error {
|
||||
r := gin.New()
|
||||
r.Use(logger.GinLogger(), logger.GinRecovery())
|
||||
cors := security.NewCorsController(a.settings.CorsOriginCheckEnabled, a.settings.CorsAllowedOrigins)
|
||||
r.Use(cors.Middleware(), api.IdentityMiddleware(), api.PrivateSiteMiddleware(), noStoreAPIResponses())
|
||||
|
||||
// The recovery UI belongs only to its temporary restricted listener.
|
||||
r.GET(recoveryweb.PagePath, func(c *gin.Context) {
|
||||
c.Redirect(http.StatusTemporaryRedirect, "/")
|
||||
})
|
||||
router.Register(r)
|
||||
|
||||
// Plugins are loaded after the router exists so server.route can bind
|
||||
// routes; a failed plugin only disables itself and is logged.
|
||||
plugin.Init(r)
|
||||
if err := plugin.LoadAll(); err != nil {
|
||||
logger.ErrorArgs("server", "Failed to load some plugins:", err)
|
||||
}
|
||||
a.addCleanup("plugins", func(context.Context) error { return plugin.CloseAll() })
|
||||
|
||||
a.registerReloadHandlers(cors)
|
||||
a.reload.Start()
|
||||
a.engine = r
|
||||
return nil
|
||||
}
|
||||
|
||||
// Run starts the normal HTTP server and blocks until shutdown or fatal error.
|
||||
func (a *App) Run() error {
|
||||
// The HTML injector runs outside the hook chain so it sees the final
|
||||
// response: plugin hooks can still rewrite the body, then the registered
|
||||
// head/body fragments are embedded into every text/html page.
|
||||
a.server = &http.Server{Addr: a.listenAddr, Handler: plugin.HTMLInjectHandler(plugin.WrapHandler(a.engine))}
|
||||
serverErr := make(chan error, 1)
|
||||
logger.Infof("server", "Starting server on %s ...", a.listenAddr)
|
||||
go func() {
|
||||
if err := a.server.ListenAndServe(); err != nil && err != http.ErrServerClosed {
|
||||
serverErr <- err
|
||||
}
|
||||
}()
|
||||
|
||||
quit := make(chan os.Signal, 1)
|
||||
signal.Notify(quit, os.Interrupt, syscall.SIGTERM)
|
||||
defer signal.Stop(quit)
|
||||
select {
|
||||
case err := <-serverErr:
|
||||
a.onFatal(err)
|
||||
return fmt.Errorf("listen: %w", err)
|
||||
case reason := <-lifecycle.RestartRequests():
|
||||
logger.Infof("server", "Restarting service for %s", reason)
|
||||
if err := a.Shutdown(); err != nil {
|
||||
logger.Errorf("server", "Cleanup before restart failed: %v", err)
|
||||
}
|
||||
return fmt.Errorf("%w: %s", ErrRestartRequested, reason)
|
||||
case <-quit:
|
||||
return a.Shutdown()
|
||||
}
|
||||
}
|
||||
|
||||
// Shutdown stops HTTP first, then releases registered resources in LIFO order.
|
||||
func (a *App) Shutdown() error {
|
||||
if a.dbReady {
|
||||
auditlog.Log("", "", "server is shutting down", "info")
|
||||
}
|
||||
httpCtx, cancelHTTP := context.WithTimeout(context.Background(), httpShutdownTimeout)
|
||||
defer cancelHTTP()
|
||||
if a.server != nil {
|
||||
if err := a.server.Shutdown(httpCtx); err != nil {
|
||||
logger.Infof("server", "HTTP server forced to shutdown: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
cleanupCtx, cancelCleanup := context.WithTimeout(context.Background(), resourceCleanupTimeout)
|
||||
defer cancelCleanup()
|
||||
return a.runCleanups(cleanupCtx)
|
||||
}
|
||||
|
||||
func (a *App) onFatal(err error) {
|
||||
if a.dbReady {
|
||||
auditlog.Log("", "", "server encountered a fatal error: "+err.Error(), "error")
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), resourceCleanupTimeout)
|
||||
defer cancel()
|
||||
if cleanupErr := a.runCleanups(ctx); cleanupErr != nil {
|
||||
logger.Errorf("server", "Cleanup after fatal server error failed: %v", cleanupErr)
|
||||
}
|
||||
}
|
||||
|
||||
func (a *App) runCleanups(ctx context.Context) error {
|
||||
var cleanupErrors []error
|
||||
for i := len(a.cleanups) - 1; i >= 0; i-- {
|
||||
cleanup := a.cleanups[i]
|
||||
if err := cleanup.fn(ctx); err != nil {
|
||||
logger.Errorf("server", "cleanup %q failed: %v", cleanup.name, err)
|
||||
cleanupErrors = append(cleanupErrors, fmt.Errorf("cleanup %q: %w", cleanup.name, err))
|
||||
}
|
||||
}
|
||||
return errors.Join(cleanupErrors...)
|
||||
}
|
||||
|
||||
func registerScheduledWork() {
|
||||
if err := tasks.ReloadPingSchedule(); err != nil {
|
||||
logger.ErrorArgs("server", "Failed to reload ping schedule:", err)
|
||||
}
|
||||
if err := d_notification.ReloadLoadNotificationSchedule(); err != nil {
|
||||
logger.ErrorArgs("server", "Failed to reload load notification schedule:", err)
|
||||
}
|
||||
if err := scheduler.AddFunc("records:cleanup", "@every 30m", cleanupScheduledData); err != nil {
|
||||
logger.ErrorArgs("server", "Failed to add cleanup scheduled task:", err)
|
||||
}
|
||||
if err := scheduler.AddContextFunc("metrics:compact", "@every 5m", true, compactMetricStore); err != nil {
|
||||
logger.ErrorArgs("server", "Failed to add metric compact scheduled task:", err)
|
||||
}
|
||||
if err := scheduler.AddContextFunc("metrics:retention", "@every 1h", true, cleanupMetricStore); err != nil {
|
||||
logger.ErrorArgs("server", "Failed to add metric retention scheduled task:", err)
|
||||
}
|
||||
if err := scheduler.AddFunc("notifier:traffic", "@every 1m", notifier.CheckTraffic); err != nil {
|
||||
logger.ErrorArgs("server", "Failed to add traffic notification task:", err)
|
||||
}
|
||||
if err := scheduler.AddFunc("notifier:expire", "0 0 9 * * *", notifier.CheckExpireScheduledWork); err != nil {
|
||||
logger.ErrorArgs("server", "Failed to add expire notification task:", err)
|
||||
}
|
||||
notifier.InitTrafficReportSchedule()
|
||||
}
|
||||
|
||||
const taskResultRetentionDays = 30
|
||||
|
||||
func cleanupScheduledData() {
|
||||
before := time.Now().UTC().Add(-24 * time.Hour * taskResultRetentionDays)
|
||||
if err := tasks.ClearTaskResultsByTimeBefore(before); err != nil {
|
||||
logger.Errorf("server", "Failed to clean expired task results: %v", err)
|
||||
}
|
||||
auditlog.RemoveOldLogs()
|
||||
accounts.RemoveExpiredSessions()
|
||||
}
|
||||
|
||||
func compactMetricStore(ctx context.Context) {
|
||||
compactCtx, cancel := context.WithTimeout(ctx, 15*time.Minute)
|
||||
defer cancel()
|
||||
written, err := metricstore.Compact(compactCtx, time.Now().UTC())
|
||||
if errors.Is(err, metricstore.ErrCompactInProgress) {
|
||||
return
|
||||
}
|
||||
if err != nil {
|
||||
logger.Errorf("server", "Failed to compact metric store after writing %d rollup buckets: %v", written, err)
|
||||
return
|
||||
}
|
||||
if written > 0 {
|
||||
logger.Infof("server", "Metric store compacted %d rollup buckets", written)
|
||||
}
|
||||
}
|
||||
|
||||
func cleanupMetricStore(ctx context.Context) {
|
||||
cleanupCtx, cancel := context.WithTimeout(ctx, 15*time.Minute)
|
||||
defer cancel()
|
||||
deleted, err := metricstore.CleanupExpired(cleanupCtx, time.Now().UTC())
|
||||
if errors.Is(err, metricstore.ErrCompactInProgress) {
|
||||
return
|
||||
}
|
||||
if err != nil {
|
||||
logger.Errorf("server", "Failed to clean expired metric data after deleting %d rows: %v", deleted, err)
|
||||
return
|
||||
}
|
||||
if deleted > 0 {
|
||||
logger.Infof("server", "Metric retention cleanup deleted %d rows", deleted)
|
||||
}
|
||||
}
|
||||
Loaded 100 of 299 files, more files were not shown because too many files have changed in this diff.
Show more
Reference in new issue
Block a user