mirror of
https://github.com/komari-monitor/komari.git
synced 2026-11-07 06:40:13 +00:00
Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
be3daa42fc | ||
|
|
42b290f169 | ||
|
|
99fda6b023 | ||
|
|
7f128509e9 | ||
|
|
fee8c860d7 | ||
|
|
3c278230f4 | ||
|
|
d853a83ec6 | ||
|
|
a78ef878eb | ||
|
|
312e320c8f | ||
|
|
73c3c436a4 | ||
|
|
f42cfe7ddc | ||
|
|
0f6f2f4294 | ||
|
|
fc3febc3be | ||
|
|
5b90804a13 | ||
|
|
43547b947f | ||
|
|
e2a8284c03 | ||
|
|
c1178ebfe1 | ||
|
|
7baf2edf26 | ||
|
|
66b7691e01 | ||
|
|
7f368817e0 | ||
|
|
5fa59ab7de | ||
|
|
0c80f0f23a | ||
|
|
1eb3549ed9 | ||
|
|
5c73eba475 | ||
|
|
76794305c8 | ||
|
|
724dd3b666 | ||
|
|
c828653d20 | ||
|
|
ff358edd8b | ||
|
|
3b48339209 | ||
|
|
42ef993cd6 | ||
|
|
39c5f7bb69 | ||
|
|
9831df0af8 | ||
|
|
29eecf55f7 | ||
|
|
0660ee74d4 | ||
|
|
5ade710f32 | ||
|
|
855ff82f97 | ||
|
|
8910c2e9c7 | ||
|
|
b2fe9ab068 | ||
|
|
8a529b4eb0 | ||
|
|
46b4e37178 | ||
|
|
5ad791fe50 | ||
|
|
8911c08a5b | ||
|
|
7d95240bc1 | ||
|
|
7e0d542694 | ||
|
|
84130dfff5 | ||
|
|
e0094447a6 | ||
|
|
ab31fc69fe | ||
|
|
73071ff1e8 | ||
|
|
4ad190a7f4 | ||
|
|
6ddb143365 | ||
|
|
eb4ca83a13 | ||
|
|
43c80d238b | ||
|
|
416ea871be | ||
|
|
5c2bd4ec74 | ||
|
|
be6af33d85 | ||
|
|
d02d9347dc | ||
|
|
64998b527c | ||
|
|
9e5b6df196 | ||
|
|
332053d8ec | ||
|
|
b6fcdd396d | ||
|
|
92ec774249 | ||
|
|
404d97d846 | ||
|
|
0132d1cbd8 | ||
|
|
ff6daa6ce3 | ||
|
|
6a5f3ef04d | ||
|
|
da047cf612 | ||
|
|
f0877d6d32 | ||
|
|
f2c46dbfb1 | ||
|
|
5f50d1ba00 | ||
|
|
7e19909dd6 | ||
|
|
9db4daac10 | ||
|
|
0a4cd6bd8c | ||
|
|
20c865ed20 | ||
|
|
e740bf2797 | ||
|
|
074958fad7 | ||
|
|
11d3ba8236 | ||
|
|
ce41745edb | ||
|
|
036121c148 | ||
|
|
dd0539392b | ||
|
|
50ba8310ca | ||
|
|
50fe286528 | ||
|
|
144efdc8ad | ||
|
|
e446bee900 | ||
|
|
a980befd60 | ||
|
|
2e0c598dae | ||
|
|
771a192a3d | ||
|
|
95cc65e28d | ||
|
|
b6e36ee0c4 | ||
|
|
25f3e1d2bd | ||
|
|
091eafa743 | ||
|
|
cbe665591f | ||
|
|
0abd9ff9bc | ||
|
|
618ced3b8f | ||
|
|
f7fd459723 |
No files matched your search
@@ -0,0 +1,55 @@
|
||||
name: Build frontend theme
|
||||
description: Clone komari-web, build the default theme, and optionally upload it as an artifact.
|
||||
|
||||
inputs:
|
||||
node-version:
|
||||
description: Node.js version used to build komari-web.
|
||||
default: "23"
|
||||
repository:
|
||||
description: komari-web repository URL.
|
||||
default: https://github.com/komari-monitor/komari-web
|
||||
artifact-name:
|
||||
description: Optional artifact name for the packed frontend theme.
|
||||
default: ""
|
||||
|
||||
runs:
|
||||
using: composite
|
||||
steps:
|
||||
- name: Setup Node.js
|
||||
uses: actions/setup-node@v4
|
||||
with:
|
||||
node-version: ${{ inputs.node-version }}
|
||||
|
||||
- name: Clone and build frontend
|
||||
shell: bash
|
||||
run: |
|
||||
set -euxo pipefail
|
||||
rm -rf komari-web
|
||||
git clone --depth=1 "${{ inputs.repository }}" komari-web
|
||||
cd komari-web
|
||||
npm install
|
||||
npm run build
|
||||
cd ..
|
||||
|
||||
mkdir -p web/public/defaultTheme/dist
|
||||
rm -rf web/public/defaultTheme/dist/*
|
||||
cp -r komari-web/dist/* web/public/defaultTheme/dist/
|
||||
cp -f komari-web/komari-theme.json web/public/defaultTheme/
|
||||
if [ -f komari-web/preview.png ]; then cp -f komari-web/preview.png web/public/defaultTheme/; fi
|
||||
if [ -f komari-web/perview.png ]; then cp -f komari-web/perview.png web/public/defaultTheme/; fi
|
||||
if [ -f web/public/defaultTheme/preview.png ] && [ ! -f web/public/defaultTheme/perview.png ]; then cp -f web/public/defaultTheme/preview.png web/public/defaultTheme/perview.png; fi
|
||||
if [ -f web/public/defaultTheme/perview.png ] && [ ! -f web/public/defaultTheme/preview.png ]; then cp -f web/public/defaultTheme/perview.png web/public/defaultTheme/preview.png; fi
|
||||
|
||||
- name: Pack frontend artifact
|
||||
if: ${{ inputs.artifact-name != '' }}
|
||||
shell: bash
|
||||
run: |
|
||||
set -euxo pipefail
|
||||
tar -czf "$RUNNER_TEMP/${{ inputs.artifact-name }}.tar.gz" -C web/public/defaultTheme .
|
||||
|
||||
- name: Upload frontend artifact
|
||||
if: ${{ inputs.artifact-name != '' }}
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: ${{ inputs.artifact-name }}
|
||||
path: ${{ runner.temp }}/${{ inputs.artifact-name }}.tar.gz
|
||||
@@ -0,0 +1,24 @@
|
||||
name: Restore frontend theme
|
||||
description: Download and unpack a previously built frontend theme artifact.
|
||||
|
||||
inputs:
|
||||
artifact-name:
|
||||
description: Frontend theme artifact name.
|
||||
default: komari-frontend
|
||||
|
||||
runs:
|
||||
using: composite
|
||||
steps:
|
||||
- name: Download frontend artifact
|
||||
uses: actions/download-artifact@v4
|
||||
with:
|
||||
name: ${{ inputs.artifact-name }}
|
||||
path: ${{ runner.temp }}/frontend-artifact
|
||||
|
||||
- name: Extract frontend artifact
|
||||
shell: bash
|
||||
run: |
|
||||
set -euxo pipefail
|
||||
mkdir -p web/public/defaultTheme
|
||||
rm -rf web/public/defaultTheme/*
|
||||
tar -xzf "$RUNNER_TEMP/frontend-artifact/${{ inputs.artifact-name }}.tar.gz" -C web/public/defaultTheme
|
||||
@@ -0,0 +1,31 @@
|
||||
name: Setup Zig
|
||||
description: Restore or install the Zig toolchain used for CGO cross-compilation.
|
||||
|
||||
inputs:
|
||||
zig-version:
|
||||
description: Zig version to install.
|
||||
default: 0.14.1
|
||||
|
||||
runs:
|
||||
using: composite
|
||||
steps:
|
||||
- name: Restore Zig cache
|
||||
id: zig-cache
|
||||
uses: actions/cache@v4
|
||||
with:
|
||||
path: ~/.cache/komari-zig/zig-x86_64-linux-${{ inputs.zig-version }}
|
||||
key: zig-${{ runner.os }}-x86_64-${{ inputs.zig-version }}
|
||||
|
||||
- name: Download Zig
|
||||
if: ${{ steps.zig-cache.outputs.cache-hit != 'true' }}
|
||||
shell: bash
|
||||
run: |
|
||||
set -euxo pipefail
|
||||
mkdir -p "$HOME/.cache/komari-zig"
|
||||
curl -fsSL "https://ziglang.org/download/${{ inputs.zig-version }}/zig-x86_64-linux-${{ inputs.zig-version }}.tar.xz" -o "$RUNNER_TEMP/zig.tar.xz"
|
||||
tar -C "$HOME/.cache/komari-zig" -Jxf "$RUNNER_TEMP/zig.tar.xz"
|
||||
|
||||
- name: Add Zig to PATH
|
||||
shell: bash
|
||||
run: |
|
||||
echo "$HOME/.cache/komari-zig/zig-x86_64-linux-${{ inputs.zig-version }}" >> "$GITHUB_PATH"
|
||||
+21
-31
@@ -1,16 +1,26 @@
|
||||
name: Build Binaries on Main Push and PR
|
||||
name: Build Binaries on PR
|
||||
|
||||
on:
|
||||
push:
|
||||
branches:
|
||||
- main
|
||||
pull_request:
|
||||
branches:
|
||||
- main
|
||||
workflow_dispatch:
|
||||
|
||||
jobs:
|
||||
build-frontend:
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Build frontend artifact
|
||||
uses: ./.github/actions/build-frontend
|
||||
with:
|
||||
artifact-name: komari-frontend
|
||||
|
||||
build-binaries:
|
||||
needs: build-frontend
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
strategy:
|
||||
@@ -49,6 +59,10 @@ jobs:
|
||||
goarch: riscv64
|
||||
zig_target_triple: riscv64-linux-musl
|
||||
binary_extension: ""
|
||||
- goos: linux
|
||||
goarch: loong64
|
||||
zig_target_triple: loongarch64-linux-musl
|
||||
binary_extension: ""
|
||||
|
||||
exclude:
|
||||
- goos: windows
|
||||
@@ -58,27 +72,8 @@ jobs:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Setup Node.js
|
||||
uses: actions/setup-node@v4
|
||||
with:
|
||||
node-version: "23"
|
||||
|
||||
- name: Clone and build frontend
|
||||
run: |
|
||||
git clone https://github.com/komari-monitor/komari-web komari-web
|
||||
cd komari-web
|
||||
npm install
|
||||
npm run build
|
||||
cd ..
|
||||
mkdir -p web/public/defaultTheme/dist
|
||||
rm -rf web/public/defaultTheme/dist/*
|
||||
cp -r komari-web/dist/* web/public/defaultTheme/dist/
|
||||
cp -f komari-web/komari-theme.json web/public/defaultTheme/
|
||||
if [ -f komari-web/preview.png ]; then cp -f komari-web/preview.png web/public/defaultTheme/; fi
|
||||
if [ -f komari-web/perview.png ]; then cp -f komari-web/perview.png web/public/defaultTheme/; fi
|
||||
# Compatibility: ensure both filenames exist if only one is present
|
||||
if [ -f web/public/defaultTheme/preview.png ] && [ ! -f web/public/defaultTheme/perview.png ]; then cp -f web/public/defaultTheme/preview.png web/public/defaultTheme/perview.png; fi
|
||||
if [ -f web/public/defaultTheme/perview.png ] && [ ! -f web/public/defaultTheme/preview.png ]; then cp -f web/public/defaultTheme/perview.png web/public/defaultTheme/preview.png; fi
|
||||
- name: Restore frontend artifact
|
||||
uses: ./.github/actions/restore-frontend
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v5
|
||||
@@ -86,12 +81,7 @@ jobs:
|
||||
go-version: "1.23"
|
||||
|
||||
- name: Install Zig cross-compiler toolchain
|
||||
run: |
|
||||
sudo apt-get update # Update apt package lists
|
||||
sudo apt-get install -y build-essential curl
|
||||
wget https://ziglang.org/download/0.14.1/zig-x86_64-linux-0.14.1.tar.xz
|
||||
sudo tar -C /usr/local -Jxf zig-x86_64-linux-0.14.1.tar.xz
|
||||
echo "PATH=$PATH:/usr/local/zig-x86_64-linux-0.14.1" >> $GITHUB_ENV
|
||||
uses: ./.github/actions/setup-zig
|
||||
|
||||
- name: Build binary
|
||||
env:
|
||||
|
||||
@@ -20,21 +20,7 @@ jobs:
|
||||
# echo "<html><body><h1>This is komari development server.</h1></body></html>" > web/public/defaultTheme/dist/index.html
|
||||
|
||||
- name: Clone and build frontend
|
||||
run: |
|
||||
git clone https://github.com/komari-monitor/komari-web komari-web
|
||||
cd komari-web
|
||||
npm install
|
||||
npm run build
|
||||
cd ..
|
||||
mkdir -p web/public/defaultTheme/dist
|
||||
rm -rf web/public/defaultTheme/dist/*
|
||||
cp -r komari-web/dist/* web/public/defaultTheme/dist/
|
||||
cp -f komari-web/komari-theme.json web/public/defaultTheme/
|
||||
if [ -f komari-web/preview.png ]; then cp -f komari-web/preview.png web/public/defaultTheme/; fi
|
||||
if [ -f komari-web/perview.png ]; then cp -f komari-web/perview.png web/public/defaultTheme/; fi
|
||||
# Compatibility: ensure both filenames exist if only one is present
|
||||
if [ -f web/public/defaultTheme/preview.png ] && [ ! -f web/public/defaultTheme/perview.png ]; then cp -f web/public/defaultTheme/preview.png web/public/defaultTheme/perview.png; fi
|
||||
if [ -f web/public/defaultTheme/perview.png ] && [ ! -f web/public/defaultTheme/preview.png ]; then cp -f web/public/defaultTheme/perview.png web/public/defaultTheme/preview.png; fi
|
||||
uses: ./.github/actions/build-frontend
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v5
|
||||
|
||||
@@ -11,37 +11,28 @@ env:
|
||||
IMAGE_NAME: ${{ github.repository }}
|
||||
|
||||
jobs:
|
||||
build-and-push-multi-arch:
|
||||
build-frontend:
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
contents: read
|
||||
packages: write
|
||||
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Setup Node.js
|
||||
uses: actions/setup-node@v4
|
||||
- name: Build frontend artifact
|
||||
uses: ./.github/actions/build-frontend
|
||||
with:
|
||||
node-version: "23"
|
||||
artifact-name: komari-frontend
|
||||
|
||||
- name: Clone and build frontend
|
||||
run: |
|
||||
git clone https://github.com/komari-monitor/komari-web komari-web
|
||||
cd komari-web
|
||||
npm install
|
||||
npm run build
|
||||
cd ..
|
||||
mkdir -p web/public/defaultTheme/dist
|
||||
rm -rf web/public/defaultTheme/dist/*
|
||||
cp -r komari-web/dist/* web/public/defaultTheme/dist/
|
||||
cp -f komari-web/komari-theme.json web/public/defaultTheme/
|
||||
if [ -f komari-web/preview.png ]; then cp -f komari-web/preview.png web/public/defaultTheme/; fi
|
||||
if [ -f komari-web/perview.png ]; then cp -f komari-web/perview.png web/public/defaultTheme/; fi
|
||||
# Compatibility: ensure both filenames exist if only one is present
|
||||
if [ -f web/public/defaultTheme/preview.png ] && [ ! -f web/public/defaultTheme/perview.png ]; then cp -f web/public/defaultTheme/preview.png web/public/defaultTheme/perview.png; fi
|
||||
if [ -f web/public/defaultTheme/perview.png ] && [ ! -f web/public/defaultTheme/preview.png ]; then cp -f web/public/defaultTheme/perview.png web/public/defaultTheme/preview.png; fi
|
||||
build-docker-binaries:
|
||||
needs: build-frontend
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Restore frontend artifact
|
||||
uses: ./.github/actions/restore-frontend
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v5
|
||||
@@ -49,10 +40,7 @@ jobs:
|
||||
go-version: "1.23"
|
||||
|
||||
- name: Install Zig
|
||||
run: |
|
||||
wget https://ziglang.org/download/0.14.1/zig-x86_64-linux-0.14.1.tar.xz
|
||||
sudo tar -C /usr/local -Jxf zig-x86_64-linux-0.14.1.tar.xz
|
||||
echo "PATH=$PATH:/usr/local/zig-x86_64-linux-0.14.1" >> $GITHUB_ENV
|
||||
uses: ./.github/actions/setup-zig
|
||||
|
||||
- name: Build static binaries for multi-arch
|
||||
env:
|
||||
@@ -70,6 +58,29 @@ jobs:
|
||||
echo "Building for linux/arm64..."
|
||||
GOARCH=arm64 CC="zig cc -target aarch64-linux-musl" go build -trimpath -ldflags="$LDFLAGS" -o komari-linux-arm64
|
||||
|
||||
- name: Upload Docker binaries
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: komari-docker-binaries
|
||||
path: komari-linux-*
|
||||
|
||||
build-and-push-multi-arch:
|
||||
needs: build-docker-binaries
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
contents: read
|
||||
packages: write
|
||||
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Download Docker binaries
|
||||
uses: actions/download-artifact@v4
|
||||
with:
|
||||
name: komari-docker-binaries
|
||||
path: .
|
||||
|
||||
- name: Set up QEMU
|
||||
uses: docker/setup-qemu-action@v3
|
||||
|
||||
|
||||
@@ -6,8 +6,12 @@ on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
tag:
|
||||
description: "Release tag to regenerate notes for."
|
||||
required: true
|
||||
description: "Release tag to regenerate notes for. Leave empty to use the latest release."
|
||||
required: false
|
||||
type: string
|
||||
base_ref:
|
||||
description: "Base ref or SHA for the release comparison. Snapshot builds pass the previous main SHA."
|
||||
required: false
|
||||
type: string
|
||||
|
||||
permissions:
|
||||
@@ -16,46 +20,588 @@ permissions:
|
||||
jobs:
|
||||
update-release-notes:
|
||||
runs-on: ubuntu-latest
|
||||
if: ${{ github.event_name != 'release' || !startsWith(github.event.release.tag_name, 'Snapshot-') }}
|
||||
|
||||
steps:
|
||||
- name: Generate and apply release notes
|
||||
uses: actions/github-script@v9
|
||||
env:
|
||||
OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
|
||||
OPENAI_BASE_URL: ${{ secrets.OPENAI_BASE_URL }}
|
||||
OPENAI_MODEL: ${{ vars.OPENAI_MODEL }}
|
||||
with:
|
||||
script: |
|
||||
const tag = context.payload.release?.tag_name || core.getInput("tag");
|
||||
if (!tag) {
|
||||
core.setFailed("No release tag was provided.");
|
||||
return;
|
||||
const { owner, repo } = context.repo;
|
||||
const webRepo = { owner: "komari-monitor", repo: "komari-web" };
|
||||
|
||||
const AI_CONTEXT_CHAR_LIMIT = 120000;
|
||||
const AI_MAX_COMMITS = 160;
|
||||
const AI_MAX_CHANGED_FILES = 120;
|
||||
const AI_MAX_PULL_REQUESTS = 80;
|
||||
const AI_MAX_PATCH_CHARS_PER_FILE = 2200;
|
||||
const OPENAI_TIMEOUT_MS = 120000;
|
||||
|
||||
const clean = (value) => (typeof value === "string" ? value.trim() : "");
|
||||
const isSnapshotTag = (tag) => /^Snapshot-/i.test(clean(tag));
|
||||
const releaseTimestamp = (release) => Date.parse(release.published_at || release.created_at || "");
|
||||
const isStableRelease = (release) => !release.draft && !release.prerelease && !isSnapshotTag(release.tag_name);
|
||||
|
||||
function limitText(value, maxLength) {
|
||||
const text = clean(value);
|
||||
if (text.length <= maxLength) {
|
||||
return text;
|
||||
}
|
||||
|
||||
return `${text.slice(0, maxLength)}\n...[truncated ${text.length - maxLength} chars]`;
|
||||
}
|
||||
|
||||
const { owner, repo } = context.repo;
|
||||
const release = context.payload.release?.tag_name === tag
|
||||
? context.payload.release
|
||||
: (await github.rest.repos.getReleaseByTag({ owner, repo, tag })).data;
|
||||
function appendWithinLimit(parts, value, maxLength) {
|
||||
const text = clean(value);
|
||||
if (!text) {
|
||||
return false;
|
||||
}
|
||||
|
||||
const generated = await github.request(
|
||||
"POST /repos/{owner}/{repo}/releases/generate-notes",
|
||||
{
|
||||
const currentLength = parts.reduce((total, part) => total + part.length + 2, 0);
|
||||
if (currentLength + text.length + 2 <= maxLength) {
|
||||
parts.push(text);
|
||||
return true;
|
||||
}
|
||||
|
||||
const remaining = maxLength - currentLength - 64;
|
||||
if (remaining > 500) {
|
||||
parts.push(`${text.slice(0, remaining)}\n...[context truncated]`);
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
async function resolveRelease() {
|
||||
const requestedTag = clean(context.payload.release?.tag_name)
|
||||
|| clean(context.payload.client_payload?.tag)
|
||||
|| clean(context.payload.inputs?.tag);
|
||||
|
||||
if (requestedTag) {
|
||||
const release = context.payload.release?.tag_name === requestedTag
|
||||
? context.payload.release
|
||||
: (await github.rest.repos.getReleaseByTag({ owner, repo, tag: requestedTag })).data;
|
||||
return { tag: requestedTag, release };
|
||||
}
|
||||
|
||||
const release = (await github.rest.repos.getLatestRelease({ owner, repo })).data;
|
||||
return { tag: release.tag_name, release };
|
||||
}
|
||||
|
||||
async function getPreviousReleaseTag(tag, release) {
|
||||
const releases = await github.paginate(github.rest.repos.listReleases, {
|
||||
owner,
|
||||
repo,
|
||||
per_page: 100,
|
||||
});
|
||||
|
||||
const currentTime = releaseTimestamp(release);
|
||||
const currentIsSnapshot = isSnapshotTag(tag);
|
||||
const currentIsStable = isStableRelease(release);
|
||||
const candidates = releases
|
||||
.filter((item) => !item.draft && item.tag_name !== tag)
|
||||
.filter((item) => {
|
||||
if (!Number.isFinite(currentTime)) {
|
||||
return true;
|
||||
}
|
||||
|
||||
const itemTime = releaseTimestamp(item);
|
||||
return Number.isFinite(itemTime) && itemTime < currentTime;
|
||||
})
|
||||
.filter((item) => {
|
||||
if (currentIsSnapshot || currentIsStable) {
|
||||
return isStableRelease(item);
|
||||
}
|
||||
|
||||
return item.prerelease && !isSnapshotTag(item.tag_name);
|
||||
})
|
||||
.sort((left, right) => releaseTimestamp(right) - releaseTimestamp(left));
|
||||
|
||||
return candidates[0]?.tag_name || "";
|
||||
}
|
||||
|
||||
async function resolveComparisonBase(tag, release) {
|
||||
const requestedBaseRef = clean(context.payload.client_payload?.base_ref)
|
||||
|| clean(context.payload.inputs?.base_ref);
|
||||
|
||||
if (requestedBaseRef) {
|
||||
return {
|
||||
ref: requestedBaseRef,
|
||||
previousTag: "",
|
||||
source: "requested base ref",
|
||||
};
|
||||
}
|
||||
|
||||
const previousTag = await getPreviousReleaseTag(tag, release);
|
||||
if (previousTag) {
|
||||
return {
|
||||
ref: previousTag,
|
||||
previousTag,
|
||||
source: isSnapshotTag(tag) ? "previous stable release" : "previous release in the same channel",
|
||||
};
|
||||
}
|
||||
|
||||
if (isSnapshotTag(tag)) {
|
||||
try {
|
||||
const commit = (await github.rest.repos.getCommit({
|
||||
owner,
|
||||
repo,
|
||||
ref: release.target_commitish || tag,
|
||||
})).data;
|
||||
const parentSha = commit.parents?.[0]?.sha || "";
|
||||
if (parentSha) {
|
||||
return {
|
||||
ref: parentSha,
|
||||
previousTag: "",
|
||||
source: "snapshot target parent commit",
|
||||
};
|
||||
}
|
||||
} catch (error) {
|
||||
core.warning(`Failed to resolve parent commit for ${tag}: ${error.message}`);
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
ref: "",
|
||||
previousTag: "",
|
||||
source: "none",
|
||||
};
|
||||
}
|
||||
|
||||
async function getGeneratedNotes(tag, release, previousTag) {
|
||||
const params = {
|
||||
owner,
|
||||
repo,
|
||||
tag_name: tag,
|
||||
target_commitish: release.target_commitish || undefined,
|
||||
},
|
||||
);
|
||||
};
|
||||
|
||||
const body = generated.data.body?.trim();
|
||||
if (!body) {
|
||||
core.setFailed(`GitHub returned empty release notes for ${tag}.`);
|
||||
if (previousTag) {
|
||||
params.previous_tag_name = previousTag;
|
||||
}
|
||||
|
||||
const generated = await github.request(
|
||||
"POST /repos/{owner}/{repo}/releases/generate-notes",
|
||||
params,
|
||||
);
|
||||
|
||||
return clean(generated.data.body);
|
||||
}
|
||||
|
||||
async function getComparison(baseRef, tag) {
|
||||
if (!baseRef) {
|
||||
return { commits: [], files: [], htmlUrl: "" };
|
||||
}
|
||||
|
||||
const commits = [];
|
||||
let files = [];
|
||||
let htmlUrl = "";
|
||||
|
||||
for (let page = 1; page <= 20; page += 1) {
|
||||
const response = await github.rest.repos.compareCommitsWithBasehead({
|
||||
owner,
|
||||
repo,
|
||||
basehead: `${baseRef}...${tag}`,
|
||||
per_page: 100,
|
||||
page,
|
||||
});
|
||||
|
||||
if (page === 1) {
|
||||
files = response.data.files || [];
|
||||
htmlUrl = response.data.html_url || "";
|
||||
}
|
||||
|
||||
commits.push(...response.data.commits);
|
||||
|
||||
if (!response.headers.link?.includes('rel="next"')) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
return { commits, files, htmlUrl };
|
||||
}
|
||||
|
||||
function renderCommitLine(commit) {
|
||||
const subject = clean(commit.commit.message.split("\n")[0]) || "(no subject)";
|
||||
const shortSha = commit.sha.slice(0, 7);
|
||||
const author = commit.author?.login
|
||||
? ` @${commit.author.login}`
|
||||
: (commit.commit.author?.name ? ` ${commit.commit.author.name}` : "");
|
||||
|
||||
return `- ${subject} ([${shortSha}](${commit.html_url}))${author}`;
|
||||
}
|
||||
|
||||
function buildWhatsChangedSection(baseRef, tag, comparison) {
|
||||
if (!baseRef) {
|
||||
return "## What's Changed\n\nNo comparison base was found for this release.";
|
||||
}
|
||||
|
||||
if (comparison.commits.length === 0) {
|
||||
return `## What's Changed\n\nNo commits found between \`${baseRef}\` and \`${tag}\`.`;
|
||||
}
|
||||
|
||||
const lines = ["## What's Changed", "", ...comparison.commits.map(renderCommitLine)];
|
||||
if (comparison.htmlUrl) {
|
||||
lines.push("", `**Full Changelog**: ${comparison.htmlUrl}`);
|
||||
}
|
||||
|
||||
return lines.join("\n");
|
||||
}
|
||||
|
||||
function appendCommitsToGeneratedNotes(generatedNotes, baseRef, tag, comparison) {
|
||||
const notes = clean(generatedNotes);
|
||||
if (!notes || comparison.commits.length === 0) {
|
||||
return notes || buildWhatsChangedSection(baseRef, tag, comparison);
|
||||
}
|
||||
|
||||
const lines = notes.split("\n");
|
||||
const fullChangelogIndex = lines.findIndex((line) => /^\*\*Full Changelog\*\*:/i.test(line.trim()));
|
||||
const nextTopLevelSectionIndex = lines.findIndex((line, index) => (
|
||||
index > 0
|
||||
&& /^##\s+/.test(line)
|
||||
&& !/^##\s+What's Changed\b/i.test(line)
|
||||
));
|
||||
const insertionIndex = nextTopLevelSectionIndex >= 0
|
||||
? nextTopLevelSectionIndex
|
||||
: (fullChangelogIndex >= 0 ? fullChangelogIndex : lines.length);
|
||||
const commitSection = [
|
||||
"### Commits",
|
||||
"",
|
||||
...comparison.commits.map(renderCommitLine),
|
||||
"",
|
||||
];
|
||||
|
||||
lines.splice(insertionIndex, 0, ...commitSection);
|
||||
return clean(lines.join("\n"));
|
||||
}
|
||||
|
||||
async function getPullRequestContext(commits) {
|
||||
const pullRequests = new Map();
|
||||
|
||||
for (const commit of commits.slice(0, AI_MAX_COMMITS)) {
|
||||
try {
|
||||
const response = await github.rest.repos.listPullRequestsAssociatedWithCommit({
|
||||
owner,
|
||||
repo,
|
||||
commit_sha: commit.sha,
|
||||
});
|
||||
|
||||
for (const pullRequest of response.data || []) {
|
||||
pullRequests.set(pullRequest.html_url, pullRequest);
|
||||
}
|
||||
} catch (error) {
|
||||
core.warning(`Failed to load pull requests for ${commit.sha.slice(0, 7)}: ${error.message}`);
|
||||
}
|
||||
|
||||
if (pullRequests.size >= AI_MAX_PULL_REQUESTS) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
return [...pullRequests.values()].slice(0, AI_MAX_PULL_REQUESTS);
|
||||
}
|
||||
|
||||
function buildChangedFilesContext(files) {
|
||||
if (!files.length) {
|
||||
return "No changed file metadata was returned by GitHub.";
|
||||
}
|
||||
|
||||
return files.slice(0, AI_MAX_CHANGED_FILES).map((file) => {
|
||||
const lines = [
|
||||
`File: ${file.filename}`,
|
||||
`Status: ${file.status}; additions: ${file.additions}; deletions: ${file.deletions}; changes: ${file.changes}`,
|
||||
];
|
||||
|
||||
if (file.patch) {
|
||||
lines.push("Patch excerpt:");
|
||||
lines.push(limitText(file.patch, AI_MAX_PATCH_CHARS_PER_FILE));
|
||||
}
|
||||
|
||||
return lines.join("\n");
|
||||
}).join("\n\n");
|
||||
}
|
||||
|
||||
function buildCommitContext(commits) {
|
||||
if (!commits.length) {
|
||||
return "No commits were returned by GitHub.";
|
||||
}
|
||||
|
||||
return commits.slice(0, AI_MAX_COMMITS).map((commit) => {
|
||||
const shortSha = commit.sha.slice(0, 7);
|
||||
const author = commit.author?.login || commit.commit.author?.name || "unknown";
|
||||
const message = limitText(commit.commit.message, 1800);
|
||||
return [
|
||||
`Commit: ${shortSha}`,
|
||||
`Author: ${author}`,
|
||||
`URL: ${commit.html_url}`,
|
||||
"Message:",
|
||||
message,
|
||||
].join("\n");
|
||||
}).join("\n\n");
|
||||
}
|
||||
|
||||
function buildPullRequestContext(pullRequests) {
|
||||
if (!pullRequests.length) {
|
||||
return "No associated pull requests were found for the compared commits.";
|
||||
}
|
||||
|
||||
return pullRequests.map((pullRequest) => {
|
||||
const labels = (pullRequest.labels || []).map((label) => label.name).filter(Boolean).join(", ") || "none";
|
||||
return [
|
||||
`PR: #${pullRequest.number} ${pullRequest.title}`,
|
||||
`URL: ${pullRequest.html_url}`,
|
||||
`Author: ${pullRequest.user?.login || "unknown"}`,
|
||||
`Labels: ${labels}`,
|
||||
"Body:",
|
||||
limitText(pullRequest.body, 1800) || "(empty)",
|
||||
].join("\n");
|
||||
}).join("\n\n");
|
||||
}
|
||||
|
||||
function buildAiContext({ tag, release, comparisonBase, whatsChangedSection, comparison, pullRequests, webReleaseSection }) {
|
||||
const sections = [];
|
||||
|
||||
appendWithinLimit(sections, [
|
||||
`Repository: ${owner}/${repo}`,
|
||||
`Release tag: ${tag}`,
|
||||
`Release title: ${release.name || tag}`,
|
||||
`Prerelease: ${release.prerelease ? "yes" : "no"}`,
|
||||
`Snapshot tag: ${isSnapshotTag(tag) ? "yes" : "no"}`,
|
||||
`Target commitish: ${release.target_commitish || "(unknown)"}`,
|
||||
`Comparison base: ${comparisonBase.ref || "(none)"}`,
|
||||
`Comparison base source: ${comparisonBase.source}`,
|
||||
`Compare URL: ${comparison.htmlUrl || "(unavailable)"}`,
|
||||
].join("\n"), AI_CONTEXT_CHAR_LIMIT);
|
||||
|
||||
appendWithinLimit(sections, `What's Changed section:\n${whatsChangedSection}`, AI_CONTEXT_CHAR_LIMIT);
|
||||
appendWithinLimit(sections, `Changed files and patch excerpts:\n${buildChangedFilesContext(comparison.files)}`, AI_CONTEXT_CHAR_LIMIT);
|
||||
appendWithinLimit(sections, `Associated pull requests:\n${buildPullRequestContext(pullRequests)}`, AI_CONTEXT_CHAR_LIMIT);
|
||||
appendWithinLimit(sections, `Commit details:\n${buildCommitContext(comparison.commits)}`, AI_CONTEXT_CHAR_LIMIT);
|
||||
|
||||
if (webReleaseSection) {
|
||||
appendWithinLimit(sections, `Komari Web release notes for the same tag:\n${webReleaseSection}`, AI_CONTEXT_CHAR_LIMIT);
|
||||
}
|
||||
|
||||
return sections.join("\n\n---\n\n");
|
||||
}
|
||||
|
||||
function buildChatCompletionsUrl(baseUrl) {
|
||||
const trimmed = clean(baseUrl).replace(/\/+$/, "");
|
||||
if (!trimmed) {
|
||||
return "";
|
||||
}
|
||||
|
||||
if (trimmed.endsWith("/chat/completions")) {
|
||||
return trimmed;
|
||||
}
|
||||
|
||||
if (trimmed.endsWith("/v1")) {
|
||||
return `${trimmed}/chat/completions`;
|
||||
}
|
||||
|
||||
return `${trimmed}/v1/chat/completions`;
|
||||
}
|
||||
|
||||
function normalizeReleaseSummary(markdown) {
|
||||
const summary = clean(markdown);
|
||||
if (!summary) {
|
||||
return "";
|
||||
}
|
||||
|
||||
const withoutAiHeading = clean(summary
|
||||
.replace(/^#{1,6}\s+AI Summary\b\s*/im, "")
|
||||
.replace(/^\*\*AI Summary\*\*\s*/im, ""));
|
||||
const whatsChangedIndex = withoutAiHeading.search(/^##\s+What's Changed\b/im);
|
||||
if (whatsChangedIndex >= 0) {
|
||||
return clean(withoutAiHeading.slice(0, whatsChangedIndex));
|
||||
}
|
||||
|
||||
return withoutAiHeading;
|
||||
}
|
||||
|
||||
async function callOpenAiChatCompletion(url, apiKey, model, messages) {
|
||||
const controller = new AbortController();
|
||||
const timeout = setTimeout(() => controller.abort(), OPENAI_TIMEOUT_MS);
|
||||
|
||||
try {
|
||||
const response = await fetch(url, {
|
||||
method: "POST",
|
||||
headers: {
|
||||
Authorization: `Bearer ${apiKey}`,
|
||||
"Content-Type": "application/json",
|
||||
},
|
||||
body: JSON.stringify({ model, messages, temperature: 0 }),
|
||||
signal: controller.signal,
|
||||
});
|
||||
|
||||
const responseText = await response.text();
|
||||
let responseJson = {};
|
||||
try {
|
||||
responseJson = responseText ? JSON.parse(responseText) : {};
|
||||
} catch {
|
||||
responseJson = {};
|
||||
}
|
||||
|
||||
if (!response.ok) {
|
||||
const message = clean(responseJson.error?.message) || limitText(responseText, 500);
|
||||
throw new Error(`OpenAI request failed with ${response.status}: ${message}`);
|
||||
}
|
||||
|
||||
return clean(responseJson.choices?.[0]?.message?.content);
|
||||
} finally {
|
||||
clearTimeout(timeout);
|
||||
}
|
||||
}
|
||||
|
||||
async function buildReleaseSummarySection({ tag, release, comparisonBase, whatsChangedSection, comparison, webReleaseSection }) {
|
||||
const apiKey = clean(process.env.OPENAI_API_KEY);
|
||||
const baseUrl = clean(process.env.OPENAI_BASE_URL);
|
||||
const model = clean(process.env.OPENAI_MODEL);
|
||||
|
||||
if (!apiKey || !baseUrl || !model) {
|
||||
core.warning("OPENAI_API_KEY, OPENAI_BASE_URL, or OPENAI_MODEL is missing. Skipping generated release summary.");
|
||||
return "";
|
||||
}
|
||||
|
||||
const pullRequests = await getPullRequestContext(comparison.commits);
|
||||
const aiContext = buildAiContext({
|
||||
tag,
|
||||
release,
|
||||
comparisonBase,
|
||||
whatsChangedSection,
|
||||
comparison,
|
||||
pullRequests,
|
||||
webReleaseSection,
|
||||
});
|
||||
|
||||
const messages = [
|
||||
{
|
||||
role: "system",
|
||||
content: [
|
||||
"You write accurate release notes for the Komari server project.",
|
||||
"Use only the supplied repository context.",
|
||||
"Include both Komari server changes and Komari Web frontend changes when frontend release notes are supplied.",
|
||||
"Prioritize user-visible behavior, compatibility or migration notes, operations/deployment changes, security fixes, and important bug fixes.",
|
||||
"Do not invent features or implementation details that are not supported by the context.",
|
||||
"Write concise Markdown in Simplified Chinese.",
|
||||
].join(" "),
|
||||
},
|
||||
{
|
||||
role: "user",
|
||||
content: [
|
||||
`Create the release summary section for release ${tag}.`,
|
||||
comparisonBase.ref
|
||||
? `Summarize only changes in the comparison ${comparisonBase.ref}...${tag}.`
|
||||
: "No comparison base was found, so summarize only the supplied release context.",
|
||||
"Return only Markdown for the summary section that appears before '## What's Changed'.",
|
||||
"Do not use the heading 'AI Summary'. Do not add a 'What's Changed' section.",
|
||||
"Use these top-level headings, in this order: '## 主要更新内容', '## Bug修复', and '## 其他'.",
|
||||
"Omit '## Bug修复' when there are no concrete bug fixes. Keep bullets short and specific.",
|
||||
"",
|
||||
aiContext,
|
||||
].join("\n"),
|
||||
},
|
||||
];
|
||||
|
||||
try {
|
||||
const url = buildChatCompletionsUrl(baseUrl);
|
||||
const summary = await callOpenAiChatCompletion(url, apiKey, model, messages);
|
||||
return normalizeReleaseSummary(summary);
|
||||
} catch (error) {
|
||||
core.warning(`Generated release summary failed: ${error.message}`);
|
||||
return "";
|
||||
}
|
||||
}
|
||||
|
||||
async function buildWebReleaseSection(tag) {
|
||||
const demoteMarkdownHeadings = (markdown) => markdown.replace(/^(#{1,5})(\s+)/gm, "#$1$2");
|
||||
|
||||
try {
|
||||
const webRelease = (await github.rest.repos.getReleaseByTag({
|
||||
owner: webRepo.owner,
|
||||
repo: webRepo.repo,
|
||||
tag,
|
||||
})).data;
|
||||
|
||||
const webBody = clean(webRelease.body);
|
||||
if (!webBody) {
|
||||
return `## Komari Web Release Notes\n\nKomari Web release [${tag}](${webRelease.html_url}) exists, but its release notes are empty.`;
|
||||
}
|
||||
|
||||
return [
|
||||
"## Komari Web Release Notes",
|
||||
"",
|
||||
`Source: [komari-web ${tag}](${webRelease.html_url})`,
|
||||
"",
|
||||
demoteMarkdownHeadings(webBody),
|
||||
].join("\n");
|
||||
} catch (error) {
|
||||
if (error.status === 404) {
|
||||
core.info(`Komari Web release ${tag} does not exist yet. Skipping web release notes for now.`);
|
||||
return "";
|
||||
}
|
||||
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
const { tag, release } = await resolveRelease();
|
||||
if (!tag) {
|
||||
core.setFailed("No release tag was provided or found.");
|
||||
return;
|
||||
}
|
||||
|
||||
await github.rest.repos.updateRelease({
|
||||
owner,
|
||||
repo,
|
||||
release_id: release.id,
|
||||
body,
|
||||
const comparisonBase = await resolveComparisonBase(tag, release);
|
||||
core.info(comparisonBase.ref
|
||||
? `Comparison base for ${tag}: ${comparisonBase.ref} (${comparisonBase.source}).`
|
||||
: `No comparison base found for ${tag}.`);
|
||||
|
||||
const comparison = await getComparison(comparisonBase.ref, tag);
|
||||
const whatsChangedSection = comparisonBase.previousTag
|
||||
? appendCommitsToGeneratedNotes(
|
||||
await getGeneratedNotes(tag, release, comparisonBase.previousTag),
|
||||
comparisonBase.ref,
|
||||
tag,
|
||||
comparison,
|
||||
)
|
||||
: (!comparisonBase.ref && !isSnapshotTag(tag)
|
||||
? await getGeneratedNotes(tag, release, "")
|
||||
: buildWhatsChangedSection(comparisonBase.ref, tag, comparison));
|
||||
|
||||
if (!whatsChangedSection) {
|
||||
core.setFailed(`Failed to build What's Changed section for ${tag}.`);
|
||||
return;
|
||||
}
|
||||
|
||||
const webReleaseSection = await buildWebReleaseSection(tag);
|
||||
const releaseSummarySection = await buildReleaseSummarySection({
|
||||
tag,
|
||||
release,
|
||||
comparisonBase,
|
||||
whatsChangedSection,
|
||||
comparison,
|
||||
webReleaseSection,
|
||||
});
|
||||
|
||||
const body = [
|
||||
releaseSummarySection,
|
||||
whatsChangedSection,
|
||||
].filter(Boolean).join("\n\n");
|
||||
|
||||
if (clean(release.body) === clean(body)) {
|
||||
core.info(`Release notes for ${tag} are already up to date.`);
|
||||
} else {
|
||||
await github.rest.repos.updateRelease({
|
||||
owner,
|
||||
repo,
|
||||
release_id: release.id,
|
||||
body,
|
||||
});
|
||||
}
|
||||
|
||||
await core.summary
|
||||
.addHeading("Release Notes")
|
||||
.addRaw(`Updated release notes for ${tag}.`)
|
||||
|
||||
@@ -1,9 +1,12 @@
|
||||
name: Publish Docker Image
|
||||
name: Publish Docker Image Manually
|
||||
|
||||
on:
|
||||
release:
|
||||
types: [published]
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
version:
|
||||
description: Docker image tag and embedded version.
|
||||
required: true
|
||||
type: string
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
@@ -14,7 +17,61 @@ env:
|
||||
IMAGE_NAME: ${{ github.repository }}
|
||||
|
||||
jobs:
|
||||
build-frontend:
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Build frontend artifact
|
||||
uses: ./.github/actions/build-frontend
|
||||
with:
|
||||
artifact-name: komari-frontend
|
||||
|
||||
build-docker-binaries:
|
||||
needs: build-frontend
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Restore frontend artifact
|
||||
uses: ./.github/actions/restore-frontend
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v5
|
||||
with:
|
||||
go-version: "1.23"
|
||||
|
||||
- name: Install Zig
|
||||
uses: ./.github/actions/setup-zig
|
||||
|
||||
- name: Build static binaries for multi-arch
|
||||
env:
|
||||
GOOS: linux
|
||||
CGO_ENABLED: 1
|
||||
GIN_MODE: release
|
||||
VERSION: ${{ github.event.inputs.version }}
|
||||
run: |
|
||||
VERSION_HASH="${{ github.sha }}"
|
||||
LDFLAGS="-s -w -X github.com/komari-monitor/komari/utils.CurrentVersion=${VERSION} -X github.com/komari-monitor/komari/utils.VersionHash=${VERSION_HASH}"
|
||||
|
||||
echo "Building for linux/amd64..."
|
||||
GOARCH=amd64 CC="zig cc -target x86_64-linux-musl" go build -trimpath -ldflags="$LDFLAGS" -o komari-linux-amd64
|
||||
|
||||
echo "Building for linux/arm64..."
|
||||
GOARCH=arm64 CC="zig cc -target aarch64-linux-musl" go build -trimpath -ldflags="$LDFLAGS" -o komari-linux-arm64
|
||||
|
||||
- name: Upload Docker binaries
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: komari-docker-binaries
|
||||
path: komari-linux-*
|
||||
|
||||
build-and-push-multi-arch:
|
||||
needs: build-docker-binaries
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
contents: read
|
||||
@@ -24,54 +81,11 @@ jobs:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Setup Node.js
|
||||
uses: actions/setup-node@v4
|
||||
- name: Download Docker binaries
|
||||
uses: actions/download-artifact@v4
|
||||
with:
|
||||
node-version: "23"
|
||||
|
||||
- name: Clone and build frontend
|
||||
run: |
|
||||
git clone https://github.com/komari-monitor/komari-web komari-web
|
||||
cd komari-web
|
||||
npm install
|
||||
npm run build
|
||||
cd ..
|
||||
mkdir -p web/public/defaultTheme/dist
|
||||
rm -rf web/public/defaultTheme/dist/*
|
||||
cp -r komari-web/dist/* web/public/defaultTheme/dist/
|
||||
cp -f komari-web/komari-theme.json web/public/defaultTheme/
|
||||
if [ -f komari-web/preview.png ]; then cp -f komari-web/preview.png web/public/defaultTheme/; fi
|
||||
if [ -f komari-web/perview.png ]; then cp -f komari-web/perview.png web/public/defaultTheme/; fi
|
||||
# Compatibility: ensure both filenames exist if only one is present
|
||||
if [ -f web/public/defaultTheme/preview.png ] && [ ! -f web/public/defaultTheme/perview.png ]; then cp -f web/public/defaultTheme/preview.png web/public/defaultTheme/perview.png; fi
|
||||
if [ -f web/public/defaultTheme/perview.png ] && [ ! -f web/public/defaultTheme/preview.png ]; then cp -f web/public/defaultTheme/perview.png web/public/defaultTheme/preview.png; fi
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v5
|
||||
with:
|
||||
go-version: "1.23"
|
||||
|
||||
- name: Install Zig
|
||||
run: |
|
||||
wget https://ziglang.org/download/0.14.1/zig-x86_64-linux-0.14.1.tar.xz
|
||||
sudo tar -C /usr/local -Jxf zig-x86_64-linux-0.14.1.tar.xz
|
||||
echo "PATH=$PATH:/usr/local/zig-x86_64-linux-0.14.1" >> $GITHUB_ENV
|
||||
|
||||
- name: Build static binaries for multi-arch
|
||||
env:
|
||||
GOOS: linux
|
||||
CGO_ENABLED: 1
|
||||
GIN_MODE: release
|
||||
run: |
|
||||
VERSION="${{ github.event.release.tag_name }}"
|
||||
VERSION_HASH="${{ github.sha }}"
|
||||
LDFLAGS="-s -w -X github.com/komari-monitor/komari/utils.CurrentVersion=${VERSION} -X github.com/komari-monitor/komari/utils.VersionHash=${VERSION_HASH}"
|
||||
|
||||
echo "Building for linux/amd64..."
|
||||
GOARCH=amd64 CC="zig cc -target x86_64-linux-musl" go build -trimpath -ldflags="$LDFLAGS" -o komari-linux-amd64
|
||||
|
||||
echo "Building for linux/arm64..."
|
||||
GOARCH=arm64 CC="zig cc -target aarch64-linux-musl" go build -trimpath -ldflags="$LDFLAGS" -o komari-linux-arm64
|
||||
name: komari-docker-binaries
|
||||
path: .
|
||||
|
||||
- name: Set up QEMU
|
||||
uses: docker/setup-qemu-action@v3
|
||||
@@ -92,7 +106,7 @@ jobs:
|
||||
with:
|
||||
images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
|
||||
tags: |
|
||||
type=raw,value=${{ github.event.release.tag_name }}
|
||||
type=raw,value=${{ github.event.inputs.version }}
|
||||
type=raw,value=latest
|
||||
|
||||
- name: Build and push Docker image
|
||||
|
||||
@@ -13,7 +13,20 @@ env:
|
||||
IMAGE_NAME: ${{ github.repository }}
|
||||
|
||||
jobs:
|
||||
build-frontend:
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Build frontend artifact
|
||||
uses: ./.github/actions/build-frontend
|
||||
with:
|
||||
artifact-name: komari-frontend
|
||||
|
||||
build-and-attach-binaries:
|
||||
needs: build-frontend
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
strategy:
|
||||
@@ -52,6 +65,10 @@ jobs:
|
||||
goarch: riscv64
|
||||
zig_target_triple: riscv64-linux-musl
|
||||
binary_extension: ""
|
||||
- goos: linux
|
||||
goarch: loong64
|
||||
zig_target_triple: loongarch64-linux-musl
|
||||
binary_extension: ""
|
||||
|
||||
exclude:
|
||||
- goos: windows
|
||||
@@ -61,27 +78,8 @@ jobs:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Setup Node.js
|
||||
uses: actions/setup-node@v4
|
||||
with:
|
||||
node-version: "23"
|
||||
|
||||
- name: Clone and build frontend
|
||||
run: |
|
||||
git clone https://github.com/komari-monitor/komari-web komari-web
|
||||
cd komari-web
|
||||
npm install
|
||||
npm run build
|
||||
cd ..
|
||||
mkdir -p web/public/defaultTheme/dist
|
||||
rm -rf web/public/defaultTheme/dist/*
|
||||
cp -r komari-web/dist/* web/public/defaultTheme/dist/
|
||||
cp -f komari-web/komari-theme.json web/public/defaultTheme/
|
||||
if [ -f komari-web/preview.png ]; then cp -f komari-web/preview.png web/public/defaultTheme/; fi
|
||||
if [ -f komari-web/perview.png ]; then cp -f komari-web/perview.png web/public/defaultTheme/; fi
|
||||
# Compatibility: ensure both filenames exist if only one is present
|
||||
if [ -f web/public/defaultTheme/preview.png ] && [ ! -f web/public/defaultTheme/perview.png ]; then cp -f web/public/defaultTheme/preview.png web/public/defaultTheme/perview.png; fi
|
||||
if [ -f web/public/defaultTheme/perview.png ] && [ ! -f web/public/defaultTheme/preview.png ]; then cp -f web/public/defaultTheme/perview.png web/public/defaultTheme/preview.png; fi
|
||||
- name: Restore frontend artifact
|
||||
uses: ./.github/actions/restore-frontend
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v5
|
||||
@@ -89,12 +87,7 @@ jobs:
|
||||
go-version: "1.23"
|
||||
|
||||
- name: Install Zig cross-compiler toolchain
|
||||
run: |
|
||||
sudo apt-get update # Update apt package lists
|
||||
sudo apt-get install -y build-essential curl
|
||||
wget https://ziglang.org/download/0.14.1/zig-x86_64-linux-0.14.1.tar.xz
|
||||
sudo tar -C /usr/local -Jxf zig-x86_64-linux-0.14.1.tar.xz
|
||||
echo "PATH=$PATH:/usr/local/zig-x86_64-linux-0.14.1" >> $GITHUB_ENV
|
||||
uses: ./.github/actions/setup-zig
|
||||
|
||||
- name: Build binary
|
||||
env:
|
||||
@@ -112,6 +105,12 @@ jobs:
|
||||
VERSION_HASH="${{ github.sha }}" # Use commit SHA for hash
|
||||
go build -trimpath -ldflags="-s -w -X github.com/komari-monitor/komari/utils.CurrentVersion=${VERSION} -X github.com/komari-monitor/komari/utils.VersionHash=${VERSION_HASH}" -o $BINARY_NAME
|
||||
|
||||
- name: Upload binary as artifact
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: komari-${{ matrix.goos }}-${{ matrix.goarch }}
|
||||
path: komari-${{ matrix.goos }}-${{ matrix.goarch }}*
|
||||
|
||||
- name: Upload binary to release
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
@@ -124,3 +123,54 @@ jobs:
|
||||
# Upload the binary as a release asset using GitHub CLI
|
||||
gh release upload ${{ github.event.release.tag_name }} $BINARY_NAME --repo ${{ github.repository }}
|
||||
shell: bash
|
||||
|
||||
build-and-push-docker:
|
||||
needs: build-and-attach-binaries
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
contents: read
|
||||
packages: write
|
||||
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Download Docker binaries
|
||||
uses: actions/download-artifact@v4
|
||||
with:
|
||||
pattern: komari-linux-*
|
||||
path: .
|
||||
merge-multiple: true
|
||||
|
||||
- name: Set up QEMU
|
||||
uses: docker/setup-qemu-action@v3
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@v3
|
||||
|
||||
- name: Log in to the Container registry
|
||||
uses: docker/login-action@v3
|
||||
with:
|
||||
registry: ${{ env.REGISTRY }}
|
||||
username: ${{ github.actor }}
|
||||
password: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
- name: Extract metadata
|
||||
id: meta
|
||||
uses: docker/metadata-action@v5
|
||||
with:
|
||||
images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
|
||||
tags: |
|
||||
type=raw,value=${{ github.event.release.tag_name }}
|
||||
type=raw,value=latest
|
||||
|
||||
- name: Build and push Docker image
|
||||
uses: docker/build-push-action@v5
|
||||
with:
|
||||
context: .
|
||||
push: true
|
||||
platforms: linux/amd64,linux/arm64
|
||||
tags: ${{ steps.meta.outputs.tags }}
|
||||
labels: ${{ steps.meta.outputs.labels }}
|
||||
no-cache: true
|
||||
provenance: false
|
||||
@@ -0,0 +1,293 @@
|
||||
name: Publish Snapshot
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
push:
|
||||
branches:
|
||||
- main
|
||||
|
||||
permissions:
|
||||
contents: write
|
||||
packages: write
|
||||
|
||||
env:
|
||||
REGISTRY: ghcr.io
|
||||
IMAGE_NAME: ${{ github.repository }}
|
||||
|
||||
concurrency:
|
||||
group: snapshot-${{ github.ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
jobs:
|
||||
validate-ref:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
fetch-depth: 1
|
||||
|
||||
- name: Ensure snapshot target is main
|
||||
run: |
|
||||
if [ "${GITHUB_REF}" != "refs/heads/main" ]; then
|
||||
echo "Snapshot builds must run from refs/heads/main. Current ref: ${GITHUB_REF}"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
git fetch --no-tags --depth=1 origin main
|
||||
MAIN_SHA="$(git rev-parse FETCH_HEAD)"
|
||||
|
||||
if [ "${GITHUB_SHA}" != "${MAIN_SHA}" ]; then
|
||||
echo "Snapshot builds must use the current main commit."
|
||||
echo "Workflow SHA: ${GITHUB_SHA}"
|
||||
echo "Current main: ${MAIN_SHA}"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
prepare:
|
||||
needs: validate-ref
|
||||
runs-on: ubuntu-latest
|
||||
outputs:
|
||||
version: ${{ steps.version.outputs.version }}
|
||||
steps:
|
||||
- name: Generate snapshot version
|
||||
id: version
|
||||
run: |
|
||||
# Snapshot 版本号格式: Snapshot-yymmddhhMM (UTC+8)
|
||||
VERSION="Snapshot-$(date -u -d '+8 hours' +%y%m%d%H%M)"
|
||||
echo "version=${VERSION}" >> "$GITHUB_OUTPUT"
|
||||
echo "Snapshot version: ${VERSION}"
|
||||
|
||||
build-frontend:
|
||||
needs: validate-ref
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Build frontend artifact
|
||||
uses: ./.github/actions/build-frontend
|
||||
with:
|
||||
artifact-name: komari-frontend
|
||||
|
||||
build-and-attach-binaries:
|
||||
needs: [prepare, build-frontend]
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
goos: [windows, linux]
|
||||
goarch: [amd64, arm64, 386, riscv64]
|
||||
include:
|
||||
# Windows
|
||||
- goos: windows
|
||||
goarch: amd64
|
||||
zig_target_triple: x86_64-windows-gnu
|
||||
binary_extension: .exe
|
||||
- goos: windows
|
||||
goarch: arm64
|
||||
zig_target_triple: aarch64-windows-gnu
|
||||
binary_extension: .exe
|
||||
- goos: windows
|
||||
goarch: 386
|
||||
zig_target_triple: x86-windows-gnu
|
||||
binary_extension: .exe
|
||||
|
||||
# Linux
|
||||
- goos: linux
|
||||
goarch: amd64
|
||||
zig_target_triple: x86_64-linux-musl
|
||||
binary_extension: ""
|
||||
- goos: linux
|
||||
goarch: arm64
|
||||
zig_target_triple: aarch64-linux-musl
|
||||
binary_extension: ""
|
||||
- goos: linux
|
||||
goarch: 386
|
||||
zig_target_triple: x86-linux-musl
|
||||
binary_extension: ""
|
||||
- goos: linux
|
||||
goarch: riscv64
|
||||
zig_target_triple: riscv64-linux-musl
|
||||
binary_extension: ""
|
||||
- goos: linux
|
||||
goarch: loong64
|
||||
zig_target_triple: loongarch64-linux-musl
|
||||
binary_extension: ""
|
||||
|
||||
exclude:
|
||||
- goos: windows
|
||||
goarch: riscv64
|
||||
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Restore frontend artifact
|
||||
uses: ./.github/actions/restore-frontend
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v5
|
||||
with:
|
||||
go-version: "1.23"
|
||||
|
||||
- name: Install Zig cross-compiler toolchain
|
||||
uses: ./.github/actions/setup-zig
|
||||
|
||||
- name: Build binary
|
||||
env:
|
||||
GOOS: ${{ matrix.goos }}
|
||||
GOARCH: ${{ matrix.goarch }}
|
||||
CGO_ENABLED: 1
|
||||
CC: zig cc -target ${{ matrix.zig_target_triple }}
|
||||
GIN_MODE: release
|
||||
run: |
|
||||
BINARY_NAME=komari-${{ matrix.goos }}-${{ matrix.goarch }}
|
||||
if [ "${{ matrix.binary_extension }}" != "" ]; then
|
||||
BINARY_NAME=${BINARY_NAME}${{ matrix.binary_extension }}
|
||||
fi
|
||||
VERSION="${{ needs.prepare.outputs.version }}"
|
||||
VERSION_HASH="${{ github.sha }}" # Use commit SHA for hash
|
||||
go build -trimpath -ldflags="-s -w -X github.com/komari-monitor/komari/utils.CurrentVersion=${VERSION} -X github.com/komari-monitor/komari/utils.VersionHash=${VERSION_HASH}" -o $BINARY_NAME
|
||||
|
||||
- name: Upload binary as artifact
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: komari-${{ matrix.goos }}-${{ matrix.goarch }}
|
||||
path: komari-${{ matrix.goos }}-${{ matrix.goarch }}*
|
||||
|
||||
publish-snapshot-release:
|
||||
needs: [prepare, build-and-attach-binaries]
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
actions: write
|
||||
contents: write
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Ensure commit is still current main
|
||||
run: |
|
||||
git fetch --no-tags --depth=1 origin main
|
||||
MAIN_SHA="$(git rev-parse FETCH_HEAD)"
|
||||
|
||||
if [ "${GITHUB_SHA}" != "${MAIN_SHA}" ]; then
|
||||
echo "Snapshot publish skipped because a newer main commit exists."
|
||||
echo "Workflow SHA: ${GITHUB_SHA}"
|
||||
echo "Current main: ${MAIN_SHA}"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
- name: Download all binaries
|
||||
uses: actions/download-artifact@v4
|
||||
with:
|
||||
path: artifacts
|
||||
merge-multiple: true
|
||||
|
||||
- name: Create snapshot prerelease and upload binaries
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
VERSION: ${{ needs.prepare.outputs.version }}
|
||||
run: |
|
||||
set -euxo pipefail
|
||||
echo "Snapshot version: ${VERSION}"
|
||||
ls -alR artifacts
|
||||
|
||||
# 如果同名 release/tag 已存在(例如重跑),先删除避免创建冲突
|
||||
if gh release view "${VERSION}" --repo "${{ github.repository }}" >/dev/null 2>&1; then
|
||||
echo "Release ${VERSION} 已存在,正在删除以重新创建..."
|
||||
gh release delete "${VERSION}" --repo "${{ github.repository }}" --cleanup-tag --yes
|
||||
fi
|
||||
|
||||
# 创建预发布并上传所有二进制文件
|
||||
gh release create "${VERSION}" \
|
||||
--repo "${{ github.repository }}" \
|
||||
--target "${{ github.sha }}" \
|
||||
--title "${VERSION}" \
|
||||
--notes "Automated snapshot build (${VERSION}) from commit ${{ github.sha }}." \
|
||||
--prerelease \
|
||||
artifacts/komari-*
|
||||
|
||||
gh workflow run generate-release-notes.yml \
|
||||
--repo "${{ github.repository }}" \
|
||||
--ref main \
|
||||
-f tag="${VERSION}"
|
||||
|
||||
gh release list \
|
||||
--repo "${{ github.repository }}" \
|
||||
--limit 1000 \
|
||||
--json tagName,isPrerelease \
|
||||
--jq '.[] | select(.isPrerelease and (.tagName | startswith("Snapshot-"))) | .tagName' |
|
||||
while IFS= read -r tag; do
|
||||
if [ -z "${tag}" ] || [ "${tag}" = "${VERSION}" ]; then
|
||||
continue
|
||||
fi
|
||||
|
||||
echo "Deleting old snapshot release ${tag}..."
|
||||
gh release delete "${tag}" --repo "${{ github.repository }}" --cleanup-tag --yes
|
||||
done
|
||||
shell: bash
|
||||
|
||||
build-and-push-docker:
|
||||
needs: [prepare, build-and-attach-binaries, publish-snapshot-release]
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
contents: read
|
||||
packages: write
|
||||
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Ensure commit is still current main
|
||||
run: |
|
||||
git fetch --no-tags --depth=1 origin main
|
||||
MAIN_SHA="$(git rev-parse FETCH_HEAD)"
|
||||
|
||||
if [ "${GITHUB_SHA}" != "${MAIN_SHA}" ]; then
|
||||
echo "Snapshot Docker publish skipped because a newer main commit exists."
|
||||
echo "Workflow SHA: ${GITHUB_SHA}"
|
||||
echo "Current main: ${MAIN_SHA}"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
- name: Download Docker binaries
|
||||
uses: actions/download-artifact@v4
|
||||
with:
|
||||
pattern: komari-linux-*
|
||||
path: .
|
||||
merge-multiple: true
|
||||
|
||||
- name: Set up QEMU
|
||||
uses: docker/setup-qemu-action@v3
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@v3
|
||||
|
||||
- name: Log in to the Container registry
|
||||
uses: docker/login-action@v3
|
||||
with:
|
||||
registry: ${{ env.REGISTRY }}
|
||||
username: ${{ github.actor }}
|
||||
password: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
- name: Extract metadata
|
||||
id: meta
|
||||
uses: docker/metadata-action@v5
|
||||
with:
|
||||
images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
|
||||
tags: |
|
||||
type=raw,value=snapshot
|
||||
|
||||
- name: Build and push Docker image
|
||||
uses: docker/build-push-action@v5
|
||||
with:
|
||||
context: .
|
||||
push: true
|
||||
platforms: linux/amd64,linux/arm64
|
||||
tags: ${{ steps.meta.outputs.tags }}
|
||||
labels: ${{ steps.meta.outputs.labels }}
|
||||
no-cache: true
|
||||
provenance: false
|
||||
+1
-18
@@ -8,29 +8,12 @@ ARG TARGETARCH
|
||||
|
||||
RUN apk add --no-cache ca-certificates curl tzdata
|
||||
|
||||
RUN set -eux; \
|
||||
case "${TARGETARCH}" in \
|
||||
amd64) cloudflared_arch="amd64" ;; \
|
||||
386) cloudflared_arch="386" ;; \
|
||||
arm64) cloudflared_arch="arm64" ;; \
|
||||
arm) cloudflared_arch="arm" ;; \
|
||||
*) echo "Unsupported TARGETARCH: ${TARGETARCH}" >&2; exit 1 ;; \
|
||||
esac; \
|
||||
curl -fsSL "https://github.com/cloudflare/cloudflared/releases/latest/download/cloudflared-linux-${cloudflared_arch}" -o /usr/local/bin/cloudflared; \
|
||||
chmod +x /usr/local/bin/cloudflared
|
||||
|
||||
COPY komari-${TARGETOS}-${TARGETARCH} /app/komari
|
||||
|
||||
|
||||
RUN chmod +x /app/komari
|
||||
|
||||
ENV GIN_MODE=release
|
||||
ENV KOMARI_DB_TYPE=sqlite
|
||||
ENV KOMARI_DB_FILE=/app/data/komari.db
|
||||
ENV KOMARI_DB_HOST=localhost
|
||||
ENV KOMARI_DB_PORT=3306
|
||||
ENV KOMARI_DB_USER=root
|
||||
ENV KOMARI_DB_PASS=
|
||||
ENV KOMARI_DB_NAME=komari
|
||||
ENV KOMARI_LISTEN=0.0.0.0:25774
|
||||
|
||||
EXPOSE 25774
|
||||
|
||||
@@ -9,6 +9,11 @@
|
||||
|
||||
Komari is a lightweight, self-hosted server monitoring tool designed to provide a simple and efficient solution for monitoring server performance. It supports viewing server status through a web interface and collects data through a lightweight agent.
|
||||
|
||||
> [!WARNING]
|
||||
> Komari is a self-hosted monitoring and control program intended only for systems you own or are authorized to administer. Do not weaponize Komari, deploy it without consent, or use it for unauthorized access, persistence, command execution, or other abusive activity. For context on real-world abuse risks, see Huntress' analysis: [Komari C2 agent abuse](https://www.huntress.com/blog/komari-c2-agent-abuse).
|
||||
> Users are solely responsible for how they deploy and operate Komari. The developers do not accept responsibility for unauthorized or abusive use, or for any resulting consequences.
|
||||
> On Windows, when remote control is enabled, the client displays a Windows notification at each user login to remind the user that Komari is remote control software.
|
||||
|
||||
[Documentation](https://komari-document.pages.dev/) | [文档(镜像站 By Geekertao)](https://www.komari.wiki) | [Telegram Group](https://t.me/komari_monitor)
|
||||
|
||||
## Features
|
||||
|
||||
+498
@@ -0,0 +1,498 @@
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"log"
|
||||
"net/http"
|
||||
"os"
|
||||
"os/signal"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
"github.com/gin-gonic/gin"
|
||||
|
||||
"github.com/komari-monitor/komari/cmd/flags"
|
||||
"github.com/komari-monitor/komari/database"
|
||||
"github.com/komari-monitor/komari/database/accounts"
|
||||
"github.com/komari-monitor/komari/database/auditlog"
|
||||
"github.com/komari-monitor/komari/database/dbcore"
|
||||
"github.com/komari-monitor/komari/database/metricstore"
|
||||
"github.com/komari-monitor/komari/database/models"
|
||||
d_notification "github.com/komari-monitor/komari/database/notification"
|
||||
"github.com/komari-monitor/komari/database/tasks"
|
||||
"github.com/komari-monitor/komari/pkg/config"
|
||||
"github.com/komari-monitor/komari/pkg/corn"
|
||||
"github.com/komari-monitor/komari/pkg/migrations"
|
||||
"github.com/komari-monitor/komari/utils"
|
||||
"github.com/komari-monitor/komari/utils/geoip"
|
||||
logutil "github.com/komari-monitor/komari/utils/log"
|
||||
"github.com/komari-monitor/komari/utils/messageSender"
|
||||
"github.com/komari-monitor/komari/utils/notifier"
|
||||
"github.com/komari-monitor/komari/web/api"
|
||||
"github.com/komari-monitor/komari/web/oauth"
|
||||
frontendpublic "github.com/komari-monitor/komari/web/public"
|
||||
"github.com/komari-monitor/komari/web/router"
|
||||
"github.com/komari-monitor/komari/web/security"
|
||||
upgradeweb "github.com/komari-monitor/komari/web/update"
|
||||
)
|
||||
|
||||
// cleanupFunc 是一个关闭阶段执行的清理函数。
|
||||
type cleanupFunc struct {
|
||||
name string
|
||||
fn func(ctx context.Context) error
|
||||
}
|
||||
|
||||
// App 显式建模服务端的启动生命周期。
|
||||
//
|
||||
// 过去 RunServer 把目录创建、数据库、metric store、GeoIP、定时任务、通知、OAuth、
|
||||
// Gin 中间件、路由、HTTP 启动与 shutdown 全部混在一个函数里,
|
||||
// 启动顺序只能靠通读整段代码推断,异步初始化过早且吞掉错误,关闭也不完整。
|
||||
//
|
||||
// App 把这些拆成有序阶段:
|
||||
//
|
||||
// Bootstrap 基础设施:目录、数据库、默认管理员、配置快照
|
||||
// InitStores 存储:metric store
|
||||
// InitProviders 外部 provider:OAuth(同步,路由依赖)、GeoIP、消息发送
|
||||
// StartBackground 后台:定时任务
|
||||
// BuildRouter 构建 Gin 引擎与路由
|
||||
// Run 启动 HTTP 服务并阻塞直到收到信号
|
||||
// Shutdown 反序执行已登记的清理函数
|
||||
//
|
||||
// 每个阶段返回错误即可让上层决定是否中止启动;各资源在创建时把对应清理登记到
|
||||
// cleanup 栈,关闭时按后进先出(LIFO)顺序释放。
|
||||
type App struct {
|
||||
settings *config.Settings
|
||||
engine *gin.Engine
|
||||
server *http.Server
|
||||
reload *ReloadManager
|
||||
dbReady bool
|
||||
oauthReady bool
|
||||
|
||||
cleanups []cleanupFunc
|
||||
}
|
||||
|
||||
// NewApp 构造一个空的 App。真正的初始化在各阶段方法中完成。
|
||||
func NewApp() *App {
|
||||
return &App{
|
||||
reload: NewReloadManager(),
|
||||
}
|
||||
}
|
||||
|
||||
// addCleanup 登记一个关闭阶段的清理函数(后进先出执行)。
|
||||
func (a *App) addCleanup(name string, fn func(ctx context.Context) error) {
|
||||
a.cleanups = append(a.cleanups, cleanupFunc{name: name, fn: fn})
|
||||
}
|
||||
|
||||
// Bootstrap 初始化基础设施:数据目录、数据库、默认管理员账号、配置快照。
|
||||
func (a *App) Bootstrap() error {
|
||||
if err := os.MkdirAll("./data/theme", os.ModePerm); err != nil {
|
||||
return fmt.Errorf("failed to create theme directory: %w", err)
|
||||
}
|
||||
|
||||
// 注入版本标识,供 dbcore 在检测到版本升级时自动备份 ./data。
|
||||
// 必须在 Initialize() 之前设置。
|
||||
dbcore.SetVersionID(utils.CurrentVersion + "-" + utils.VersionHash)
|
||||
|
||||
// 显式初始化数据库(返回错误而非在 getter 里 log.Fatal)。
|
||||
if err := dbcore.Initialize(); err != nil {
|
||||
return fmt.Errorf("failed to initialize database: %w", err)
|
||||
}
|
||||
|
||||
a.dbReady = true
|
||||
a.addCleanup("database", func(context.Context) error {
|
||||
return dbcore.Close()
|
||||
})
|
||||
|
||||
if utils.VersionHash != "unknown" {
|
||||
gin.SetMode(gin.ReleaseMode)
|
||||
}
|
||||
|
||||
// 首次启动创建默认管理员账号。
|
||||
if err := ensureDefaultAdmin(); err != nil {
|
||||
return fmt.Errorf("failed to ensure default admin account: %w", err)
|
||||
}
|
||||
|
||||
conf, err := config.GetManyAs[config.Settings]()
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to load settings: %w", err)
|
||||
}
|
||||
a.settings = conf
|
||||
return nil
|
||||
}
|
||||
|
||||
// InitStores 初始化独立存储组件(metric store)并执行 metrics 迁移。
|
||||
//
|
||||
// metric store 现在始终启用(旧的 metric_store_enabled 开关已废弃):
|
||||
// 未显式配置时使用 SQLite(./data/metrics.db),否则使用配置的 MySQL/PostgreSQL。
|
||||
// 初始化失败即启动失败,不再静默 fallback 到旧 records 表。
|
||||
//
|
||||
// 初始化成功后先执行需要 metric store 的一次性迁移,再执行启动迁移:当 metrics
|
||||
// 存储后端发生变化(例如从默认 SQLite 切换到 MySQL/PostgreSQL)时,把上一个
|
||||
// metrics 目标库的数据搬运到当前目标。迁移失败同样让启动失败,并打印明确错误。
|
||||
func (a *App) InitStores() error {
|
||||
if err := metricstore.InitializeStore(); err != nil {
|
||||
auditlog.EventLog("error", fmt.Sprintf("Failed to initialize metric store: %v", err))
|
||||
return fmt.Errorf("failed to initialize metric store: %w", err)
|
||||
}
|
||||
a.addCleanup("metric-store", func(ctx context.Context) error {
|
||||
return metricstore.CloseStoreContext(ctx)
|
||||
})
|
||||
|
||||
if err := migrations.RunMetricStoreMigrations(migrations.MetricContext{
|
||||
DB: dbcore.GetDBInstance(),
|
||||
Store: metricstore.GetStore(),
|
||||
}); err != nil {
|
||||
auditlog.EventLog("error", fmt.Sprintf("Metric store one-shot migrations failed: %v", err))
|
||||
return fmt.Errorf("metric store one-shot migrations failed: %w", err)
|
||||
}
|
||||
|
||||
// 存储后端切换时把上一个 metrics 目标库的数据搬运到当前目标(失败即启动失败)。
|
||||
if err := metricstore.RunStartupMigration(); err != nil {
|
||||
auditlog.EventLog("error", fmt.Sprintf("Metrics startup migration failed: %v", err))
|
||||
|
||||
return fmt.Errorf("metrics startup migration failed: %w", err)
|
||||
}
|
||||
metricstore.StartReportBatcher()
|
||||
a.addCleanup("metric-report-batcher", func(ctx context.Context) error {
|
||||
return metricstore.StopReportBatcher(ctx)
|
||||
})
|
||||
return nil
|
||||
}
|
||||
|
||||
// InitProviders 初始化外部 provider。
|
||||
//
|
||||
// OAuth 必须在 HTTP 服务开始接收请求之前同步完成,否则 oauth.CurrentProvider()
|
||||
// 存在空指针风险;GeoIP 与消息发送允许后台初始化。
|
||||
func (a *App) InitProviders() error {
|
||||
a.initOAuth()
|
||||
|
||||
// GeoIP:可能涉及下载/加载 mmdb,放后台执行避免拖慢启动。
|
||||
go geoip.InitGeoIp()
|
||||
a.addCleanup("geoip", func(context.Context) error {
|
||||
return geoip.Shutdown()
|
||||
})
|
||||
|
||||
// 消息发送 provider。
|
||||
messageSender.Initialize()
|
||||
a.addCleanup("message-sender", func(context.Context) error {
|
||||
return messageSender.Shutdown()
|
||||
})
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// initOAuth initializes the provider once. The restricted upgrade server also
|
||||
// needs OAuth login, so provider setup may happen before the normal provider
|
||||
// stage without registering duplicate cleanup work.
|
||||
func (a *App) initOAuth() {
|
||||
if a.oauthReady {
|
||||
return
|
||||
}
|
||||
if err := oauth.Initialize(); err != nil {
|
||||
// Keep password login available when an OAuth provider is misconfigured.
|
||||
log.Printf("Failed to initialize OAuth provider: %v", err)
|
||||
auditlog.EventLog("error", fmt.Sprintf("Failed to initialize OAuth provider: %v", err))
|
||||
}
|
||||
a.oauthReady = true
|
||||
a.addCleanup("oauth", func(context.Context) error {
|
||||
return oauth.Shutdown()
|
||||
})
|
||||
}
|
||||
|
||||
func (a *App) LegacyUpgradeRequired() (bool, migrations.LegacyMonitoringSummary, error) {
|
||||
return migrations.LegacyMonitoringMigrationRequired(dbcore.GetDBInstance())
|
||||
}
|
||||
|
||||
// RunLegacyUpgrade starts a restricted HTTP server on the normal listen
|
||||
// address. It returns only after the migration finishes and the restricted
|
||||
// listener has released the port, or after shutdown/interruption.
|
||||
func (a *App) RunLegacyUpgrade(summary migrations.LegacyMonitoringSummary) (bool, error) {
|
||||
a.initOAuth()
|
||||
controller := upgradeweb.NewController(dbcore.GetDBInstance(), summary)
|
||||
controller.Activate()
|
||||
defer controller.Deactivate()
|
||||
|
||||
r := gin.New()
|
||||
r.Use(logutil.GinLogger())
|
||||
r.Use(logutil.GinRecovery())
|
||||
cors := security.NewCorsController(a.settings.CorsOriginCheckEnabled, a.settings.CorsAllowedOrigins)
|
||||
r.Use(cors.Middleware())
|
||||
r.Use(api.IdentityMiddleware())
|
||||
r.Use(func(c *gin.Context) {
|
||||
if strings.HasPrefix(c.Request.URL.Path, "/api") {
|
||||
c.Header("Cache-Control", "no-store")
|
||||
}
|
||||
c.Next()
|
||||
})
|
||||
|
||||
controller.Register(r)
|
||||
frontendpublic.Static(r.Group("/"), func(handlers ...gin.HandlerFunc) {
|
||||
r.NoRoute(func(c *gin.Context) {
|
||||
requestPath := c.Request.URL.Path
|
||||
if strings.HasPrefix(requestPath, "/api") {
|
||||
api.RespondError(c, http.StatusNotFound, "Not found in upgrade mode")
|
||||
return
|
||||
}
|
||||
if c.Request.Method == http.MethodGet && requestPath != upgradeweb.PagePath && filepath.Ext(requestPath) == "" {
|
||||
c.Redirect(http.StatusTemporaryRedirect, upgradeweb.PagePath)
|
||||
return
|
||||
}
|
||||
for _, handler := range handlers {
|
||||
handler(c)
|
||||
if c.IsAborted() {
|
||||
return
|
||||
}
|
||||
}
|
||||
})
|
||||
})
|
||||
|
||||
a.engine = r
|
||||
a.server = &http.Server{Addr: flags.Listen, Handler: r}
|
||||
serverErr := make(chan error, 1)
|
||||
log.Printf("Legacy monitoring data requires the 1.2.7 upgrade wizard on %s", flags.Listen)
|
||||
go func() {
|
||||
if err := a.server.ListenAndServe(); err != nil && err != http.ErrServerClosed {
|
||||
serverErr <- err
|
||||
}
|
||||
}()
|
||||
|
||||
quit := make(chan os.Signal, 1)
|
||||
signal.Notify(quit, os.Interrupt, syscall.SIGTERM)
|
||||
defer signal.Stop(quit)
|
||||
|
||||
select {
|
||||
case err := <-serverErr:
|
||||
return false, fmt.Errorf("listen in upgrade mode: %w", err)
|
||||
case <-quit:
|
||||
return false, a.Shutdown()
|
||||
case <-controller.Done():
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||
defer cancel()
|
||||
if err := a.server.Shutdown(ctx); err != nil {
|
||||
return false, fmt.Errorf("stop upgrade server: %w", err)
|
||||
}
|
||||
a.server = nil
|
||||
a.engine = nil
|
||||
return true, nil
|
||||
}
|
||||
}
|
||||
|
||||
// StartBackground 启动后台工作:定时任务。
|
||||
func (a *App) StartBackground() error {
|
||||
registerScheduledWork()
|
||||
a.addCleanup("scheduler", func(context.Context) error {
|
||||
corn.StopAll()
|
||||
return nil
|
||||
})
|
||||
return nil
|
||||
}
|
||||
|
||||
// registerReloadHandlers 把此前散落各处的 config.Subscribe 统一登记到 reload 管理器。
|
||||
func (a *App) registerReloadHandlers(cors *security.CorsController) {
|
||||
// OAuth provider 切换。
|
||||
a.reload.Register("oauth-provider", func(event config.ConfigEvent) {
|
||||
if ok, t := config.IsChangedT[string](event, config.OAuthProviderKey); ok {
|
||||
if t == "" || t == "none" {
|
||||
t = "github"
|
||||
}
|
||||
oidcProvider, err := database.GetOidcConfigByName(t)
|
||||
if err != nil {
|
||||
log.Printf("Failed to get OIDC provider config: %v", err)
|
||||
return
|
||||
}
|
||||
log.Printf("Using %s as OIDC provider", oidcProvider.Name)
|
||||
if err := oauth.LoadProvider(oidcProvider.Name, oidcProvider.Addition); err != nil {
|
||||
auditlog.EventLog("error", fmt.Sprintf("Failed to load OIDC provider: %v", err))
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
// GeoIP provider 切换。
|
||||
a.reload.Register("geoip-provider", func(event config.ConfigEvent) {
|
||||
if event.IsChanged(config.GeoIpProviderKey) {
|
||||
go geoip.InitGeoIp()
|
||||
}
|
||||
})
|
||||
|
||||
// 消息发送方式切换。
|
||||
a.reload.Register("message-sender", func(event config.ConfigEvent) {
|
||||
if event.IsChanged(config.NotificationMethodKey) {
|
||||
go messageSender.Initialize()
|
||||
}
|
||||
})
|
||||
|
||||
// CORS 配置热更新。
|
||||
a.reload.Register("cors", func(event config.ConfigEvent) {
|
||||
cors.Update(event)
|
||||
})
|
||||
}
|
||||
|
||||
// BuildRouter 构建 Gin 引擎、中间件与全部路由,并登记热重载处理器。
|
||||
func (a *App) BuildRouter() error {
|
||||
r := gin.New()
|
||||
r.Use(logutil.GinLogger())
|
||||
r.Use(logutil.GinRecovery())
|
||||
|
||||
cors := security.NewCorsController(a.settings.CorsOriginCheckEnabled, a.settings.CorsAllowedOrigins)
|
||||
r.Use(cors.Middleware())
|
||||
|
||||
r.Use(api.IdentityMiddleware())
|
||||
r.Use(api.PrivateSiteMiddleware())
|
||||
|
||||
r.Use(func(c *gin.Context) {
|
||||
if len(c.Request.URL.Path) >= 4 && c.Request.URL.Path[:4] == "/api" {
|
||||
c.Header("Cache-Control", "no-store")
|
||||
}
|
||||
c.Next()
|
||||
})
|
||||
|
||||
router.Register(r)
|
||||
|
||||
// 集中登记并启动热重载订阅。
|
||||
a.registerReloadHandlers(cors)
|
||||
a.reload.Start()
|
||||
|
||||
a.engine = r
|
||||
return nil
|
||||
}
|
||||
|
||||
// Run 启动 HTTP 服务并阻塞直到收到中断信号或服务异常退出。
|
||||
func (a *App) Run() error {
|
||||
a.server = &http.Server{
|
||||
Addr: flags.Listen,
|
||||
Handler: a.engine,
|
||||
}
|
||||
|
||||
serverErr := make(chan error, 1)
|
||||
log.Printf("Starting server on %s ...", flags.Listen)
|
||||
go func() {
|
||||
if err := a.server.ListenAndServe(); err != nil && err != http.ErrServerClosed {
|
||||
serverErr <- err
|
||||
}
|
||||
}()
|
||||
|
||||
quit := make(chan os.Signal, 1)
|
||||
signal.Notify(quit, os.Interrupt, syscall.SIGTERM)
|
||||
|
||||
select {
|
||||
case err := <-serverErr:
|
||||
a.onFatal(err)
|
||||
return fmt.Errorf("listen: %w", err)
|
||||
case <-quit:
|
||||
return a.Shutdown()
|
||||
}
|
||||
}
|
||||
|
||||
// Shutdown 优雅关闭:先停止接收新请求,再反序执行已登记的清理函数。
|
||||
func (a *App) Shutdown() error {
|
||||
if a.dbReady {
|
||||
auditlog.Log("", "", "server is shutting down", "info")
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||
defer cancel()
|
||||
|
||||
// 先关闭 HTTP 服务,停止接收新请求。
|
||||
if a.server != nil {
|
||||
if err := a.server.Shutdown(ctx); err != nil {
|
||||
log.Printf("HTTP server forced to shutdown: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// 反序释放资源(后进先出)。
|
||||
for i := len(a.cleanups) - 1; i >= 0; i-- {
|
||||
c := a.cleanups[i]
|
||||
if err := c.fn(ctx); err != nil {
|
||||
log.Printf("cleanup %q failed: %v", c.name, err)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// onFatal 处理 HTTP 服务致命错误:尽力释放已登记资源。
|
||||
func (a *App) onFatal(err error) {
|
||||
if a.dbReady {
|
||||
auditlog.Log("", "", "server encountered a fatal error: "+err.Error(), "error")
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||
defer cancel()
|
||||
for i := len(a.cleanups) - 1; i >= 0; i-- {
|
||||
c := a.cleanups[i]
|
||||
if cerr := c.fn(ctx); cerr != nil {
|
||||
log.Printf("cleanup %q failed: %v", c.name, cerr)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ensureDefaultAdmin 首次启动(无任何用户)时创建默认管理员账号。
|
||||
func ensureDefaultAdmin() error {
|
||||
var count int64
|
||||
dbcore.GetDBInstance().Model(&models.User{}).Count(&count)
|
||||
if count != 0 {
|
||||
return nil
|
||||
}
|
||||
user, passwd, err := accounts.CreateDefaultAdminAccount()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
log.Println("Default admin account created. Username:", user, ", Password:", passwd)
|
||||
return nil
|
||||
}
|
||||
|
||||
// registerScheduledWork 注册所有定时任务与首启动同步逻辑。
|
||||
func registerScheduledWork() {
|
||||
if err := tasks.ReloadPingSchedule(); err != nil {
|
||||
log.Println("Failed to reload ping schedule:", err)
|
||||
}
|
||||
if err := d_notification.ReloadLoadNotificationSchedule(); err != nil {
|
||||
log.Println("Failed to reload load notification schedule:", err)
|
||||
}
|
||||
|
||||
if err := corn.AddFunc("records:cleanup", "@every 30m", cleanupScheduledData); err != nil {
|
||||
log.Println("Failed to add cleanup scheduled task:", err)
|
||||
}
|
||||
if err := corn.AddContextFunc("metrics:compact", "@every 5m", true, compactMetricStore); err != nil {
|
||||
log.Println("Failed to add metric compact scheduled task:", err)
|
||||
}
|
||||
if err := corn.AddFunc("notifier:traffic", "@every 1m", notifier.CheckTraffic); err != nil {
|
||||
log.Println("Failed to add traffic notification task:", err)
|
||||
}
|
||||
if err := corn.AddFunc("notifier:expire", "0 0 9 * * *", notifier.CheckExpireScheduledWork); err != nil {
|
||||
log.Println("Failed to add expire notification scheduled task:", err)
|
||||
}
|
||||
|
||||
notifier.InitTrafficReportSchedule()
|
||||
}
|
||||
|
||||
const taskResultRetentionDays = 30
|
||||
|
||||
func cleanupScheduledData() {
|
||||
before := time.Now().UTC().Add(-24 * time.Hour * taskResultRetentionDays)
|
||||
if err := tasks.ClearTaskResultsByTimeBefore(before); err != nil {
|
||||
log.Printf("Failed to clean expired task results: %v", err)
|
||||
}
|
||||
|
||||
auditlog.RemoveOldLogs()
|
||||
accounts.RemoveExpiredSessions()
|
||||
}
|
||||
|
||||
func compactMetricStore(ctx context.Context) {
|
||||
compactCtx, cancel := context.WithTimeout(ctx, 15*time.Minute)
|
||||
defer cancel()
|
||||
|
||||
written, err := metricstore.Compact(compactCtx, time.Now().UTC())
|
||||
if errors.Is(err, metricstore.ErrCompactInProgress) {
|
||||
return
|
||||
}
|
||||
if err != nil {
|
||||
log.Printf("Failed to compact metric store after writing %d rollup buckets: %v", written, err)
|
||||
return
|
||||
}
|
||||
if written > 0 {
|
||||
log.Printf("Metric store compacted %d rollup buckets", written)
|
||||
}
|
||||
}
|
||||
@@ -10,11 +10,6 @@ var (
|
||||
// 数据库配置
|
||||
DatabaseType string // 数据库类型:sqlite
|
||||
DatabaseFile string // SQLite数据库文件路径
|
||||
DatabaseHost string // 保留的兼容参数,当前未使用
|
||||
DatabasePort string // 保留的兼容参数,当前未使用
|
||||
DatabaseUser string // 保留的兼容参数,当前未使用
|
||||
DatabasePass string // 保留的兼容参数,当前未使用
|
||||
DatabaseName string // 保留的兼容参数,当前未使用
|
||||
|
||||
Listen string
|
||||
)
|
||||
|
||||
@@ -0,0 +1,67 @@
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"log"
|
||||
|
||||
"github.com/komari-monitor/komari/pkg/config"
|
||||
)
|
||||
|
||||
// reloadHandler 是单个配置热重载处理器。
|
||||
//
|
||||
// name 仅用于日志定位,handler 接收配置变更事件,自行判断关心的 key 是否变化。
|
||||
// 每个 handler 相互独立:一个 panic 不应影响其它 handler。
|
||||
type reloadHandler struct {
|
||||
name string
|
||||
handler func(config.ConfigEvent)
|
||||
}
|
||||
|
||||
// ReloadManager 统一收敛所有配置热重载逻辑。
|
||||
//
|
||||
// 过去 config.Subscribe 分散在 cmd/server.go、cors.go 等多处,导致:
|
||||
// - 谁监听哪些 key 不清晰;
|
||||
// - 某个 handler panic 会连带影响同一订阅回调里的其它逻辑;
|
||||
// - 关闭时无法统一停止。
|
||||
//
|
||||
// ReloadManager 把这些 handler 注册在一处,用单个 config.Subscribe 分发,
|
||||
// 并对每个 handler 做 panic 隔离。
|
||||
type ReloadManager struct {
|
||||
handlers []reloadHandler
|
||||
started bool
|
||||
}
|
||||
|
||||
// NewReloadManager 创建一个空的热重载管理器。
|
||||
func NewReloadManager() *ReloadManager {
|
||||
return &ReloadManager{}
|
||||
}
|
||||
|
||||
// Register 注册一个命名的热重载处理器。必须在 Start 之前调用。
|
||||
func (m *ReloadManager) Register(name string, handler func(config.ConfigEvent)) {
|
||||
if handler == nil {
|
||||
return
|
||||
}
|
||||
m.handlers = append(m.handlers, reloadHandler{name: name, handler: handler})
|
||||
}
|
||||
|
||||
// Start 向 config 订阅一次,之后每个事件会分发给全部已注册 handler。
|
||||
// 重复调用无副作用。
|
||||
func (m *ReloadManager) Start() {
|
||||
if m.started {
|
||||
return
|
||||
}
|
||||
m.started = true
|
||||
config.Subscribe(func(event config.ConfigEvent) {
|
||||
for _, h := range m.handlers {
|
||||
m.dispatch(h, event)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// dispatch 执行单个 handler,并隔离 panic,避免一个 handler 影响其它。
|
||||
func (m *ReloadManager) dispatch(h reloadHandler, event config.ConfigEvent) {
|
||||
defer func() {
|
||||
if r := recover(); r != nil {
|
||||
log.Printf("config reload handler %q panicked: %v", h.name, r)
|
||||
}
|
||||
}()
|
||||
h.handler(event)
|
||||
}
|
||||
+2
-22
@@ -17,17 +17,6 @@ func GetEnv(key, defaultValue string) string {
|
||||
return value
|
||||
}
|
||||
|
||||
// 从环境变量获取默认值
|
||||
var (
|
||||
dbTypeEnv = GetEnv("KOMARI_DB_TYPE", "sqlite")
|
||||
dbFileEnv = GetEnv("KOMARI_DB_FILE", "./data/komari.db")
|
||||
dbHostEnv = GetEnv("KOMARI_DB_HOST", "localhost")
|
||||
dbPortEnv = GetEnv("KOMARI_DB_PORT", "3306")
|
||||
dbUserEnv = GetEnv("KOMARI_DB_USER", "root")
|
||||
dbPassEnv = GetEnv("KOMARI_DB_PASS", "")
|
||||
dbNameEnv = GetEnv("KOMARI_DB_NAME", "komari")
|
||||
)
|
||||
|
||||
var RootCmd = &cobra.Command{
|
||||
Use: "Komari",
|
||||
Short: "Komari is a simple server monitoring tool",
|
||||
@@ -47,15 +36,6 @@ func Execute() {
|
||||
}
|
||||
|
||||
func init() {
|
||||
// 设置命令行参数,提供环境变量作为默认值
|
||||
RootCmd.PersistentFlags().StringVarP(&flags.DatabaseType, "db-type", "t", dbTypeEnv, "Database type (sqlite) [env: KOMARI_DB_TYPE]")
|
||||
RootCmd.PersistentFlags().StringVarP(&flags.DatabaseFile, "database", "d", dbFileEnv, "SQLite database file path [env: KOMARI_DB_FILE]")
|
||||
RootCmd.PersistentFlags().StringVar(&flags.DatabaseHost, "db-host", dbHostEnv, "Reserved database host parameter [env: KOMARI_DB_HOST]")
|
||||
RootCmd.PersistentFlags().StringVar(&flags.DatabasePort, "db-port", dbPortEnv, "Reserved database port parameter [env: KOMARI_DB_PORT]")
|
||||
RootCmd.PersistentFlags().StringVar(&flags.DatabaseUser, "db-user", dbUserEnv, "Reserved database username parameter [env: KOMARI_DB_USER]")
|
||||
RootCmd.PersistentFlags().StringVar(&flags.DatabasePass, "db-pass", dbPassEnv, "Reserved database password parameter [env: KOMARI_DB_PASS]")
|
||||
RootCmd.PersistentFlags().StringVar(&flags.DatabaseName, "db-name", dbNameEnv, "Reserved database name parameter [env: KOMARI_DB_NAME]")
|
||||
for _, name := range []string{"db-host", "db-port", "db-user", "db-pass", "db-name"} {
|
||||
_ = RootCmd.PersistentFlags().MarkHidden(name)
|
||||
}
|
||||
RootCmd.PersistentFlags().StringVarP(&flags.DatabaseType, "db-type", "t", "sqlite", "Database type (sqlite)")
|
||||
RootCmd.PersistentFlags().StringVarP(&flags.DatabaseFile, "database", "d", "./data/komari.db", "SQLite database file path")
|
||||
}
|
||||
+42
-204
@@ -1,40 +1,9 @@
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"log"
|
||||
"net/http"
|
||||
"os"
|
||||
"os/signal"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
"github.com/gin-gonic/gin"
|
||||
"github.com/komari-monitor/komari/cmd/flags"
|
||||
"github.com/komari-monitor/komari/pkg/corn"
|
||||
"github.com/komari-monitor/komari/web/api"
|
||||
|
||||
"github.com/komari-monitor/komari/database"
|
||||
"github.com/komari-monitor/komari/database/accounts"
|
||||
"github.com/komari-monitor/komari/database/auditlog"
|
||||
"github.com/komari-monitor/komari/database/dbcore"
|
||||
"github.com/komari-monitor/komari/database/models"
|
||||
d_notification "github.com/komari-monitor/komari/database/notification"
|
||||
"github.com/komari-monitor/komari/database/records"
|
||||
"github.com/komari-monitor/komari/database/tasks"
|
||||
"github.com/komari-monitor/komari/pkg/config"
|
||||
"github.com/komari-monitor/komari/utils"
|
||||
"github.com/komari-monitor/komari/utils/cloudflared"
|
||||
"github.com/komari-monitor/komari/utils/geoip"
|
||||
logutil "github.com/komari-monitor/komari/utils/log"
|
||||
"github.com/komari-monitor/komari/utils/messageSender"
|
||||
"github.com/komari-monitor/komari/utils/notifier"
|
||||
"github.com/komari-monitor/komari/web/nezha"
|
||||
"github.com/komari-monitor/komari/web/oauth"
|
||||
report_cache "github.com/komari-monitor/komari/web/report"
|
||||
"github.com/komari-monitor/komari/web/router"
|
||||
"github.com/komari-monitor/komari/web/security"
|
||||
"github.com/spf13/cobra"
|
||||
)
|
||||
|
||||
@@ -54,184 +23,53 @@ func init() {
|
||||
RootCmd.AddCommand(ServerCmd)
|
||||
}
|
||||
|
||||
// RunServer 按显式的生命周期阶段启动服务端。
|
||||
//
|
||||
// 具体各阶段的职责与顺序见 App(cmd/app.go)。这里只负责串联:
|
||||
// 任一初始化阶段失败即中止启动,避免在半初始化状态下对外提供服务。
|
||||
func RunServer() {
|
||||
// #region 初始化
|
||||
if err := os.MkdirAll("./data/theme", os.ModePerm); err != nil {
|
||||
log.Fatalf("Failed to create theme directory: %v", err)
|
||||
app := NewApp()
|
||||
if err := app.Bootstrap(); err != nil {
|
||||
_ = app.Shutdown()
|
||||
log.Fatalf("server startup failed at %q: %v", "bootstrap", err)
|
||||
}
|
||||
InitDatabase()
|
||||
if utils.VersionHash != "unknown" {
|
||||
gin.SetMode(gin.ReleaseMode)
|
||||
}
|
||||
conf, err := config.GetManyAs[config.Settings]()
|
||||
|
||||
required, summary, err := app.LegacyUpgradeRequired()
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
_ = app.Shutdown()
|
||||
log.Fatalf("server startup failed at %q: %v", "detect-1.2.7-upgrade", err)
|
||||
}
|
||||
go geoip.InitGeoIp()
|
||||
go DoScheduledWork()
|
||||
go messageSender.Initialize()
|
||||
// oidcInit
|
||||
go oauth.Initialize()
|
||||
|
||||
if conf.NezhaCompatEnabled {
|
||||
go func() {
|
||||
if err := nezha.StartNezhaCompat(conf.NezhaCompatListen); err != nil {
|
||||
log.Printf("Nezha compat server error: %v", err)
|
||||
auditlog.EventLog("error", fmt.Sprintf("Nezha compat server error: %v", err))
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
config.Subscribe(func(event config.ConfigEvent) {
|
||||
if ok, t := config.IsChangedT[string](event, config.OAuthProviderKey); ok {
|
||||
if t == "" || t == "none" {
|
||||
t = "github"
|
||||
}
|
||||
oidcProvider, err := database.GetOidcConfigByName(t)
|
||||
if err != nil {
|
||||
log.Printf("Failed to get OIDC provider config: %v", err)
|
||||
} else {
|
||||
log.Printf("Using %s as OIDC provider", oidcProvider.Name)
|
||||
}
|
||||
err = oauth.LoadProvider(oidcProvider.Name, oidcProvider.Addition)
|
||||
if err != nil {
|
||||
auditlog.EventLog("error", fmt.Sprintf("Failed to load OIDC provider: %v", err))
|
||||
}
|
||||
}
|
||||
|
||||
if ok, t := config.IsChangedT[bool](event, config.NezhaCompatEnabledKey); ok {
|
||||
if t {
|
||||
l, _ := config.GetAs[string](config.NezhaCompatListenKey)
|
||||
if err := nezha.StartNezhaCompat(l); err != nil {
|
||||
log.Printf("start Nezha compat server error: %v", err)
|
||||
auditlog.EventLog("error", fmt.Sprintf("start Nezha compat server error: %v", err))
|
||||
}
|
||||
} else {
|
||||
if err := nezha.StopNezhaCompat(); err != nil {
|
||||
log.Printf("stop Nezha compat server error: %v", err)
|
||||
auditlog.EventLog("error", fmt.Sprintf("stop Nezha compat server error: %v", err))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
})
|
||||
// 初始化 cloudflared
|
||||
if err := cloudflared.AutoStart(GetEnv("KOMARI_CLOUDFLARED_TOKEN", "")); err != nil {
|
||||
log.Printf("failed to auto start cloudflared: %v", err)
|
||||
}
|
||||
|
||||
r := gin.New()
|
||||
r.Use(logutil.GinLogger())
|
||||
r.Use(logutil.GinRecovery())
|
||||
|
||||
config.Subscribe(func(event config.ConfigEvent) {
|
||||
if event.IsChanged(config.GeoIpProviderKey) {
|
||||
go geoip.InitGeoIp()
|
||||
}
|
||||
|
||||
if event.IsChanged(config.NotificationMethodKey) {
|
||||
go messageSender.Initialize()
|
||||
}
|
||||
|
||||
})
|
||||
r.Use(security.CorsMiddleware(conf.CorsOriginCheckEnabled, conf.CorsAllowedOrigins))
|
||||
|
||||
r.Use(api.IdentityMiddleware())
|
||||
r.Use(api.PrivateSiteMiddleware())
|
||||
|
||||
r.Use(func(c *gin.Context) {
|
||||
if len(c.Request.URL.Path) >= 4 && c.Request.URL.Path[:4] == "/api" {
|
||||
c.Header("Cache-Control", "no-store")
|
||||
}
|
||||
c.Next()
|
||||
})
|
||||
|
||||
router.Register(r)
|
||||
|
||||
srv := &http.Server{
|
||||
Addr: flags.Listen,
|
||||
Handler: r,
|
||||
}
|
||||
log.Printf("Starting server on %s ...", flags.Listen)
|
||||
go func() {
|
||||
if err := srv.ListenAndServe(); err != nil && err != http.ErrServerClosed {
|
||||
OnFatal(err)
|
||||
log.Fatalf("listen: %s\n", err)
|
||||
}
|
||||
}()
|
||||
quit := make(chan os.Signal, 1)
|
||||
signal.Notify(quit, os.Interrupt, syscall.SIGTERM)
|
||||
<-quit
|
||||
OnShutdown()
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
defer cancel()
|
||||
if err := srv.Shutdown(ctx); err != nil {
|
||||
log.Fatalf("Server forced to shutdown: %v", err)
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
func InitDatabase() {
|
||||
var count int64 = 0
|
||||
if dbcore.GetDBInstance().Model(&models.User{}).Count(&count); count == 0 {
|
||||
user, passwd, err := accounts.CreateDefaultAdminAccount()
|
||||
if required {
|
||||
completed, err := app.RunLegacyUpgrade(summary)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
_ = app.Shutdown()
|
||||
log.Fatalf("server startup failed at %q: %v", "run-1.2.7-upgrade", err)
|
||||
}
|
||||
log.Println("Default admin account created. Username:", user, ", Password:", passwd)
|
||||
if !completed {
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
// 初始化阶段:任一步失败都不应继续对外服务。
|
||||
type stage struct {
|
||||
name string
|
||||
fn func() error
|
||||
}
|
||||
stages := []stage{
|
||||
{"init-stores", app.InitStores},
|
||||
{"init-providers", app.InitProviders},
|
||||
{"start-background", app.StartBackground},
|
||||
{"build-router", app.BuildRouter},
|
||||
}
|
||||
for _, s := range stages {
|
||||
if err := s.fn(); err != nil {
|
||||
// 已登记的资源尽力回收,再退出。
|
||||
_ = app.Shutdown()
|
||||
log.Fatalf("server startup failed at %q: %v", s.name, err)
|
||||
}
|
||||
}
|
||||
|
||||
if err := app.Run(); err != nil {
|
||||
log.Fatalf("server exited with error: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// #region 定时任务
|
||||
func DoScheduledWork() {
|
||||
if err := tasks.ReloadPingSchedule(); err != nil {
|
||||
log.Println("Failed to reload ping schedule:", err)
|
||||
}
|
||||
if err := d_notification.ReloadLoadNotificationSchedule(); err != nil {
|
||||
log.Println("Failed to reload load notification schedule:", err)
|
||||
}
|
||||
records.CompactRecord()
|
||||
|
||||
if err := corn.AddFunc("records:cleanup", "@every 30m", cleanupScheduledData); err != nil {
|
||||
log.Println("Failed to add cleanup scheduled task:", err)
|
||||
}
|
||||
if err := corn.AddFunc("records:minute", "@every 1m", minuteScheduledWork); err != nil {
|
||||
log.Println("Failed to add minute scheduled task:", err)
|
||||
}
|
||||
if err := corn.AddFunc("notifier:expire", "0 0 9 * * *", notifier.CheckExpireScheduledWork); err != nil {
|
||||
log.Println("Failed to add expire notification scheduled task:", err)
|
||||
}
|
||||
notifier.InitTrafficReportSchedule()
|
||||
}
|
||||
|
||||
func cleanupScheduledData() {
|
||||
cfg, _ := config.GetManyAs[config.Settings]()
|
||||
records.DeleteRecordBefore(time.Now().Add(-time.Hour * time.Duration(cfg.RecordPreserveTime)))
|
||||
records.CompactRecord()
|
||||
tasks.ClearTaskResultsByTimeBefore(time.Now().Add(-time.Hour * time.Duration(cfg.RecordPreserveTime)))
|
||||
tasks.DeletePingRecordsBefore(time.Now().Add(-time.Hour * time.Duration(cfg.PingRecordPreserveTime)))
|
||||
auditlog.RemoveOldLogs()
|
||||
accounts.RemoveExpiredSessions()
|
||||
}
|
||||
|
||||
func minuteScheduledWork() {
|
||||
cfg, _ := config.GetManyAs[config.Settings]()
|
||||
report_cache.SaveClientReportToDB()
|
||||
if !cfg.RecordEnabled {
|
||||
records.DeleteAll()
|
||||
tasks.DeleteAllPingRecords()
|
||||
}
|
||||
// 每分钟检查一次流量提醒
|
||||
notifier.CheckTraffic()
|
||||
}
|
||||
|
||||
func OnShutdown() {
|
||||
auditlog.Log("", "", "server is shutting down", "info")
|
||||
corn.StopAll()
|
||||
cloudflared.Shutdown()
|
||||
}
|
||||
|
||||
func OnFatal(err error) {
|
||||
auditlog.Log("", "", "server encountered a fatal error: "+err.Error(), "error")
|
||||
cloudflared.Shutdown()
|
||||
}
|
||||
@@ -100,8 +100,8 @@ func CreateDefaultAdminAccount() (username, passwd string, err error) {
|
||||
Username: username,
|
||||
Passwd: hashedPassword,
|
||||
SSOID: "",
|
||||
CreatedAt: models.FromTime(time.Now()),
|
||||
UpdatedAt: models.FromTime(time.Now()),
|
||||
CreatedAt: time.Now().UTC(),
|
||||
UpdatedAt: time.Now().UTC(),
|
||||
}
|
||||
|
||||
err = db.Create(&user).Error
|
||||
@@ -171,7 +171,7 @@ func UpdateUser(uuid string, name, password, sso_type *string) error {
|
||||
if sso_type != nil {
|
||||
updates["sso_type"] = *sso_type
|
||||
}
|
||||
updates["updated_at"] = time.Now()
|
||||
updates["updated_at"] = time.Now().UTC()
|
||||
err := db.Model(&models.User{}).Where("uuid = ?", uuid).Updates(updates).Error
|
||||
if err != nil {
|
||||
return err
|
||||
|
||||
@@ -33,11 +33,11 @@ func CreateSession(uuid string, expires int, userAgent, ip, login_method string)
|
||||
sessionRecord := models.Session{
|
||||
UUID: uuid,
|
||||
Session: session,
|
||||
Expires: models.FromTime(time.Now().Add(time.Duration(expires) * time.Second)),
|
||||
Expires: time.Now().UTC().Add(time.Duration(expires) * time.Second),
|
||||
UserAgent: userAgent,
|
||||
Ip: ip,
|
||||
LoginMethod: login_method,
|
||||
LatestOnline: models.FromTime(time.Now()),
|
||||
LatestOnline: time.Now().UTC(),
|
||||
}
|
||||
go func() {
|
||||
LoginNotification, _ := config.GetAs[bool](config.LoginNotificationKey, false)
|
||||
@@ -50,7 +50,7 @@ func CreateSession(uuid string, expires int, userAgent, ip, login_method string)
|
||||
}
|
||||
messageSender.SendEvent(models.EventMessage{
|
||||
Event: messageevent.Login,
|
||||
Time: time.Now(),
|
||||
Time: time.Now().UTC(),
|
||||
Message: fmt.Sprintf("%s: %s (%s)\n%s", login_method, ip, loc, userAgent),
|
||||
Emoji: "🔑",
|
||||
})
|
||||
@@ -73,7 +73,7 @@ func GetSession(session string) (uuid string, err error) {
|
||||
return "", err
|
||||
}
|
||||
|
||||
if time.Now().After(sessionRecord.Expires.ToTime()) {
|
||||
if time.Now().UTC().After(sessionRecord.Expires) {
|
||||
// 会话已过期,删除它
|
||||
_ = DeleteSession(session)
|
||||
return "", errors.New("session expired")
|
||||
@@ -111,24 +111,10 @@ func DeleteAllSessions() error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func UpdateLatestOnline(session string) error {
|
||||
db := dbcore.GetDBInstance()
|
||||
return db.Model(&models.Session{}).Where("session = ?", session).Update("latest_online", time.Now()).Error
|
||||
}
|
||||
|
||||
func UpdateLatestUserAgent(session, userAgent string) error {
|
||||
db := dbcore.GetDBInstance()
|
||||
return db.Model(&models.Session{}).Where("session = ?", session).Update("latest_user_agent", userAgent).Error
|
||||
}
|
||||
func UpdateLatestIp(session, ip string) error {
|
||||
db := dbcore.GetDBInstance()
|
||||
return db.Model(&models.Session{}).Where("session = ?", session).Update("latest_ip", ip).Error
|
||||
}
|
||||
|
||||
func UpdateLatest(session, useragent, ip string) error {
|
||||
db := dbcore.GetDBInstance()
|
||||
return db.Model(&models.Session{}).Where("session = ?", session).Updates(map[string]interface{}{
|
||||
"latest_online": time.Now(),
|
||||
"latest_online": time.Now().UTC(),
|
||||
"latest_user_agent": useragent,
|
||||
"latest_ip": ip,
|
||||
}).Error
|
||||
@@ -136,7 +122,7 @@ func UpdateLatest(session, useragent, ip string) error {
|
||||
|
||||
func RemoveExpiredSessions() error {
|
||||
db := dbcore.GetDBInstance()
|
||||
result := db.Where("expires < ?", time.Now()).Delete(&models.Session{})
|
||||
result := db.Where("expires < ?", time.Now().UTC()).Delete(&models.Session{})
|
||||
if result.Error != nil {
|
||||
return result.Error
|
||||
}
|
||||
|
||||
@@ -9,14 +9,13 @@ import (
|
||||
)
|
||||
|
||||
func Log(ip, uuid, message, msgType string) {
|
||||
now := time.Now()
|
||||
db := dbcore.GetDBInstance()
|
||||
logEntry := &models.Log{
|
||||
IP: ip,
|
||||
UUID: uuid,
|
||||
Message: message,
|
||||
MsgType: msgType,
|
||||
Time: models.FromTime(now),
|
||||
Time: time.Now().UTC(),
|
||||
}
|
||||
db.Create(logEntry)
|
||||
}
|
||||
@@ -28,7 +27,7 @@ func EventLog(eventType, message string) {
|
||||
// Delete logs older than 30 days
|
||||
func RemoveOldLogs() {
|
||||
db := dbcore.GetDBInstance()
|
||||
threshold := time.Now().AddDate(0, 0, -30)
|
||||
threshold := time.Now().UTC().AddDate(0, 0, -30)
|
||||
if err := db.Where("time < ?", threshold).Delete(&models.Log{}).Error; err != nil {
|
||||
log.Println("Failed to remove old logs:", err)
|
||||
}
|
||||
|
||||
+35
-38
@@ -1,6 +1,7 @@
|
||||
package clients
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"log"
|
||||
@@ -8,15 +9,20 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/komari-monitor/komari/database/dbcore"
|
||||
"github.com/komari-monitor/komari/database/metricstore"
|
||||
"github.com/komari-monitor/komari/database/models"
|
||||
"github.com/komari-monitor/komari/database/tasks"
|
||||
"github.com/komari-monitor/komari/utils"
|
||||
"gorm.io/gorm"
|
||||
|
||||
"github.com/google/uuid"
|
||||
)
|
||||
|
||||
func DeleteClient(clientUuid string) error {
|
||||
// Clean the independent metric store before removing the client from the main DB.
|
||||
if err := metricstore.DeleteEntity(context.Background(), clientUuid); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
db := dbcore.GetDBInstance()
|
||||
err := db.Delete(&models.Client{}, "uuid = ?", clientUuid).Error
|
||||
if err != nil {
|
||||
@@ -37,7 +43,7 @@ func SaveClientInfo(update map[string]interface{}) error {
|
||||
return fmt.Errorf("no fields to update")
|
||||
}
|
||||
|
||||
update["updated_at"] = time.Now()
|
||||
update["updated_at"] = time.Now().UTC()
|
||||
|
||||
toFloat64 := func(value interface{}) (float64, bool) {
|
||||
switch typed := value.(type) {
|
||||
@@ -122,24 +128,6 @@ func SaveClientInfo(update map[string]interface{}) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func EditClientName(clientUUID, clientName string) error {
|
||||
db := dbcore.GetDBInstance()
|
||||
err := db.Model(&models.Client{}).Where("uuid = ?", clientUUID).Update("name", clientName).Error
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func EditClientToken(clientUUID, token string) error {
|
||||
db := dbcore.GetDBInstance()
|
||||
err := db.Model(&models.Client{}).Where("uuid = ?", clientUUID).Update("token", token).Error
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// CreateClient 创建新客户端
|
||||
func CreateClient() (clientUUID, token string, err error) {
|
||||
db := dbcore.GetDBInstance()
|
||||
@@ -150,8 +138,8 @@ func CreateClient() (clientUUID, token string, err error) {
|
||||
UUID: clientUUID,
|
||||
Token: token,
|
||||
Name: "client_" + clientUUID[0:8],
|
||||
CreatedAt: models.FromTime(time.Now()),
|
||||
UpdatedAt: models.FromTime(time.Now()),
|
||||
CreatedAt: time.Now().UTC(),
|
||||
UpdatedAt: time.Now().UTC(),
|
||||
}
|
||||
|
||||
err = db.Create(&client).Error
|
||||
@@ -175,8 +163,8 @@ func CreateClientWithName(name string) (clientUUID, token string, err error) {
|
||||
UUID: clientUUID,
|
||||
Token: token,
|
||||
Name: name,
|
||||
CreatedAt: models.FromTime(time.Now()),
|
||||
UpdatedAt: models.FromTime(time.Now()),
|
||||
CreatedAt: time.Now().UTC(),
|
||||
UpdatedAt: time.Now().UTC(),
|
||||
}
|
||||
|
||||
err = db.Create(&client).Error
|
||||
@@ -210,19 +198,6 @@ func GetClientByUUID(uuid string) (client models.Client, err error) {
|
||||
return client, nil
|
||||
}
|
||||
|
||||
// GetClientBasicInfo 获取指定 UUID 的客户端基本信息
|
||||
func GetClientBasicInfo(uuid string) (client models.Client, err error) {
|
||||
db := dbcore.GetDBInstance()
|
||||
err = db.Where("uuid = ?", uuid).First(&client).Error
|
||||
if err != nil {
|
||||
if err == gorm.ErrRecordNotFound {
|
||||
return models.Client{}, fmt.Errorf("客户端不存在: %s", uuid)
|
||||
}
|
||||
return models.Client{}, err
|
||||
}
|
||||
return client, nil
|
||||
}
|
||||
|
||||
func GetClientTokenByUUID(uuid string) (token string, err error) {
|
||||
db := dbcore.GetDBInstance()
|
||||
var client models.Client
|
||||
@@ -261,8 +236,30 @@ func SaveClient(updates map[string]interface{}) error {
|
||||
}
|
||||
}
|
||||
}
|
||||
if value, exists := updates["expired_at"]; exists {
|
||||
switch typed := value.(type) {
|
||||
case nil:
|
||||
updates["expired_at"] = nil
|
||||
case time.Time:
|
||||
updates["expired_at"] = typed.UTC()
|
||||
case *time.Time:
|
||||
if typed == nil {
|
||||
updates["expired_at"] = nil
|
||||
} else {
|
||||
updates["expired_at"] = typed.UTC()
|
||||
}
|
||||
case string:
|
||||
stamp, err := time.Parse(time.RFC3339Nano, typed)
|
||||
if err != nil {
|
||||
return fmt.Errorf("expired_at must be an RFC3339 timestamp with a timezone: %w", err)
|
||||
}
|
||||
updates["expired_at"] = stamp.UTC()
|
||||
default:
|
||||
return fmt.Errorf("expired_at must be an RFC3339 timestamp with a timezone")
|
||||
}
|
||||
}
|
||||
|
||||
updates["updated_at"] = time.Now()
|
||||
updates["updated_at"] = time.Now().UTC()
|
||||
|
||||
err := db.Model(&models.Client{}).Where("uuid = ?", clientUUID).Updates(updates).Error
|
||||
if err != nil {
|
||||
|
||||
@@ -1,76 +1,14 @@
|
||||
package clients
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"math"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/komari-monitor/komari/database/dbcore"
|
||||
"github.com/komari-monitor/komari/database/models"
|
||||
v1 "github.com/komari-monitor/komari/protocol/v1"
|
||||
|
||||
"gorm.io/gorm"
|
||||
)
|
||||
|
||||
var reportSaveLocks sync.Map
|
||||
|
||||
func getReportSaveLock(clientUUID string) *sync.Mutex {
|
||||
lock, _ := reportSaveLocks.LoadOrStore(clientUUID, &sync.Mutex{})
|
||||
return lock.(*sync.Mutex)
|
||||
}
|
||||
|
||||
func computeTrafficDelta(current, previous int64) int64 {
|
||||
if previous < 0 {
|
||||
return 0
|
||||
}
|
||||
if current >= previous {
|
||||
return current - previous
|
||||
}
|
||||
// Counter reset or reboot: treat the new counter value as traffic since reset.
|
||||
return current
|
||||
}
|
||||
|
||||
func getLatestTrafficRecord(tx *gorm.DB, clientUUID string) (*models.Record, error) {
|
||||
var record models.Record
|
||||
err := tx.Where("client = ?", clientUUID).Order("time DESC").First(&record).Error
|
||||
if err == nil {
|
||||
return &record, nil
|
||||
}
|
||||
if !errors.Is(err, gorm.ErrRecordNotFound) {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
err = tx.Table("records_long_term").Where("client = ?", clientUUID).Order("time DESC").First(&record).Error
|
||||
if err == nil {
|
||||
return &record, nil
|
||||
}
|
||||
if !errors.Is(err, gorm.ErrRecordNotFound) {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
// Report 表示客户端报告数据
|
||||
// SaveReport 保存报告数据
|
||||
func SaveReport(uuid string, data map[string]interface{}) (err error) {
|
||||
|
||||
report, err := ParseReport(data)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
err = SaveClientReport(uuid, report)
|
||||
if err != nil {
|
||||
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
|
||||
}
|
||||
|
||||
func GetClientUUIDByToken(token string) (clientUUID string, err error) {
|
||||
db := dbcore.GetDBInstance()
|
||||
var client models.Client
|
||||
@@ -81,18 +19,6 @@ func GetClientUUIDByToken(token string) (clientUUID string, err error) {
|
||||
return client.UUID, nil
|
||||
}
|
||||
|
||||
func ParseReport(data map[string]interface{}) (report v1.Report, err error) {
|
||||
jsonData, err := json.Marshal(data)
|
||||
if err != nil {
|
||||
return v1.Report{}, err
|
||||
}
|
||||
err = json.Unmarshal(jsonData, &report)
|
||||
if err != nil {
|
||||
return v1.Report{}, err
|
||||
}
|
||||
return report, nil
|
||||
}
|
||||
|
||||
// 检查数据防止异常数据导致数据库损坏
|
||||
func ReportVerify(report v1.Report) error {
|
||||
// 防止输入不合理范围
|
||||
@@ -176,132 +102,3 @@ func ReportVerify(report v1.Report) error {
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// SaveClientReport 保存客户端报告到 Record 表
|
||||
func SaveClientReport(clientUUID string, report v1.Report) (err error) {
|
||||
db := dbcore.GetDBInstance()
|
||||
lock := getReportSaveLock(clientUUID)
|
||||
lock.Lock()
|
||||
defer lock.Unlock()
|
||||
|
||||
if err := ReportVerify(report); err != nil {
|
||||
return fmt.Errorf("failed to save Record: %w", err)
|
||||
}
|
||||
|
||||
// 保存GPU详细记录到独立表
|
||||
currentTime := time.Now()
|
||||
if report.GPU != nil && len(report.GPU.DetailedInfo) > 0 {
|
||||
for idx, gpu := range report.GPU.DetailedInfo {
|
||||
gpuRecord := models.GPURecord{
|
||||
Client: clientUUID,
|
||||
Time: models.FromTime(currentTime),
|
||||
DeviceIndex: idx,
|
||||
DeviceName: gpu.Name,
|
||||
MemTotal: gpu.MemoryTotal,
|
||||
MemUsed: gpu.MemoryUsed,
|
||||
Utilization: float32(gpu.Utilization),
|
||||
Temperature: gpu.Temperature,
|
||||
}
|
||||
if err := db.Create(&gpuRecord).Error; err != nil {
|
||||
return fmt.Errorf("failed to save GPU record: %w", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 计算平均GPU使用率,用于向后兼容
|
||||
averageGPUUsage := float32(0)
|
||||
if report.GPU != nil && len(report.GPU.DetailedInfo) > 0 {
|
||||
averageGPUUsage = float32(report.GPU.AverageUsage)
|
||||
}
|
||||
|
||||
Record := models.Record{
|
||||
Client: clientUUID,
|
||||
Time: models.FromTime(currentTime),
|
||||
Cpu: float32(report.CPU.Usage),
|
||||
Gpu: averageGPUUsage, // 使用平均GPU使用率
|
||||
Ram: report.Ram.Used,
|
||||
RamTotal: report.Ram.Total,
|
||||
Swap: report.Swap.Used,
|
||||
SwapTotal: report.Swap.Total,
|
||||
Load: float32(report.Load.Load1), // 使用 Load1 作为主要负载指标
|
||||
Temp: 0, // Report 未提供 TEMP,设为 0(与原 nil 行为类似)
|
||||
Disk: report.Disk.Used,
|
||||
DiskTotal: report.Disk.Total,
|
||||
NetIn: report.Network.Down,
|
||||
NetOut: report.Network.Up,
|
||||
NetTotalUp: report.Network.TotalUp,
|
||||
NetTotalDown: report.Network.TotalDown,
|
||||
TrafficUp: 0,
|
||||
TrafficDown: 0,
|
||||
Process: report.Process,
|
||||
Connections: report.Connections.TCP,
|
||||
ConnectionsUdp: report.Connections.UDP,
|
||||
//Uptime: report.Uptime,
|
||||
}
|
||||
|
||||
// 使用事务确保 Record 和 ClientsInfo 一致性
|
||||
err = db.Transaction(func(tx *gorm.DB) error {
|
||||
previous, err := getLatestTrafficRecord(tx, clientUUID)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to load previous Record: %w", err)
|
||||
}
|
||||
if previous != nil {
|
||||
Record.TrafficUp = computeTrafficDelta(report.Network.TotalUp, previous.NetTotalUp)
|
||||
Record.TrafficDown = computeTrafficDelta(report.Network.TotalDown, previous.NetTotalDown)
|
||||
}
|
||||
|
||||
// 保存 Record
|
||||
if err := tx.Create(&Record).Error; err != nil {
|
||||
return fmt.Errorf("failed to save Record: %w", err)
|
||||
}
|
||||
return nil
|
||||
})
|
||||
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
/*
|
||||
// getString 从 map 中获取字符串
|
||||
func getString(data map[string]interface{}, key string) string {
|
||||
if val, ok := data[key]; ok {
|
||||
if str, ok := val.(string); ok {
|
||||
return str
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// getInt 从 map 中获取整数
|
||||
func getInt(data map[string]interface{}, key string) int {
|
||||
if val, ok := data[key]; ok {
|
||||
if num, ok := val.(float64); ok {
|
||||
return int(num)
|
||||
}
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
// getInt64 从 map 中获取 int64
|
||||
func getInt64(data map[string]interface{}, key string) int64 {
|
||||
if val, ok := data[key]; ok {
|
||||
if num, ok := val.(float64); ok {
|
||||
return int64(num)
|
||||
}
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
// getFloat 从 map 中获取 float64
|
||||
func getFloat(data map[string]interface{}, key string) float64 {
|
||||
if val, ok := data[key]; ok {
|
||||
if num, ok := val.(float64); ok {
|
||||
return num
|
||||
}
|
||||
}
|
||||
return 0.0
|
||||
}
|
||||
*/
|
||||
@@ -1,82 +0,0 @@
|
||||
package clients
|
||||
|
||||
import (
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
)
|
||||
|
||||
func TestComputeTrafficDeltaHandlesZeroAndReset(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
current int64
|
||||
previous int64
|
||||
want int64
|
||||
}{
|
||||
{name: "previous zero counts current delta", current: 120, previous: 0, want: 120},
|
||||
{name: "monotonic counter uses difference", current: 250, previous: 200, want: 50},
|
||||
{name: "counter reset uses current", current: 15, previous: 250, want: 15},
|
||||
{name: "negative previous remains guarded", current: 15, previous: -1, want: 0},
|
||||
}
|
||||
|
||||
for _, test := range tests {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
assert.Equal(t, test.want, computeTrafficDelta(test.current, test.previous))
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestReportSaveLockSerializesSameClient(t *testing.T) {
|
||||
clientUUID := "client-lock-same"
|
||||
lock := getReportSaveLock(clientUUID)
|
||||
lock.Lock()
|
||||
defer lock.Unlock()
|
||||
|
||||
acquired := make(chan struct{})
|
||||
go func() {
|
||||
secondLock := getReportSaveLock(clientUUID)
|
||||
secondLock.Lock()
|
||||
defer secondLock.Unlock()
|
||||
close(acquired)
|
||||
}()
|
||||
|
||||
select {
|
||||
case <-acquired:
|
||||
t.Fatal("same-client report save lock was acquired while already held")
|
||||
case <-time.After(20 * time.Millisecond):
|
||||
}
|
||||
|
||||
lock.Unlock()
|
||||
select {
|
||||
case <-acquired:
|
||||
case <-time.After(time.Second):
|
||||
t.Fatal("same-client report save lock was not released")
|
||||
}
|
||||
lock.Lock()
|
||||
}
|
||||
|
||||
func TestReportSaveLockAllowsDifferentClients(t *testing.T) {
|
||||
firstLock := getReportSaveLock("client-lock-a")
|
||||
firstLock.Lock()
|
||||
defer firstLock.Unlock()
|
||||
|
||||
var waitGroup sync.WaitGroup
|
||||
waitGroup.Add(1)
|
||||
acquired := make(chan struct{})
|
||||
go func() {
|
||||
defer waitGroup.Done()
|
||||
secondLock := getReportSaveLock("client-lock-b")
|
||||
secondLock.Lock()
|
||||
defer secondLock.Unlock()
|
||||
close(acquired)
|
||||
}()
|
||||
|
||||
select {
|
||||
case <-acquired:
|
||||
case <-time.After(time.Second):
|
||||
t.Fatal("different-client report save lock should not block")
|
||||
}
|
||||
waitGroup.Wait()
|
||||
}
|
||||
@@ -1,6 +1,8 @@
|
||||
package clipboard
|
||||
|
||||
import (
|
||||
"time"
|
||||
|
||||
"github.com/komari-monitor/komari/database/dbcore"
|
||||
"github.com/komari-monitor/komari/database/models"
|
||||
)
|
||||
@@ -8,6 +10,8 @@ import (
|
||||
// CreateClipboard 创建剪贴板记录
|
||||
func CreateClipboard(cb *models.Clipboard) error {
|
||||
db := dbcore.GetDBInstance()
|
||||
cb.CreatedAt = time.Now().UTC()
|
||||
cb.UpdatedAt = cb.CreatedAt
|
||||
return db.Create(cb).Error
|
||||
}
|
||||
|
||||
@@ -24,6 +28,8 @@ func GetClipboardByID(id int) (*models.Clipboard, error) {
|
||||
// UpdateClipboardFields 更新剪贴板记录
|
||||
func UpdateClipboardFields(id int, fields map[string]interface{}) error {
|
||||
db := dbcore.GetDBInstance()
|
||||
delete(fields, "created_at")
|
||||
fields["updated_at"] = time.Now().UTC()
|
||||
return db.Model(&models.Clipboard{}).Where("id = ?", id).Updates(fields).Error
|
||||
}
|
||||
|
||||
|
||||
+290
-131
@@ -168,141 +168,300 @@ func unzipToDir(zipPath, dstDir string) error {
|
||||
var (
|
||||
instance *gorm.DB
|
||||
once sync.Once
|
||||
initErr error
|
||||
)
|
||||
|
||||
func GetDBInstance() *gorm.DB {
|
||||
// SystemVersionKey 是记录“上次启动所用版本标识”的配置键(存于 configs 表)。
|
||||
// 取代旧的 ./data/.komari-version 文件:版本标识随配置库一起备份/恢复,
|
||||
// 也避免额外的裸文件依赖。
|
||||
const SystemVersionKey = "system_version"
|
||||
|
||||
// versionID 是当前构建的版本标识,由 SetVersionID 在 Initialize 前注入。
|
||||
var versionID string
|
||||
|
||||
// dbFileExistedAtStartup 记录本次进程启动、打开数据库之前 komari.db 是否已存在,
|
||||
// 用于区分“全新安装”与“从旧版升级(无版本标记)”。在 doInitialize 打开数据库
|
||||
// 之前采集。
|
||||
var dbFileExistedAtStartup bool
|
||||
|
||||
// SetVersionID 设置当前构建的版本标识(通常为 CurrentVersion+"-"+VersionHash),
|
||||
// 用于版本升级检测与自动备份。应在 Initialize() 之前调用;为空则跳过升级备份。
|
||||
func SetVersionID(id string) {
|
||||
versionID = id
|
||||
}
|
||||
|
||||
// resolveDatabaseFile 返回当前使用的 SQLite 数据库文件路径。
|
||||
func resolveDatabaseFile() string {
|
||||
dbFile := flags.DatabaseFile
|
||||
if dbFile == "" {
|
||||
dbFile = "./data/komari.db"
|
||||
}
|
||||
return dbFile
|
||||
}
|
||||
|
||||
// backupOnVersionUpgrade 在检测到版本升级时,把当前 ./data 打包到
|
||||
// ./backup/upgrade-{time}.zip,便于升级(含 metrics 迁移)异常时回滚。
|
||||
//
|
||||
// 版本标识存放于配置库(configs 表,键 system_version),因此本函数必须在
|
||||
// config.SetDb 之后、一次性 metrics 迁移(InitStores)之前调用。
|
||||
//
|
||||
// 触发规则:
|
||||
// - versionID 为空:跳过(未注入版本,如部分测试场景)。
|
||||
// - 配置中无版本且启动前无数据库文件:全新安装,仅写版本,不备份。
|
||||
// - 配置中无版本但启动前已有数据库文件:从无版本标记的旧稳定版升级,备份。
|
||||
// - 配置中版本与当前不同:版本升级,备份。
|
||||
// - 配置中版本与当前一致:无需备份。
|
||||
//
|
||||
// 备份失败不阻止启动,但打印明确错误;备份成功(或无需备份)后写入/更新版本。
|
||||
func backupOnVersionUpgrade() {
|
||||
if versionID == "" {
|
||||
return
|
||||
}
|
||||
|
||||
prevVersion, readErr := config.GetAs[string](SystemVersionKey)
|
||||
prevVersion = strings.TrimSpace(prevVersion)
|
||||
versionRecorded := readErr == nil && prevVersion != ""
|
||||
|
||||
// 版本未变化,无需备份。
|
||||
if versionRecorded && prevVersion == versionID {
|
||||
return
|
||||
}
|
||||
|
||||
// 全新安装:配置中无版本且启动前无数据库文件,直接写版本不备份。
|
||||
if !versionRecorded && !dbFileExistedAtStartup {
|
||||
writeVersionMarker()
|
||||
return
|
||||
}
|
||||
|
||||
// 需要备份(升级或从旧稳定版首次带版本标记启动)。
|
||||
// 先做一次 WAL checkpoint,确保 komari.db 主文件包含最新数据,
|
||||
// 避免备份出的库缺少仍留在 -wal 中的写入。
|
||||
if instance != nil {
|
||||
instance.Exec("PRAGMA wal_checkpoint(TRUNCATE);")
|
||||
}
|
||||
|
||||
if err := os.MkdirAll("./backup", 0755); err != nil {
|
||||
log.Printf("[upgrade-backup] failed to create backup dir: %v", err)
|
||||
return
|
||||
}
|
||||
tsName := time.Now().UTC().Format("20060102-150405")
|
||||
bakPath := filepath.Join("./backup", fmt.Sprintf("upgrade-%s.zip", tsName))
|
||||
backupZipPath := filepath.Join(".", "data", "backup.zip")
|
||||
if zipErr := zipDirectoryExcluding("./data", bakPath, map[string]struct{}{backupZipPath: {}}); zipErr != nil {
|
||||
log.Printf("[upgrade-backup] failed to backup ./data before upgrade (from %q to %q): %v", prevVersion, versionID, zipErr)
|
||||
return
|
||||
}
|
||||
log.Printf("[upgrade-backup] ./data backed up to %s before upgrade (from %q to %q)", bakPath, prevVersion, versionID)
|
||||
|
||||
writeVersionMarker()
|
||||
}
|
||||
|
||||
// writeVersionMarker 将当前 versionID 写入配置库。
|
||||
func writeVersionMarker() {
|
||||
if err := config.Set(SystemVersionKey, versionID); err != nil {
|
||||
log.Printf("[upgrade-backup] failed to persist version marker: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func buildSQLiteDSN(databaseFile string) string {
|
||||
if databaseFile == "" {
|
||||
databaseFile = "./data/komari.db"
|
||||
}
|
||||
|
||||
params := "_busy_timeout=5000&_txlock=immediate&_journal_mode=WAL&_synchronous=NORMAL"
|
||||
separator := "?"
|
||||
if strings.Contains(databaseFile, "?") {
|
||||
separator = "&"
|
||||
}
|
||||
|
||||
if strings.HasPrefix(databaseFile, "file:") {
|
||||
return databaseFile + separator + params
|
||||
}
|
||||
|
||||
if databaseFile == ":memory:" {
|
||||
return "file::memory:?cache=shared&" + params
|
||||
}
|
||||
|
||||
return "file:" + filepath.ToSlash(databaseFile) + separator + params
|
||||
}
|
||||
|
||||
// Initialize 显式初始化数据库连接与表结构,仅执行一次。
|
||||
// 与 GetDBInstance 不同,Initialize 返回错误而非直接退出进程,
|
||||
// 便于启动生命周期统一处理错误、以及在测试/CLI 命令中做隔离。
|
||||
func Initialize() error {
|
||||
once.Do(func() {
|
||||
|
||||
var err error
|
||||
|
||||
// 在数据库初始化前执行:如果存在 ./data/backup.zip,则进行恢复逻辑
|
||||
func() {
|
||||
backupZipPath := filepath.Join(".", "data", "backup.zip")
|
||||
if _, statErr := os.Stat(backupZipPath); statErr == nil {
|
||||
// 4. 把除了 ./data/backup.zip 之外的所有文件压缩到 ./backup/{time}.zip
|
||||
if err := os.MkdirAll("./backup", 0755); err != nil {
|
||||
log.Printf("[restore] failed to create backup dir: %v", err)
|
||||
} else {
|
||||
tsName := time.Now().Format("20060102-150405")
|
||||
bakPath := filepath.Join("./backup", fmt.Sprintf("%s.zip", tsName))
|
||||
if zipErr := zipDirectoryExcluding("./data", bakPath, map[string]struct{}{backupZipPath: {}}); zipErr != nil {
|
||||
log.Printf("[restore] failed to zip current data: %v", zipErr)
|
||||
} else {
|
||||
log.Printf("[restore] current data zipped to %s", bakPath)
|
||||
}
|
||||
}
|
||||
|
||||
// 5. 删除除了 ./data/backup.zip 之外的所有文件
|
||||
if delErr := removeAllInDirExcept("./data", map[string]struct{}{backupZipPath: {}}); delErr != nil {
|
||||
log.Printf("[restore] failed to cleanup data dir: %v", delErr)
|
||||
}
|
||||
|
||||
// 6. 解压 ./data/backup.zip 到 ./data
|
||||
if unzipErr := unzipToDir(backupZipPath, "./data"); unzipErr != nil {
|
||||
log.Printf("[restore] failed to unzip backup into data: %v", unzipErr)
|
||||
} else {
|
||||
log.Printf("[restore] backup.zip extracted to ./data")
|
||||
}
|
||||
|
||||
// 7. 删除 ./data/backup.zip
|
||||
if rmErr := os.Remove(backupZipPath); rmErr != nil {
|
||||
log.Printf("[restore] failed to remove backup.zip: %v", rmErr)
|
||||
} else {
|
||||
log.Printf("[restore] backup.zip removed")
|
||||
}
|
||||
// 8. 删除标记
|
||||
if rmErr := os.Remove("./data/komari-backup-markup"); rmErr != nil {
|
||||
log.Printf("[restore] failed to remove komari-backup-markup: %v", rmErr)
|
||||
} else {
|
||||
log.Printf("[restore] komari-backup-markup removed")
|
||||
}
|
||||
}
|
||||
}()
|
||||
|
||||
logConfig := &gorm.Config{
|
||||
Logger: logutil.NewGormLogger(),
|
||||
}
|
||||
|
||||
// 根据数据库类型选择不同的连接方式
|
||||
switch flags.ApplyDatabaseTypeNormalization() {
|
||||
case flags.DatabaseTypeSQLite:
|
||||
// SQLite 连接
|
||||
instance, err = gorm.Open(sqlite.Open(flags.DatabaseFile), logConfig)
|
||||
if err != nil {
|
||||
log.Fatalf("Failed to connect to SQLite3 database: %v", err)
|
||||
}
|
||||
log.Printf("Using SQLite database file: %s", flags.DatabaseFile)
|
||||
instance.Exec("PRAGMA wal = ON;")
|
||||
if err := instance.Exec("PRAGMA journal_mode = WAL;").Error; err != nil {
|
||||
log.Printf("Failed to enable WAL mode for SQLite: %v", err)
|
||||
}
|
||||
instance.Exec("PRAGMA synchronous = NORMAL;")
|
||||
instance.Exec("PRAGMA cache_size = -65536;")
|
||||
instance.Exec("PRAGMA temp_store = MEMORY;")
|
||||
instance.Exec("PRAGMA wal_checkpoint(TRUNCATE);")
|
||||
default:
|
||||
log.Fatalf("Unsupported database type: %s (supported: %s)", flags.DatabaseType, flags.SupportedDatabaseTypes())
|
||||
}
|
||||
if err := migrations.Run(migrations.Context{DB: instance}); err != nil {
|
||||
log.Fatalf("Failed to run startup migrations: %v", err)
|
||||
}
|
||||
config.SetDb(instance)
|
||||
// 自动迁移模型
|
||||
err = instance.AutoMigrate(
|
||||
&models.User{},
|
||||
&models.Client{},
|
||||
&models.Record{},
|
||||
&models.GPURecord{},
|
||||
&models.Log{},
|
||||
&models.Clipboard{},
|
||||
&models.LoadNotification{},
|
||||
&models.OfflineNotification{},
|
||||
&models.TrafficReportNotification{},
|
||||
&models.PingRecord{},
|
||||
&models.PingTask{},
|
||||
&models.OidcProvider{},
|
||||
&models.MessageSenderProvider{},
|
||||
&models.ThemeConfiguration{},
|
||||
)
|
||||
if err != nil {
|
||||
log.Fatalf("Failed to create tables: %v", err)
|
||||
}
|
||||
err = instance.Table("records_long_term").AutoMigrate(
|
||||
&models.Record{},
|
||||
)
|
||||
if err != nil {
|
||||
log.Printf("Failed to create records_long_term table, it may already exist: %v", err)
|
||||
}
|
||||
err = instance.Table("gpu_records_long_term").AutoMigrate(
|
||||
&models.GPURecord{},
|
||||
)
|
||||
if err != nil {
|
||||
log.Printf("Failed to create gpu_records_long_term table, it may already exist: %v", err)
|
||||
}
|
||||
err = instance.AutoMigrate(
|
||||
&models.Session{},
|
||||
)
|
||||
if err != nil {
|
||||
log.Printf("Failed to create Session table, it may already exist: %v", err)
|
||||
}
|
||||
err = instance.AutoMigrate(
|
||||
&models.Task{},
|
||||
&models.TaskResult{},
|
||||
)
|
||||
if err != nil {
|
||||
log.Printf("Failed to create Task and TaskResult table, it may already exist: %v", err)
|
||||
}
|
||||
|
||||
// Manually create composite indexes
|
||||
if flags.IsSQLite() {
|
||||
instance.Exec("CREATE INDEX IF NOT EXISTS idx_record_client_time ON records(client, time)")
|
||||
instance.Exec("CREATE INDEX IF NOT EXISTS idx_record_lt_client_time ON records_long_term(client, time)")
|
||||
instance.Exec("CREATE INDEX IF NOT EXISTS idx_gpu_record_client_time ON gpu_records(client, time)")
|
||||
instance.Exec("CREATE INDEX IF NOT EXISTS idx_gpu_record_lt_client_time ON gpu_records_long_term(client, time)")
|
||||
instance.Exec("CREATE INDEX IF NOT EXISTS idx_ping_record_client_time ON ping_records(client, time)")
|
||||
}
|
||||
|
||||
initErr = doInitialize()
|
||||
})
|
||||
return initErr
|
||||
}
|
||||
|
||||
// GetDBInstance 返回全局数据库实例。
|
||||
// 为兼容既有大量调用点,这里保留“出错即退出”的语义;
|
||||
// 需要错误处理的启动流程应优先调用 Initialize()。
|
||||
func GetDBInstance() *gorm.DB {
|
||||
if err := Initialize(); err != nil {
|
||||
log.Fatalf("Failed to initialize database: %v", err)
|
||||
}
|
||||
return instance
|
||||
}
|
||||
|
||||
// Close 关闭底层数据库连接,供关闭流程调用。
|
||||
func Close() error {
|
||||
if instance == nil {
|
||||
return nil
|
||||
}
|
||||
sqlDB, err := instance.DB()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return sqlDB.Close()
|
||||
}
|
||||
|
||||
func doInitialize() error {
|
||||
var err error
|
||||
|
||||
// 在数据库初始化前执行:如果存在 ./data/backup.zip,则进行恢复逻辑
|
||||
func() {
|
||||
backupZipPath := filepath.Join(".", "data", "backup.zip")
|
||||
if _, statErr := os.Stat(backupZipPath); statErr == nil {
|
||||
// 4. 把除了 ./data/backup.zip 之外的所有文件压缩到 ./backup/{time}.zip
|
||||
if err := os.MkdirAll("./backup", 0755); err != nil {
|
||||
log.Printf("[restore] failed to create backup dir: %v", err)
|
||||
} else {
|
||||
tsName := time.Now().UTC().Format("20060102-150405")
|
||||
bakPath := filepath.Join("./backup", fmt.Sprintf("%s.zip", tsName))
|
||||
if zipErr := zipDirectoryExcluding("./data", bakPath, map[string]struct{}{backupZipPath: {}}); zipErr != nil {
|
||||
log.Printf("[restore] failed to zip current data: %v", zipErr)
|
||||
} else {
|
||||
log.Printf("[restore] current data zipped to %s", bakPath)
|
||||
}
|
||||
}
|
||||
|
||||
// 5. 删除除了 ./data/backup.zip 之外的所有文件
|
||||
if delErr := removeAllInDirExcept("./data", map[string]struct{}{backupZipPath: {}}); delErr != nil {
|
||||
log.Printf("[restore] failed to cleanup data dir: %v", delErr)
|
||||
}
|
||||
|
||||
// 6. 解压 ./data/backup.zip 到 ./data
|
||||
if unzipErr := unzipToDir(backupZipPath, "./data"); unzipErr != nil {
|
||||
log.Printf("[restore] failed to unzip backup into data: %v", unzipErr)
|
||||
} else {
|
||||
log.Printf("[restore] backup.zip extracted to ./data")
|
||||
}
|
||||
|
||||
// 7. 删除 ./data/backup.zip
|
||||
if rmErr := os.Remove(backupZipPath); rmErr != nil {
|
||||
log.Printf("[restore] failed to remove backup.zip: %v", rmErr)
|
||||
} else {
|
||||
log.Printf("[restore] backup.zip removed")
|
||||
}
|
||||
// 8. 删除标记
|
||||
if rmErr := os.Remove("./data/komari-backup-markup"); rmErr != nil {
|
||||
log.Printf("[restore] failed to remove komari-backup-markup: %v", rmErr)
|
||||
} else {
|
||||
log.Printf("[restore] komari-backup-markup removed")
|
||||
}
|
||||
}
|
||||
}()
|
||||
|
||||
// 记录“打开数据库之前”komari.db 是否已存在,用于区分全新安装与旧版升级。
|
||||
// 必须在(可能的)恢复逻辑之后、gorm.Open 之前采集:恢复会解压出旧库,
|
||||
// gorm.Open 会创建空库。
|
||||
if _, statErr := os.Stat(resolveDatabaseFile()); statErr == nil {
|
||||
dbFileExistedAtStartup = true
|
||||
}
|
||||
|
||||
logConfig := &gorm.Config{
|
||||
Logger: logutil.NewGormLogger(),
|
||||
NowFunc: func() time.Time { return time.Now().UTC() },
|
||||
}
|
||||
|
||||
// 根据数据库类型选择不同的连接方式
|
||||
switch flags.ApplyDatabaseTypeNormalization() {
|
||||
case flags.DatabaseTypeSQLite:
|
||||
// SQLite 连接
|
||||
// 通过 DSN 传入 _busy_timeout / _txlock 等参数,确保连接池中的每一条连接
|
||||
// 都生效:
|
||||
// - _busy_timeout=5000:遇到写锁时最多等待 5s 再返回,避免瞬时
|
||||
// "database is locked" 直接失败(仅靠后续 PRAGMA Exec 只作用于
|
||||
// 当时执行该语句的单条连接,池内其它连接不生效)。
|
||||
// - _txlock=immediate:事务一开始即获取写锁,避免「先 SELECT 后写」
|
||||
// 的锁升级在并发写入下产生死锁式的立即 SQLITE_BUSY。
|
||||
// - _journal_mode=WAL / _synchronous=NORMAL:与下方 PRAGMA 保持一致,
|
||||
// 在 DSN 层为所有连接预设。
|
||||
dsn := buildSQLiteDSN(flags.DatabaseFile)
|
||||
instance, err = gorm.Open(sqlite.Open(dsn), logConfig)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to connect to SQLite3 database: %w", err)
|
||||
}
|
||||
log.Printf("Using SQLite database file: %s", flags.DatabaseFile)
|
||||
if sqlDB, dbErr := instance.DB(); dbErr == nil {
|
||||
// SQLite 同一时刻只允许一个写者;限制连接数可避免连接池层面的写写竞争。
|
||||
// 负载历史每分钟会执行包含读和写的事务,若连接池允许多个连接,容易与
|
||||
// ping 结果等短写入撞锁并导致整批负载记录回滚。
|
||||
sqlDB.SetMaxOpenConns(1)
|
||||
sqlDB.SetMaxIdleConns(1)
|
||||
sqlDB.SetConnMaxLifetime(0)
|
||||
} else {
|
||||
log.Printf("Failed to access underlying sql.DB for SQLite tuning: %v", dbErr)
|
||||
}
|
||||
instance.Exec("PRAGMA wal = ON;")
|
||||
if err := instance.Exec("PRAGMA journal_mode = WAL;").Error; err != nil {
|
||||
log.Printf("Failed to enable WAL mode for SQLite: %v", err)
|
||||
}
|
||||
instance.Exec("PRAGMA synchronous = NORMAL;")
|
||||
instance.Exec("PRAGMA busy_timeout = 5000;")
|
||||
instance.Exec("PRAGMA cache_size = -65536;")
|
||||
instance.Exec("PRAGMA temp_store = MEMORY;")
|
||||
instance.Exec("PRAGMA wal_checkpoint(TRUNCATE);")
|
||||
default:
|
||||
return fmt.Errorf("unsupported database type: %s (supported: %s)", flags.DatabaseType, flags.SupportedDatabaseTypes())
|
||||
}
|
||||
if err := migrations.Run(migrations.Context{DB: instance}); err != nil {
|
||||
return fmt.Errorf("failed to run startup migrations: %w", err)
|
||||
}
|
||||
config.SetDb(instance)
|
||||
|
||||
// 配置库就绪后、执行后续 AutoMigrate 与一次性 metrics 迁移之前:
|
||||
// 基于配置中的版本标记检测升级并自动备份 ./data,便于回滚。
|
||||
backupOnVersionUpgrade()
|
||||
|
||||
// 自动迁移模型
|
||||
//
|
||||
// 注意:负载/GPU/ping 历史监控数据运行期全部走 metric store(默认 SQLite
|
||||
// ./data/metrics.db,或配置的 MySQL/PostgreSQL)。旧的 records /
|
||||
// records_long_term / gpu_records / ping_records 表不再建表、不再写入。
|
||||
// 若升级时旧表仍存在,会在 pkg/migrations.RunMetricStoreMigrations 中先导入再清理。
|
||||
// models.Record / models.PingRecord / models.GPURecord 结构体仍作为
|
||||
// metric store 的读写 DTO 和旧表导入 DTO 保留在 models 包中。
|
||||
|
||||
err = instance.AutoMigrate(
|
||||
&models.User{},
|
||||
&models.Client{},
|
||||
&models.Log{},
|
||||
&models.Clipboard{},
|
||||
&models.LoadNotification{},
|
||||
&models.OfflineNotification{},
|
||||
&models.TrafficReportNotification{},
|
||||
&models.PingTask{},
|
||||
&models.OidcProvider{},
|
||||
&models.MessageSenderProvider{},
|
||||
&models.ThemeConfiguration{},
|
||||
)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to create tables: %w", err)
|
||||
}
|
||||
if err := instance.AutoMigrate(
|
||||
&models.Session{},
|
||||
); err != nil {
|
||||
log.Printf("Failed to create Session table, it may already exist: %v", err)
|
||||
}
|
||||
if err := instance.AutoMigrate(
|
||||
&models.Task{},
|
||||
&models.TaskResult{},
|
||||
); err != nil {
|
||||
log.Printf("Failed to create Task and TaskResult table, it may already exist: %v", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,92 @@
|
||||
package dbcore
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"strings"
|
||||
"sync"
|
||||
|
||||
"github.com/komari-monitor/komari/cmd/flags"
|
||||
)
|
||||
|
||||
var maintenanceMu sync.Mutex
|
||||
|
||||
// StorageSize returns the bytes occupied by the main SQLite database and its
|
||||
// WAL/SHM sidecar files.
|
||||
func StorageSize() (int64, error) {
|
||||
if !flags.IsSQLite() {
|
||||
return 0, errors.New("main database size is only available for SQLite")
|
||||
}
|
||||
return sqliteFileSetSize(resolveDatabaseFile())
|
||||
}
|
||||
|
||||
// ReclaimSpace checkpoints the main database WAL and rewrites the SQLite file.
|
||||
func ReclaimSpace(ctx context.Context) error {
|
||||
if !flags.IsSQLite() {
|
||||
return errors.New("main database maintenance is only supported for SQLite")
|
||||
}
|
||||
|
||||
maintenanceMu.Lock()
|
||||
defer maintenanceMu.Unlock()
|
||||
|
||||
db, err := GetDBInstance().DB()
|
||||
if err != nil {
|
||||
return fmt.Errorf("get main database connection: %w", err)
|
||||
}
|
||||
if err := checkpointSQLiteWAL(ctx, db); err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := db.ExecContext(ctx, "VACUUM"); err != nil {
|
||||
return fmt.Errorf("vacuum main database: %w", err)
|
||||
}
|
||||
return checkpointSQLiteWAL(ctx, db)
|
||||
}
|
||||
|
||||
func checkpointSQLiteWAL(ctx context.Context, db *sql.DB) error {
|
||||
var busy, logFrames, checkpointedFrames int
|
||||
if err := db.QueryRowContext(ctx, "PRAGMA wal_checkpoint(TRUNCATE)").Scan(&busy, &logFrames, &checkpointedFrames); err != nil {
|
||||
return fmt.Errorf("checkpoint main database WAL: %w", err)
|
||||
}
|
||||
if busy != 0 {
|
||||
return fmt.Errorf(
|
||||
"checkpoint main database WAL: database is busy (%d log frames, %d checkpointed)",
|
||||
logFrames,
|
||||
checkpointedFrames,
|
||||
)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func sqliteFileSetSize(dsn string) (int64, error) {
|
||||
path := strings.TrimPrefix(strings.TrimSpace(dsn), "file:")
|
||||
if index := strings.IndexByte(path, '?'); index >= 0 {
|
||||
path = path[:index]
|
||||
}
|
||||
if path == "" || path == ":memory:" || strings.Contains(strings.ToLower(dsn), "mode=memory") {
|
||||
return 0, errors.New("database is not backed by a local file")
|
||||
}
|
||||
|
||||
var total int64
|
||||
foundDatabase := false
|
||||
for _, suffix := range []string{"", "-wal", "-shm"} {
|
||||
info, err := os.Stat(path + suffix)
|
||||
switch {
|
||||
case err == nil:
|
||||
total += info.Size()
|
||||
if suffix == "" {
|
||||
foundDatabase = true
|
||||
}
|
||||
case errors.Is(err, os.ErrNotExist):
|
||||
continue
|
||||
default:
|
||||
return 0, fmt.Errorf("stat database file %q: %w", path+suffix, err)
|
||||
}
|
||||
}
|
||||
if !foundDatabase {
|
||||
return 0, fmt.Errorf("database file %q does not exist", path)
|
||||
}
|
||||
return total, nil
|
||||
}
|
||||
@@ -0,0 +1,37 @@
|
||||
package dbcore
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestSQLiteFileSetSizeIncludesWALAndSHM(t *testing.T) {
|
||||
databasePath := filepath.Join(t.TempDir(), "komari.db")
|
||||
files := map[string]int{
|
||||
databasePath: 11,
|
||||
databasePath + "-wal": 7,
|
||||
databasePath + "-shm": 5,
|
||||
}
|
||||
for path, size := range files {
|
||||
if err := os.WriteFile(path, make([]byte, size), 0o600); err != nil {
|
||||
t.Fatalf("write fixture %s: %v", path, err)
|
||||
}
|
||||
}
|
||||
|
||||
got, err := sqliteFileSetSize("file:" + filepath.ToSlash(databasePath) + "?mode=rwc")
|
||||
if err != nil {
|
||||
t.Fatalf("sqliteFileSetSize: %v", err)
|
||||
}
|
||||
if got != 23 {
|
||||
t.Fatalf("size = %d, want 23", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSQLiteFileSetSizeRejectsNonFileDatabase(t *testing.T) {
|
||||
for _, dsn := range []string{":memory:", "file:metrics?MODE=MEMORY&cache=shared"} {
|
||||
if _, err := sqliteFileSetSize(dsn); err == nil {
|
||||
t.Fatalf("sqliteFileSetSize(%q) should fail", dsn)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,95 @@
|
||||
package metricstore
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
|
||||
"github.com/komari-monitor/komari/pkg/metric"
|
||||
)
|
||||
|
||||
var (
|
||||
ErrStoreNotInitialized = errors.New("metric store not initialized")
|
||||
ErrStoreBusy = errors.New("metric store is busy")
|
||||
)
|
||||
|
||||
// StorageInfo describes the physical storage owned by the active metric store.
|
||||
// Size remains useful even when the store points at an external database because
|
||||
// pkg/metric limits its query to the three tables managed by this Store.
|
||||
type StorageInfo struct {
|
||||
Driver metric.Driver
|
||||
Action metric.MaintenanceAction
|
||||
Size int64
|
||||
}
|
||||
|
||||
// MaintenanceResult keeps measurement failures separate from the maintenance
|
||||
// error. A database may allow table maintenance while denying catalog queries,
|
||||
// and callers should still be able to report that the operation succeeded.
|
||||
type MaintenanceResult struct {
|
||||
Driver metric.Driver
|
||||
Action metric.MaintenanceAction
|
||||
Before int64
|
||||
After int64
|
||||
BeforeSizeError error
|
||||
AfterSizeError error
|
||||
}
|
||||
|
||||
// InspectStorage reads physical storage information while preventing a store
|
||||
// reload from closing the active connection underneath the query.
|
||||
func InspectStorage(ctx context.Context) (StorageInfo, error) {
|
||||
if err := storeOperations.Acquire(ctx); err != nil {
|
||||
return StorageInfo{}, fmt.Errorf("wait for metric store operations before inspection: %w", err)
|
||||
}
|
||||
defer storeOperations.Release()
|
||||
|
||||
storeMu.RLock()
|
||||
activeStore := store
|
||||
storeMu.RUnlock()
|
||||
if activeStore == nil {
|
||||
return StorageInfo{}, ErrStoreNotInitialized
|
||||
}
|
||||
|
||||
info := StorageInfo{
|
||||
Driver: activeStore.Driver(),
|
||||
Action: activeStore.MaintenanceAction(),
|
||||
}
|
||||
size, err := activeStore.StorageSize(ctx)
|
||||
info.Size = size
|
||||
return info, err
|
||||
}
|
||||
|
||||
// ReclaimSpace performs the driver-specific physical maintenance operation.
|
||||
// It shares the exclusive operation lock with Compact so retention compaction
|
||||
// and a table/file rewrite can never run against the metric store concurrently.
|
||||
func ReclaimSpace(ctx context.Context) (MaintenanceResult, error) {
|
||||
if !storeOperations.TryAcquire() {
|
||||
storeMu.RLock()
|
||||
defer storeMu.RUnlock()
|
||||
if store == nil {
|
||||
return MaintenanceResult{}, ErrStoreNotInitialized
|
||||
}
|
||||
return MaintenanceResult{
|
||||
Driver: store.Driver(),
|
||||
Action: store.MaintenanceAction(),
|
||||
BeforeSizeError: ErrStoreBusy,
|
||||
AfterSizeError: ErrStoreBusy,
|
||||
}, ErrStoreBusy
|
||||
}
|
||||
defer storeOperations.Release()
|
||||
|
||||
storeMu.RLock()
|
||||
activeStore := store
|
||||
storeMu.RUnlock()
|
||||
if activeStore == nil {
|
||||
return MaintenanceResult{}, ErrStoreNotInitialized
|
||||
}
|
||||
|
||||
result := MaintenanceResult{
|
||||
Driver: activeStore.Driver(),
|
||||
Action: activeStore.MaintenanceAction(),
|
||||
}
|
||||
result.Before, result.BeforeSizeError = activeStore.StorageSize(ctx)
|
||||
maintenanceErr := activeStore.ReclaimSpace(ctx)
|
||||
result.After, result.AfterSizeError = activeStore.StorageSize(ctx)
|
||||
return result, maintenanceErr
|
||||
}
|
||||
@@ -0,0 +1,167 @@
|
||||
package metricstore
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/komari-monitor/komari/pkg/metric"
|
||||
)
|
||||
|
||||
func TestInspectAndReclaimStorage(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
s, err := metric.Open(ctx, metric.SQLite(":memory:"))
|
||||
if err != nil {
|
||||
t.Fatalf("open metric store: %v", err)
|
||||
}
|
||||
installTestStore(t, s)
|
||||
|
||||
info, err := InspectStorage(ctx)
|
||||
if err != nil {
|
||||
t.Fatalf("inspect storage: %v", err)
|
||||
}
|
||||
if info.Driver != metric.DriverSQLite || info.Action != metric.MaintenanceVacuum {
|
||||
t.Fatalf("unexpected storage info: %#v", info)
|
||||
}
|
||||
if info.Size != 0 {
|
||||
t.Fatalf("in-memory storage size = %d, want 0", info.Size)
|
||||
}
|
||||
|
||||
result, err := ReclaimSpace(ctx)
|
||||
if err != nil {
|
||||
t.Fatalf("reclaim space: %v", err)
|
||||
}
|
||||
if result.Driver != metric.DriverSQLite || result.Action != metric.MaintenanceVacuum {
|
||||
t.Fatalf("unexpected maintenance result: %#v", result)
|
||||
}
|
||||
if result.BeforeSizeError != nil || result.AfterSizeError != nil {
|
||||
t.Fatalf("unexpected size errors: before=%v after=%v", result.BeforeSizeError, result.AfterSizeError)
|
||||
}
|
||||
}
|
||||
|
||||
func TestReclaimSpaceReportsBusyStore(t *testing.T) {
|
||||
s, err := metric.Open(context.Background(), metric.SQLite(":memory:"))
|
||||
if err != nil {
|
||||
t.Fatalf("open metric store: %v", err)
|
||||
}
|
||||
installTestStore(t, s)
|
||||
|
||||
if !storeOperations.TryAcquire() {
|
||||
t.Fatal("acquire store operation gate")
|
||||
}
|
||||
defer storeOperations.Release()
|
||||
|
||||
result, err := ReclaimSpace(context.Background())
|
||||
if !errors.Is(err, ErrStoreBusy) {
|
||||
t.Fatalf("reclaim error = %v, want %v", err, ErrStoreBusy)
|
||||
}
|
||||
if result.Driver != metric.DriverSQLite || result.Action != metric.MaintenanceVacuum {
|
||||
t.Fatalf("busy result lost store metadata: %#v", result)
|
||||
}
|
||||
if !errors.Is(result.BeforeSizeError, ErrStoreBusy) || !errors.Is(result.AfterSizeError, ErrStoreBusy) {
|
||||
t.Fatalf("busy result should mark both measurements unavailable: %#v", result)
|
||||
}
|
||||
if _, err := Compact(context.Background(), time.Now()); !errors.Is(err, ErrCompactInProgress) {
|
||||
t.Fatalf("compact error = %v, want %v", err, ErrCompactInProgress)
|
||||
}
|
||||
}
|
||||
|
||||
func TestInspectStorageRequiresInitializedStore(t *testing.T) {
|
||||
storeMu.Lock()
|
||||
previous := store
|
||||
store = nil
|
||||
storeMu.Unlock()
|
||||
t.Cleanup(func() {
|
||||
storeMu.Lock()
|
||||
store = previous
|
||||
storeMu.Unlock()
|
||||
})
|
||||
|
||||
if _, err := InspectStorage(context.Background()); !errors.Is(err, ErrStoreNotInitialized) {
|
||||
t.Fatalf("inspect error = %v, want %v", err, ErrStoreNotInitialized)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCloseStoreContextCancelsMigrationBeforeTakingStoreLock(t *testing.T) {
|
||||
s, err := metric.Open(context.Background(), metric.SQLite(":memory:"))
|
||||
if err != nil {
|
||||
t.Fatalf("open metric store: %v", err)
|
||||
}
|
||||
installTestStore(t, s)
|
||||
|
||||
migrationCtx, cancelMigration := context.WithCancel(context.Background())
|
||||
done := make(chan struct{})
|
||||
if !storeOperations.TryAcquire() {
|
||||
t.Fatal("acquire store operation gate")
|
||||
}
|
||||
storeMigMu.Lock()
|
||||
storeMigCancel = cancelMigration
|
||||
storeMigDone = done
|
||||
storeMigMu.Unlock()
|
||||
|
||||
go func() {
|
||||
<-migrationCtx.Done()
|
||||
storeOperations.Release()
|
||||
storeMigMu.Lock()
|
||||
storeMigCancel = nil
|
||||
storeMigDone = nil
|
||||
close(done)
|
||||
storeMigMu.Unlock()
|
||||
}()
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), time.Second)
|
||||
defer cancel()
|
||||
if err := CloseStoreContext(ctx); err != nil {
|
||||
t.Fatalf("close store: %v", err)
|
||||
}
|
||||
if migrationCtx.Err() == nil {
|
||||
t.Fatal("store migration was not canceled before close")
|
||||
}
|
||||
if err := Reload(context.Background()); !errors.Is(err, ErrStoreBusy) {
|
||||
t.Fatalf("reload after close error = %v, want %v", err, ErrStoreBusy)
|
||||
}
|
||||
}
|
||||
|
||||
func TestStoreOperationWaitsRespectContext(t *testing.T) {
|
||||
if !storeOperations.TryAcquire() {
|
||||
t.Fatal("acquire store operation gate")
|
||||
}
|
||||
defer storeOperations.Release()
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
cancel()
|
||||
if _, err := InspectStorage(ctx); !errors.Is(err, context.Canceled) {
|
||||
t.Fatalf("inspect error = %v, want context canceled", err)
|
||||
}
|
||||
if err := Reload(ctx); !errors.Is(err, context.Canceled) {
|
||||
t.Fatalf("reload error = %v, want context canceled", err)
|
||||
}
|
||||
if err := CloseStoreContext(ctx); !errors.Is(err, context.Canceled) {
|
||||
t.Fatalf("close error = %v, want context canceled", err)
|
||||
}
|
||||
}
|
||||
|
||||
func installTestStore(t *testing.T, s *metric.Store) {
|
||||
t.Helper()
|
||||
storeMigMu.Lock()
|
||||
previousClosing := storeClosing
|
||||
storeClosing = false
|
||||
storeMigMu.Unlock()
|
||||
storeMu.Lock()
|
||||
previous := store
|
||||
previousFingerprint := storeFingerprint
|
||||
store = s
|
||||
storeFingerprint = "test|memory"
|
||||
storeMu.Unlock()
|
||||
t.Cleanup(func() {
|
||||
storeMigMu.Lock()
|
||||
storeClosing = previousClosing
|
||||
storeMigMu.Unlock()
|
||||
storeMu.Lock()
|
||||
store = previous
|
||||
storeFingerprint = previousFingerprint
|
||||
storeMu.Unlock()
|
||||
_ = s.Close()
|
||||
})
|
||||
}
|
||||
@@ -0,0 +1,61 @@
|
||||
package metricstore
|
||||
|
||||
const (
|
||||
MetricCPU = "cpu.usage"
|
||||
MetricGPU = "gpu.usage"
|
||||
MetricGPUDeviceUsage = "gpu.device.usage"
|
||||
MetricGPUMem = "gpu.memory.used"
|
||||
MetricGPUMemTotal = "gpu.memory.total"
|
||||
MetricGPUTemp = "gpu.temperature"
|
||||
MetricRAM = "memory.used"
|
||||
MetricRAMTotal = "memory.total"
|
||||
MetricSwap = "swap.used"
|
||||
MetricSwapTotal = "swap.total"
|
||||
MetricLoad = "load.average"
|
||||
MetricTemp = "temperature"
|
||||
MetricDisk = "disk.used"
|
||||
MetricDiskTotal = "disk.total"
|
||||
MetricNetIn = "net.in.rate"
|
||||
MetricNetOut = "net.out.rate"
|
||||
MetricNetTotalUp = "net.total.up"
|
||||
MetricNetTotalDown = "net.total.down"
|
||||
MetricTrafficUp = "traffic.up"
|
||||
MetricTrafficDown = "traffic.down"
|
||||
MetricProcess = "process.count"
|
||||
MetricConnections = "connections.tcp"
|
||||
MetricConnectionsUDP = "connections.udp"
|
||||
MetricPingLatency = "ping.latency_ms"
|
||||
MetricPingLoss = "ping.loss"
|
||||
)
|
||||
|
||||
// loadRecordMetricNames are the entity-level metrics used to reconstruct the
|
||||
// legacy Record response shape.
|
||||
var loadRecordMetricNames = []string{
|
||||
MetricCPU, MetricGPU, MetricRAM, MetricRAMTotal, MetricSwap, MetricSwapTotal,
|
||||
MetricLoad, MetricTemp, MetricDisk, MetricDiskTotal, MetricNetIn, MetricNetOut,
|
||||
MetricNetTotalUp, MetricNetTotalDown, MetricTrafficUp, MetricTrafficDown,
|
||||
MetricProcess, MetricConnections, MetricConnectionsUDP,
|
||||
}
|
||||
|
||||
// gpuDeviceRecordMetricNames are stored separately from the entity-level GPU
|
||||
// average and are included when deleting all system records.
|
||||
var gpuDeviceRecordMetricNames = []string{
|
||||
MetricGPUDeviceUsage, MetricGPUMem, MetricGPUMemTotal, MetricGPUTemp,
|
||||
}
|
||||
|
||||
var recordMetricNames = joinMetricNames(loadRecordMetricNames, gpuDeviceRecordMetricNames)
|
||||
|
||||
// Ping has an independent retention and cleanup boundary.
|
||||
var pingMetricNames = []string{MetricPingLatency, MetricPingLoss}
|
||||
|
||||
func joinMetricNames(groups ...[]string) []string {
|
||||
total := 0
|
||||
for _, group := range groups {
|
||||
total += len(group)
|
||||
}
|
||||
names := make([]string, 0, total)
|
||||
for _, group := range groups {
|
||||
names = append(names, group...)
|
||||
}
|
||||
return names
|
||||
}
|
||||
@@ -0,0 +1,999 @@
|
||||
package metricstore
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"log"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/komari-monitor/komari/database/models"
|
||||
"github.com/komari-monitor/komari/pkg/config"
|
||||
"github.com/komari-monitor/komari/pkg/metric"
|
||||
)
|
||||
|
||||
var (
|
||||
store *metric.Store
|
||||
storeFingerprint string
|
||||
storeMu sync.RWMutex
|
||||
storeOnce sync.Once
|
||||
storeOperations = newStoreOperationGate()
|
||||
compactAt int
|
||||
)
|
||||
|
||||
var ErrCompactInProgress = errors.New("metric store compact already in progress")
|
||||
|
||||
const (
|
||||
// DefaultRollupRawRetention keeps a short hot raw window; older samples are
|
||||
// served from rollups after compaction.
|
||||
DefaultRollupRawRetention = 15 * time.Minute
|
||||
DefaultRollupFinestTier = time.Minute
|
||||
)
|
||||
|
||||
// MetricStoreConfig 保存 metric store 配置。
|
||||
//
|
||||
// 注意:metric store 现在始终启用(旧的 metric_store_enabled 开关已废弃)。
|
||||
// 未显式配置时默认使用 SQLite(./data/metrics.db)。
|
||||
type MetricStoreConfig struct {
|
||||
Driver string `json:"metric_db_driver" default:"sqlite"` // 数据库类型: sqlite, mysql, postgresql
|
||||
DSN string `json:"metric_db_dsn" default:"./data/metrics.db"` // 数据库连接串
|
||||
DownsamplingEnabled bool `json:"metric_downsampling_enabled" default:"false"` // 是否启用分层降采样
|
||||
TablePrefix string `json:"metric_table_prefix" default:"metric_"` // 表名前缀
|
||||
MaxOpenConns int `json:"metric_max_open_conns" default:"25"` // 最大连接数
|
||||
MaxIdleConns int `json:"metric_max_idle_conns" default:"5"` // 最大空闲连接数
|
||||
}
|
||||
|
||||
// MetricStoreConfigKeys 配置键
|
||||
//
|
||||
// MetricStoreEnabledKey 已废弃:metric store 始终启用,保留常量仅用于清理旧配置。
|
||||
const (
|
||||
MetricStoreEnabledKey = "metric_store_enabled" // Deprecated: metric store 始终启用
|
||||
MetricDBDriverKey = "metric_db_driver"
|
||||
MetricDBDSNKey = "metric_db_dsn"
|
||||
MetricDownsamplingEnabledKey = "metric_downsampling_enabled"
|
||||
MetricTablePrefixKey = "metric_table_prefix"
|
||||
MetricMaxOpenConnsKey = "metric_max_open_conns"
|
||||
MetricMaxIdleConnsKey = "metric_max_idle_conns"
|
||||
// MigrationTargetKey 记录上一次成功完成启动迁移的目标指纹(driver+dsn)。
|
||||
// 当目标发生变化(例如从 SQLite 切换到 MySQL/PostgreSQL)时,启动迁移会
|
||||
// 重新执行,把上一个目标库的数据搬运到新的目标 metrics 库。
|
||||
MigrationTargetKey = "metric_migration_target"
|
||||
)
|
||||
|
||||
// buildMetricConfig 根据 MetricStoreConfig 构造底层 metric.Config。
|
||||
// autoMigrate 控制是否在 Open 时自动建表:正式初始化/热加载时为 true,
|
||||
// 仅做连接测试时为 false(不写入 schema,避免对目标库产生副作用)。
|
||||
func buildMetricConfig(cfg *MetricStoreConfig, autoMigrate bool) (metric.Config, error) {
|
||||
driver := ResolveDriverFromConfig(cfg.Driver, cfg.DSN)
|
||||
|
||||
tablePrefix := cfg.TablePrefix
|
||||
if tablePrefix == "" {
|
||||
tablePrefix = "metric_"
|
||||
}
|
||||
opts := []metric.Option{
|
||||
metric.WithTablePrefix(tablePrefix),
|
||||
metric.WithAutoMigrate(autoMigrate),
|
||||
}
|
||||
if cfg.DownsamplingEnabled {
|
||||
opts = append(opts, metric.WithRollupPolicy(defaultRollupPolicy()))
|
||||
}
|
||||
|
||||
switch driver {
|
||||
case metric.DriverSQLite:
|
||||
dsn := cfg.DSN
|
||||
if dsn == "" || dsn == "./data/metrics.db" {
|
||||
// 注意:刻意不使用 cache=shared。SQLite 共享缓存模式使用表级锁,
|
||||
// 当一个连接持有读锁、另一个连接尝试写入时会立即返回
|
||||
// SQLITE_LOCKED("database table is locked"),且 busy_timeout
|
||||
// 对共享缓存的表级锁无效,迁移期间与前台查询/实时写入并发时必然报错。
|
||||
// _txlock=immediate 让写事务开始即获取写锁,避免锁升级死锁。
|
||||
dsn = "file:./data/metrics.db?mode=rwc&_txlock=immediate"
|
||||
} else {
|
||||
// 用户自定义 DSN 时,剥离 cache=shared,避免上述表级锁问题。
|
||||
dsn = stripSharedCache(dsn)
|
||||
}
|
||||
// SQLite 串行化写入:固定单写连接以避免 "database is locked" 竞争,
|
||||
// 同时启用独立的 WAL 只读连接池提升前台查询并发(写仍走单主连接)。
|
||||
// 这里刻意忽略 cfg.MaxOpenConns/MaxIdleConns —— 对 SQLite 而言多写连接
|
||||
// 只会引入锁竞争而非提升吞吐。
|
||||
opts = append(opts,
|
||||
metric.WithMaxOpenConns(1),
|
||||
metric.WithMaxIdleConns(1),
|
||||
metric.WithSQLiteReadPool(4),
|
||||
)
|
||||
return metric.SQLite(dsn, opts...), nil
|
||||
case metric.DriverMySQL:
|
||||
opts = append(opts,
|
||||
metric.WithMaxOpenConns(cfg.MaxOpenConns),
|
||||
metric.WithMaxIdleConns(cfg.MaxIdleConns),
|
||||
)
|
||||
return metric.MySQL(cfg.DSN, opts...), nil
|
||||
case metric.DriverPostgreSQL:
|
||||
opts = append(opts,
|
||||
metric.WithMaxOpenConns(cfg.MaxOpenConns),
|
||||
metric.WithMaxIdleConns(cfg.MaxIdleConns),
|
||||
)
|
||||
return metric.PostgreSQL(cfg.DSN, opts...), nil
|
||||
default:
|
||||
return metric.Config{}, fmt.Errorf("unsupported metric database driver: %s", cfg.Driver)
|
||||
}
|
||||
}
|
||||
|
||||
func defaultRollupPolicy() metric.RollupPolicy {
|
||||
return metric.RollupPolicy{
|
||||
RawRetention: DefaultRollupRawRetention,
|
||||
Tiers: []metric.RollupTier{
|
||||
{Interval: DefaultRollupFinestTier, Retention: 48 * time.Hour},
|
||||
{Interval: 5 * time.Minute, Retention: 14 * 24 * time.Hour},
|
||||
{Interval: time.Hour, Retention: 14 * 24 * time.Hour},
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
// ResolveDriverFromConfig 根据 DSN 自动推断 metrics 数据库类型;当 DSN 不能可靠
|
||||
// 识别时回退到旧配置中的 driver,以兼容已有配置和非常规 DSN。
|
||||
func ResolveDriverFromConfig(configuredDriver, dsn string) metric.Driver {
|
||||
if driver, ok := InferDriverFromDSN(dsn); ok {
|
||||
return driver
|
||||
}
|
||||
|
||||
switch driver := metric.Driver(strings.ToLower(strings.TrimSpace(configuredDriver))); driver {
|
||||
case metric.DriverSQLite, metric.DriverMySQL, metric.DriverPostgreSQL:
|
||||
return driver
|
||||
default:
|
||||
return metric.DriverSQLite
|
||||
}
|
||||
}
|
||||
|
||||
// InferDriverFromDSN 尽量根据常见 DSN 格式推断数据库类型。
|
||||
// 返回 ok=false 表示格式不够明确,调用方应使用已有配置作为兜底。
|
||||
func InferDriverFromDSN(dsn string) (metric.Driver, bool) {
|
||||
raw := strings.TrimSpace(dsn)
|
||||
if raw == "" {
|
||||
return metric.DriverSQLite, true
|
||||
}
|
||||
lower := strings.ToLower(raw)
|
||||
|
||||
// PostgreSQL URL DSN: postgres://... 或 postgresql://...
|
||||
if strings.HasPrefix(lower, "postgres://") || strings.HasPrefix(lower, "postgresql://") {
|
||||
return metric.DriverPostgreSQL, true
|
||||
}
|
||||
|
||||
// SQLite 常见文件/内存 DSN。
|
||||
if raw == ":memory:" || strings.HasPrefix(lower, "file:") || strings.HasPrefix(lower, "sqlite://") || strings.HasPrefix(lower, "sqlite3://") {
|
||||
return metric.DriverSQLite, true
|
||||
}
|
||||
|
||||
// MySQL URL(虽然 go-sql-driver/mysql 原生 DSN 通常不是 URL,但这里用于给出
|
||||
// 类型推断;连接测试仍会校验 DSN 是否可被驱动接受)。
|
||||
if strings.HasPrefix(lower, "mysql://") {
|
||||
return metric.DriverMySQL, true
|
||||
}
|
||||
|
||||
// PostgreSQL 关键字/值 DSN: host=... user=... dbname=...
|
||||
if looksLikePostgreSQLKeyValueDSN(lower) {
|
||||
return metric.DriverPostgreSQL, true
|
||||
}
|
||||
|
||||
// go-sql-driver/mysql DSN: user:pass@tcp(host:3306)/db、user@unix(...)/db、user:pass@/db 等。
|
||||
if looksLikeMySQLDSN(lower) {
|
||||
return metric.DriverMySQL, true
|
||||
}
|
||||
|
||||
// SQLite 路径:./data/metrics.db、/var/lib/metrics.sqlite3、metrics.sqlite 等。
|
||||
if looksLikeSQLitePath(lower) {
|
||||
return metric.DriverSQLite, true
|
||||
}
|
||||
|
||||
return "", false
|
||||
}
|
||||
|
||||
func looksLikePostgreSQLKeyValueDSN(lower string) bool {
|
||||
if !strings.Contains(lower, "=") || strings.Contains(lower, "://") {
|
||||
return false
|
||||
}
|
||||
keys := []string{"host=", "user=", "password=", "dbname=", "port=", "sslmode="}
|
||||
matched := 0
|
||||
for _, key := range keys {
|
||||
if strings.Contains(lower, key) {
|
||||
matched++
|
||||
}
|
||||
}
|
||||
// dbname= 基本是 PostgreSQL libpq DSN 的强特征;否则至少匹配两个常见键。
|
||||
return strings.Contains(lower, "dbname=") || matched >= 2
|
||||
}
|
||||
|
||||
func looksLikeMySQLDSN(lower string) bool {
|
||||
if strings.Contains(lower, "://") || strings.Contains(lower, " ") {
|
||||
return false
|
||||
}
|
||||
if strings.Contains(lower, "@tcp(") || strings.Contains(lower, "@unix(") || strings.Contains(lower, "@/") {
|
||||
return true
|
||||
}
|
||||
// user:pass@host/db、user@host/db 这类虽然不是推荐格式,但也明显偏 MySQL。
|
||||
return strings.Contains(lower, "@") && strings.Contains(lower, "/")
|
||||
}
|
||||
|
||||
func looksLikeSQLitePath(lower string) bool {
|
||||
path := lower
|
||||
if idx := strings.IndexAny(path, "?"); idx >= 0 {
|
||||
path = path[:idx]
|
||||
}
|
||||
return strings.HasSuffix(path, ".db") || strings.HasSuffix(path, ".sqlite") || strings.HasSuffix(path, ".sqlite3")
|
||||
}
|
||||
|
||||
// stripSharedCache 从 SQLite DSN 中移除 cache=shared 参数,避免共享缓存模式下的
|
||||
// 表级锁(SQLITE_LOCKED "database table is locked")。其它参数保持不变。
|
||||
func stripSharedCache(dsn string) string {
|
||||
if !strings.Contains(dsn, "cache=shared") {
|
||||
return dsn
|
||||
}
|
||||
idx := strings.Index(dsn, "?")
|
||||
if idx < 0 {
|
||||
return dsn
|
||||
}
|
||||
base := dsn[:idx]
|
||||
query := dsn[idx+1:]
|
||||
parts := strings.Split(query, "&")
|
||||
kept := parts[:0]
|
||||
for _, p := range parts {
|
||||
if p == "cache=shared" {
|
||||
continue
|
||||
}
|
||||
kept = append(kept, p)
|
||||
}
|
||||
if len(kept) == 0 {
|
||||
return base
|
||||
}
|
||||
return base + "?" + strings.Join(kept, "&")
|
||||
}
|
||||
|
||||
// openStore 按配置打开 metric store 并创建指标定义。
|
||||
func openStore(ctx context.Context, cfg *MetricStoreConfig) (*metric.Store, error) {
|
||||
return openStoreWithDefaultRetention(ctx, cfg, defaultBuiltinMetricRetentionDays)
|
||||
}
|
||||
|
||||
func openStoreWithDefaultRetention(ctx context.Context, cfg *MetricStoreConfig, defaultRetentionDays int) (*metric.Store, error) {
|
||||
metricCfg, err := buildMetricConfig(cfg, true)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
s, err := metric.Open(ctx, metricCfg)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to open metric store: %w", err)
|
||||
}
|
||||
|
||||
if err := createMetricDefinitionsWithDefaultRetention(ctx, s, defaultRetentionDays); err != nil {
|
||||
s.Close()
|
||||
return nil, fmt.Errorf("failed to create metric definitions: %w", err)
|
||||
}
|
||||
|
||||
return s, nil
|
||||
}
|
||||
|
||||
// OpenStore opens an isolated metric store using the supplied configuration.
|
||||
// It is used by the pre-start upgrade flow before the process-wide store is
|
||||
// initialized. The caller owns the returned store and must close it.
|
||||
func OpenStore(ctx context.Context, cfg *MetricStoreConfig) (*metric.Store, error) {
|
||||
return openStore(ctx, cfg)
|
||||
}
|
||||
|
||||
// OpenStoreForMigration opens an isolated target and uses the legacy data span
|
||||
// as the initial retention for definitions that do not exist yet. Existing
|
||||
// definitions keep their configured retention, including an explicit zero.
|
||||
func OpenStoreForMigration(ctx context.Context, cfg *MetricStoreConfig, legacyRetentionDays int) (*metric.Store, error) {
|
||||
if legacyRetentionDays < defaultBuiltinMetricRetentionDays {
|
||||
legacyRetentionDays = defaultBuiltinMetricRetentionDays
|
||||
}
|
||||
return openStoreWithDefaultRetention(ctx, cfg, legacyRetentionDays)
|
||||
}
|
||||
|
||||
// TestConnection 使用给定配置尝试连接 metrics 数据库(不影响当前运行的 store)。
|
||||
// 仅打开连接并 Ping,不执行自动建表,连接成功后立即关闭。失败时返回可读错误。
|
||||
func TestConnection(ctx context.Context, cfg *MetricStoreConfig) error {
|
||||
metricCfg, err := buildMetricConfig(cfg, false)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
s, err := metric.Open(ctx, metricCfg)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer s.Close()
|
||||
|
||||
return s.Ping(ctx)
|
||||
}
|
||||
|
||||
// InitializeStore 初始化 metric store(启动时调用,仅执行一次)。
|
||||
func InitializeStore() error {
|
||||
var initErr error
|
||||
storeOnce.Do(func() {
|
||||
cfg, err := config.GetManyAs[MetricStoreConfig]()
|
||||
if err != nil {
|
||||
initErr = fmt.Errorf("failed to load metric store config: %w", err)
|
||||
return
|
||||
}
|
||||
|
||||
// metric store 始终启用;未配置时默认 SQLite(./data/metrics.db)。
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||||
defer cancel()
|
||||
|
||||
s, err := openStore(ctx, cfg)
|
||||
if err != nil {
|
||||
initErr = err
|
||||
return
|
||||
}
|
||||
|
||||
storeMu.Lock()
|
||||
store = s
|
||||
storeFingerprint = targetFingerprint(cfg)
|
||||
storeMu.Unlock()
|
||||
clearStoreClosing()
|
||||
|
||||
log.Printf("Metric store initialized successfully (driver=%s)", ResolveDriverFromConfig(cfg.Driver, cfg.DSN))
|
||||
})
|
||||
|
||||
return initErr
|
||||
}
|
||||
|
||||
// Reload 根据最新配置热重载 metric store,无需重启进程。
|
||||
// metric store 始终启用:用新配置打开并建表(内部已 Ping 校验连接),
|
||||
// 成功后再替换运行中的 store,最后关闭旧实例。任何失败都会保留旧 store 不变。
|
||||
//
|
||||
// 注意:Reload 只切换运行中的连接,不会把旧目标(如 SQLite)中的历史数据
|
||||
// 搬运到新目标(如 MySQL/PostgreSQL)。跨库数据迁移由启动迁移
|
||||
// (RunStartupMigration)在下次启动时按目标指纹自动完成。
|
||||
func Reload(ctx context.Context) error {
|
||||
if err := storeOperations.Acquire(ctx); err != nil {
|
||||
return fmt.Errorf("wait for metric store operations before reload: %w", err)
|
||||
}
|
||||
defer storeOperations.Release()
|
||||
if isStoreClosing() {
|
||||
return ErrStoreBusy
|
||||
}
|
||||
|
||||
cfg, err := config.GetManyAs[MetricStoreConfig]()
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to load metric store config: %w", err)
|
||||
}
|
||||
|
||||
// 用新配置打开并建表(内部已 Ping 校验连接)。
|
||||
s, err := openStore(ctx, cfg)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
storeMu.Lock()
|
||||
old := store
|
||||
store = s
|
||||
storeFingerprint = targetFingerprint(cfg)
|
||||
storeMu.Unlock()
|
||||
|
||||
if old != nil {
|
||||
if cerr := old.Close(); cerr != nil {
|
||||
log.Printf("Failed to close previous metric store on reload: %v", cerr)
|
||||
}
|
||||
}
|
||||
|
||||
log.Printf("Metric store reloaded successfully (driver=%s)", ResolveDriverFromConfig(cfg.Driver, cfg.DSN))
|
||||
return nil
|
||||
}
|
||||
|
||||
// GetStore 获取 metric store 实例(如果未启用返回 nil)
|
||||
func GetStore() *metric.Store {
|
||||
storeMu.RLock()
|
||||
defer storeMu.RUnlock()
|
||||
return store
|
||||
}
|
||||
|
||||
// RetentionSummary is the compatibility view of all persisted metric policies.
|
||||
type RetentionSummary struct {
|
||||
AllPositive bool
|
||||
MaxDays int
|
||||
}
|
||||
|
||||
// GetRetentionSummary aggregates the active store's metric definitions. An
|
||||
// empty definition set is not considered record-enabled.
|
||||
func GetRetentionSummary(ctx context.Context) (RetentionSummary, error) {
|
||||
s := GetStore()
|
||||
if s == nil {
|
||||
return RetentionSummary{}, fmt.Errorf("metric store not initialized")
|
||||
}
|
||||
defs, err := s.ListMetrics(ctx)
|
||||
if err != nil {
|
||||
return RetentionSummary{}, err
|
||||
}
|
||||
return summarizeRetentionDefinitions(defs), nil
|
||||
}
|
||||
|
||||
func summarizeRetentionDefinitions(defs []metric.Definition) RetentionSummary {
|
||||
if len(defs) == 0 {
|
||||
return RetentionSummary{}
|
||||
}
|
||||
summary := RetentionSummary{AllPositive: true}
|
||||
for _, def := range defs {
|
||||
if def.RetentionDays <= 0 {
|
||||
summary.AllPositive = false
|
||||
}
|
||||
if def.RetentionDays > summary.MaxDays {
|
||||
summary.MaxDays = def.RetentionDays
|
||||
}
|
||||
}
|
||||
return summary
|
||||
}
|
||||
|
||||
func Compact(ctx context.Context, now time.Time) (int, error) {
|
||||
if !storeOperations.TryAcquire() {
|
||||
return 0, ErrCompactInProgress
|
||||
}
|
||||
defer storeOperations.Release()
|
||||
|
||||
storeMu.RLock()
|
||||
activeStore := store
|
||||
storeMu.RUnlock()
|
||||
if activeStore == nil {
|
||||
return 0, fmt.Errorf("metric store not initialized")
|
||||
}
|
||||
|
||||
defs, err := activeStore.ListMetrics(ctx)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
if len(defs) == 0 {
|
||||
compactAt = 0
|
||||
return 0, nil
|
||||
}
|
||||
if compactAt >= len(defs) {
|
||||
compactAt = 0
|
||||
}
|
||||
|
||||
total := 0
|
||||
start := compactAt
|
||||
for i := 0; i < len(defs); i++ {
|
||||
idx := (start + i) % len(defs)
|
||||
n, err := activeStore.CompactMetric(ctx, defs[idx].Name, now)
|
||||
if err != nil {
|
||||
compactAt = idx
|
||||
return total, fmt.Errorf("compact metric %q: %w", defs[idx].Name, err)
|
||||
}
|
||||
total += n
|
||||
compactAt = (idx + 1) % len(defs)
|
||||
}
|
||||
if _, err := activeStore.CleanupExpired(ctx, now); err != nil {
|
||||
return total, fmt.Errorf("clean up expired raw metrics: %w", err)
|
||||
}
|
||||
return total, nil
|
||||
}
|
||||
|
||||
// CloseStoreContext stops the asynchronous store migration before taking the
|
||||
// store write lock, so shutdown cannot wait forever on the migration's lease.
|
||||
func CloseStoreContext(ctx context.Context) error {
|
||||
if err := stopStoreMigrationForClose(ctx); err != nil {
|
||||
clearStoreClosing()
|
||||
return err
|
||||
}
|
||||
if err := storeOperations.Acquire(ctx); err != nil {
|
||||
clearStoreClosing()
|
||||
return fmt.Errorf("wait for metric store operations before close: %w", err)
|
||||
}
|
||||
defer storeOperations.Release()
|
||||
|
||||
storeMu.Lock()
|
||||
defer storeMu.Unlock()
|
||||
|
||||
if store != nil {
|
||||
err := store.Close()
|
||||
store = nil
|
||||
storeFingerprint = ""
|
||||
return err
|
||||
}
|
||||
storeFingerprint = ""
|
||||
return nil
|
||||
}
|
||||
|
||||
const defaultBuiltinMetricRetentionDays = 1
|
||||
|
||||
// createMetricDefinitions creates built-in definitions with explicit policies.
|
||||
func createMetricDefinitions(ctx context.Context, s *metric.Store) error {
|
||||
return createMetricDefinitionsWithDefaultRetention(ctx, s, defaultBuiltinMetricRetentionDays)
|
||||
}
|
||||
|
||||
func createMetricDefinitionsWithDefaultRetention(ctx context.Context, s *metric.Store, defaultRetentionDays int) error {
|
||||
if defaultRetentionDays < defaultBuiltinMetricRetentionDays {
|
||||
defaultRetentionDays = defaultBuiltinMetricRetentionDays
|
||||
}
|
||||
definitions := []metric.Definition{
|
||||
{Name: MetricCPU, Type: metric.TypeGauge, Unit: "%", Description: "CPU usage percentage", RetentionDays: defaultRetentionDays},
|
||||
{Name: MetricGPU, Type: metric.TypeGauge, Unit: "%", Description: "GPU usage percentage", RetentionDays: defaultRetentionDays},
|
||||
{Name: MetricGPUDeviceUsage, Type: metric.TypeGauge, Unit: "%", Description: "Per-device GPU utilization", RetentionDays: defaultRetentionDays},
|
||||
{Name: MetricGPUMem, Type: metric.TypeGauge, Unit: "bytes", Description: "GPU memory used", RetentionDays: defaultRetentionDays},
|
||||
{Name: MetricGPUMemTotal, Type: metric.TypeGauge, Unit: "bytes", Description: "GPU memory total", RetentionDays: defaultRetentionDays},
|
||||
{Name: MetricGPUTemp, Type: metric.TypeGauge, Unit: "°C", Description: "GPU temperature", RetentionDays: defaultRetentionDays},
|
||||
{Name: MetricRAM, Type: metric.TypeGauge, Unit: "bytes", Description: "RAM used", RetentionDays: defaultRetentionDays},
|
||||
{Name: MetricRAMTotal, Type: metric.TypeGauge, Unit: "bytes", Description: "RAM total", RetentionDays: defaultRetentionDays},
|
||||
{Name: MetricSwap, Type: metric.TypeGauge, Unit: "bytes", Description: "Swap used", RetentionDays: defaultRetentionDays},
|
||||
{Name: MetricSwapTotal, Type: metric.TypeGauge, Unit: "bytes", Description: "Swap total", RetentionDays: defaultRetentionDays},
|
||||
{Name: MetricLoad, Type: metric.TypeGauge, Unit: "", Description: "System load average", RetentionDays: defaultRetentionDays},
|
||||
{Name: MetricTemp, Type: metric.TypeGauge, Unit: "°C", Description: "System temperature", RetentionDays: defaultRetentionDays},
|
||||
{Name: MetricDisk, Type: metric.TypeGauge, Unit: "bytes", Description: "Disk used", RetentionDays: defaultRetentionDays},
|
||||
{Name: MetricDiskTotal, Type: metric.TypeGauge, Unit: "bytes", Description: "Disk total", RetentionDays: defaultRetentionDays},
|
||||
{Name: MetricNetIn, Type: metric.TypeGauge, Unit: "bytes/s", Description: "Network in rate", RetentionDays: defaultRetentionDays},
|
||||
{Name: MetricNetOut, Type: metric.TypeGauge, Unit: "bytes/s", Description: "Network out rate", RetentionDays: defaultRetentionDays},
|
||||
{Name: MetricNetTotalUp, Type: metric.TypeCounter, Unit: "bytes", Description: "Network total upload", RetentionDays: defaultRetentionDays},
|
||||
{Name: MetricNetTotalDown, Type: metric.TypeCounter, Unit: "bytes", Description: "Network total download", RetentionDays: defaultRetentionDays},
|
||||
{Name: MetricTrafficUp, Type: metric.TypeGauge, Unit: "bytes", Description: "Traffic upload delta", RetentionDays: defaultRetentionDays},
|
||||
{Name: MetricTrafficDown, Type: metric.TypeGauge, Unit: "bytes", Description: "Traffic download delta", RetentionDays: defaultRetentionDays},
|
||||
{Name: MetricProcess, Type: metric.TypeGauge, Unit: "count", Description: "Process count", RetentionDays: defaultRetentionDays},
|
||||
{Name: MetricConnections, Type: metric.TypeGauge, Unit: "count", Description: "TCP connections", RetentionDays: defaultRetentionDays},
|
||||
{Name: MetricConnectionsUDP, Type: metric.TypeGauge, Unit: "count", Description: "UDP connections", RetentionDays: defaultRetentionDays},
|
||||
{Name: MetricPingLatency, Type: metric.TypeGauge, Unit: "ms", Description: "Ping latency", RetentionDays: defaultRetentionDays},
|
||||
{Name: MetricPingLoss, Type: metric.TypeGauge, Unit: "ratio", Description: "Ping packet loss indicator", RetentionDays: defaultRetentionDays},
|
||||
}
|
||||
|
||||
for _, def := range definitions {
|
||||
existing, err := s.GetMetric(ctx, def.Name)
|
||||
if err != nil && !errors.Is(err, metric.ErrNotFound) {
|
||||
return fmt.Errorf("failed to get metric %s: %w", def.Name, err)
|
||||
}
|
||||
if err == nil {
|
||||
if existing.RetentionDays == 0 {
|
||||
if _, err := s.SetMetricRetention(ctx, def.Name, 0); err != nil {
|
||||
return fmt.Errorf("failed to preserve disabled metric %s: %w", def.Name, err)
|
||||
}
|
||||
continue
|
||||
}
|
||||
def.RetentionDays = existing.RetentionDays
|
||||
}
|
||||
if err := s.UpsertMetric(ctx, def); err != nil {
|
||||
return fmt.Errorf("failed to create metric %s: %w", def.Name, err)
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// WritePingRecord 将 ping 记录写入 metric store
|
||||
func WritePingRecord(ctx context.Context, rec models.PingRecord) error {
|
||||
s := GetStore()
|
||||
if s == nil {
|
||||
return fmt.Errorf("metric store not enabled")
|
||||
}
|
||||
|
||||
ts := rec.Time
|
||||
entityID := rec.Client
|
||||
tags := map[string]string{
|
||||
"task_id": fmt.Sprintf("%d", rec.TaskId),
|
||||
}
|
||||
|
||||
loss := 0.0
|
||||
if rec.Value < 0 {
|
||||
loss = 1
|
||||
}
|
||||
points := []metric.Point{
|
||||
{
|
||||
MetricName: MetricPingLatency,
|
||||
EntityID: entityID,
|
||||
Timestamp: ts,
|
||||
Value: float64(rec.Value),
|
||||
Tags: tags,
|
||||
},
|
||||
{
|
||||
MetricName: MetricPingLoss,
|
||||
EntityID: entityID,
|
||||
Timestamp: ts,
|
||||
Value: loss,
|
||||
Tags: tags,
|
||||
},
|
||||
}
|
||||
|
||||
return s.WriteBatch(ctx, points)
|
||||
}
|
||||
|
||||
// GetRecordsByClientAndTime 从 metric store 查询记录并重构为 models.Record
|
||||
func GetRecordsByClientAndTime(ctx context.Context, clientUUID string, start, end time.Time) ([]models.Record, error) {
|
||||
s := GetStore()
|
||||
if s == nil {
|
||||
return nil, fmt.Errorf("metric store not enabled")
|
||||
}
|
||||
|
||||
return getRecordsByClientAndTimeFromSeries(ctx, s, clientUUID, start, end)
|
||||
}
|
||||
|
||||
// GetRecordsByTime 从 metric store 查询所有客户端在时间范围内的记录
|
||||
func GetRecordsByTime(ctx context.Context, start, end time.Time) ([]models.Record, error) {
|
||||
s := GetStore()
|
||||
if s == nil {
|
||||
return nil, fmt.Errorf("metric store not enabled")
|
||||
}
|
||||
|
||||
interval := recordSeriesInterval(s, start, end, time.Now().UTC())
|
||||
entityIDs, err := listRecordEntityIDs(ctx, s, start, end, interval)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var records []models.Record
|
||||
for _, entityID := range entityIDs {
|
||||
items, err := getRecordsByClientAndTimeFromSeries(ctx, s, entityID, start, end)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
records = append(records, items...)
|
||||
}
|
||||
sortRecords(records)
|
||||
return records, nil
|
||||
}
|
||||
|
||||
type recordSeriesKey struct {
|
||||
client string
|
||||
ts int64
|
||||
}
|
||||
|
||||
func getRecordsByClientAndTimeFromSeries(ctx context.Context, s *metric.Store, clientUUID string, start, end time.Time) ([]models.Record, error) {
|
||||
now := time.Now().UTC()
|
||||
interval := recordSeriesInterval(s, start, end, now)
|
||||
recordMap := make(map[recordSeriesKey]*models.Record)
|
||||
|
||||
for _, metricName := range loadRecordMetricNames {
|
||||
points, err := s.Series(ctx, metric.AggregateQuery{
|
||||
Query: metric.Query{
|
||||
MetricName: metricName,
|
||||
EntityID: clientUUID,
|
||||
Start: start,
|
||||
End: end,
|
||||
Order: metric.OrderAsc,
|
||||
},
|
||||
Aggregation: recordMetricAggregation(metricName),
|
||||
Interval: interval,
|
||||
}, now)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to query metric %s: %w", metricName, err)
|
||||
}
|
||||
for _, point := range points {
|
||||
entityID := point.EntityID
|
||||
if entityID == "" {
|
||||
entityID = clientUUID
|
||||
}
|
||||
key := recordSeriesKey{client: entityID, ts: point.Bucket.Unix()}
|
||||
if recordMap[key] == nil {
|
||||
recordMap[key] = &models.Record{
|
||||
Client: entityID,
|
||||
Time: point.Bucket.UTC(),
|
||||
}
|
||||
}
|
||||
applyRecordMetricValue(recordMap[key], metricName, point.Value)
|
||||
}
|
||||
}
|
||||
|
||||
records := make([]models.Record, 0, len(recordMap))
|
||||
for _, rec := range recordMap {
|
||||
records = append(records, *rec)
|
||||
}
|
||||
sortRecords(records)
|
||||
return records, nil
|
||||
}
|
||||
|
||||
func recordMetricAggregation(metricName string) metric.Aggregation {
|
||||
switch metricName {
|
||||
case MetricTrafficUp, MetricTrafficDown:
|
||||
return metric.AggSum
|
||||
case MetricNetTotalUp, MetricNetTotalDown:
|
||||
return metric.AggLast
|
||||
default:
|
||||
return metric.AggAvg
|
||||
}
|
||||
}
|
||||
|
||||
func recordSeriesInterval(s *metric.Store, start, end, now time.Time) time.Duration {
|
||||
interval := recordDownsampleInterval(end.Sub(start), 500)
|
||||
return s.CompatibleSeriesInterval(start, now, interval)
|
||||
}
|
||||
|
||||
func recordDownsampleInterval(rangeDuration time.Duration, maxPoints int) time.Duration {
|
||||
if maxPoints <= 0 {
|
||||
maxPoints = 500
|
||||
}
|
||||
nanos := rangeDuration.Nanoseconds()
|
||||
if nanos <= 0 {
|
||||
return time.Second
|
||||
}
|
||||
interval := time.Duration((nanos + int64(maxPoints) - 1) / int64(maxPoints))
|
||||
if interval < time.Second {
|
||||
return time.Second
|
||||
}
|
||||
return metric.FloorStandardInterval(interval)
|
||||
}
|
||||
|
||||
func listRecordEntityIDs(ctx context.Context, s *metric.Store, start, end time.Time, interval time.Duration) ([]string, error) {
|
||||
seen := make(map[string]struct{})
|
||||
for _, metricName := range loadRecordMetricNames {
|
||||
ids, err := s.EntityIDs(ctx, metric.Query{
|
||||
MetricName: metricName,
|
||||
Start: start.Add(-interval),
|
||||
End: end,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for _, id := range ids {
|
||||
seen[id] = struct{}{}
|
||||
}
|
||||
}
|
||||
out := make([]string, 0, len(seen))
|
||||
for id := range seen {
|
||||
out = append(out, id)
|
||||
}
|
||||
sort.Strings(out)
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func applyRecordMetricValue(rec *models.Record, metricName string, value float64) {
|
||||
switch metricName {
|
||||
case MetricCPU:
|
||||
rec.Cpu = float32(value)
|
||||
case MetricGPU:
|
||||
rec.Gpu = float32(value)
|
||||
case MetricRAM:
|
||||
rec.Ram = int64(value)
|
||||
case MetricRAMTotal:
|
||||
rec.RamTotal = int64(value)
|
||||
case MetricSwap:
|
||||
rec.Swap = int64(value)
|
||||
case MetricSwapTotal:
|
||||
rec.SwapTotal = int64(value)
|
||||
case MetricLoad:
|
||||
rec.Load = float32(value)
|
||||
case MetricTemp:
|
||||
rec.Temp = float32(value)
|
||||
case MetricDisk:
|
||||
rec.Disk = int64(value)
|
||||
case MetricDiskTotal:
|
||||
rec.DiskTotal = int64(value)
|
||||
case MetricNetIn:
|
||||
rec.NetIn = int64(value)
|
||||
case MetricNetOut:
|
||||
rec.NetOut = int64(value)
|
||||
case MetricNetTotalUp:
|
||||
rec.NetTotalUp = int64(value)
|
||||
case MetricNetTotalDown:
|
||||
rec.NetTotalDown = int64(value)
|
||||
case MetricTrafficUp:
|
||||
rec.TrafficUp = int64(value)
|
||||
case MetricTrafficDown:
|
||||
rec.TrafficDown = int64(value)
|
||||
case MetricProcess:
|
||||
rec.Process = int(value)
|
||||
case MetricConnections:
|
||||
rec.Connections = int(value)
|
||||
case MetricConnectionsUDP:
|
||||
rec.ConnectionsUdp = int(value)
|
||||
}
|
||||
}
|
||||
|
||||
func sortRecords(records []models.Record) {
|
||||
sort.Slice(records, func(i, j int) bool {
|
||||
if records[i].Client != records[j].Client {
|
||||
return records[i].Client < records[j].Client
|
||||
}
|
||||
return records[i].Time.Before(records[j].Time)
|
||||
})
|
||||
}
|
||||
|
||||
// GetGPURecordsByClientAndTime 从 metric store 查询 GPU 记录
|
||||
func GetGPURecordsByClientAndTime(ctx context.Context, clientUUID string, start, end time.Time) ([]models.GPURecord, error) {
|
||||
s := GetStore()
|
||||
if s == nil {
|
||||
return nil, fmt.Errorf("metric store not enabled")
|
||||
}
|
||||
|
||||
// 查询 GPU 相关指标(每设备利用率使用独立指标 gpu.device.usage)
|
||||
gpuMetrics := []string{MetricGPUDeviceUsage, MetricGPUMem, MetricGPUMemTotal, MetricGPUTemp}
|
||||
|
||||
// 按设备索引和时间组织数据
|
||||
type gpuKey struct {
|
||||
deviceIndex int
|
||||
timestamp int64
|
||||
}
|
||||
recordMap := make(map[gpuKey]*models.GPURecord)
|
||||
|
||||
for _, metricName := range gpuMetrics {
|
||||
points, err := s.Query(ctx, metric.Query{
|
||||
MetricName: metricName,
|
||||
EntityID: clientUUID,
|
||||
Start: start,
|
||||
End: end,
|
||||
Order: metric.OrderAsc,
|
||||
})
|
||||
if err != nil {
|
||||
continue // GPU 数据可能不存在
|
||||
}
|
||||
|
||||
for _, p := range points {
|
||||
deviceIndex := 0
|
||||
deviceName := ""
|
||||
if idx, ok := p.Tags["device_index"]; ok {
|
||||
fmt.Sscanf(idx, "%d", &deviceIndex)
|
||||
}
|
||||
if name, ok := p.Tags["device_name"]; ok {
|
||||
deviceName = name
|
||||
}
|
||||
|
||||
key := gpuKey{deviceIndex: deviceIndex, timestamp: p.Timestamp.Unix()}
|
||||
if recordMap[key] == nil {
|
||||
recordMap[key] = &models.GPURecord{
|
||||
Client: clientUUID,
|
||||
Time: p.Timestamp.UTC(),
|
||||
DeviceIndex: deviceIndex,
|
||||
DeviceName: deviceName,
|
||||
}
|
||||
}
|
||||
rec := recordMap[key]
|
||||
|
||||
switch metricName {
|
||||
case MetricGPUDeviceUsage:
|
||||
rec.Utilization = float32(p.Value)
|
||||
case MetricGPUMem:
|
||||
rec.MemUsed = int64(p.Value)
|
||||
case MetricGPUMemTotal:
|
||||
rec.MemTotal = int64(p.Value)
|
||||
case MetricGPUTemp:
|
||||
rec.Temperature = int(p.Value)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 转换为切片
|
||||
records := make([]models.GPURecord, 0, len(recordMap))
|
||||
for _, rec := range recordMap {
|
||||
records = append(records, *rec)
|
||||
}
|
||||
|
||||
return records, nil
|
||||
}
|
||||
|
||||
// GetPingRecords 从 metric store 查询兼容旧接口的 ping 记录。
|
||||
//
|
||||
// 旧接口过去直接读取 ping_records 的原始点。启用 metric rollup 后,较旧
|
||||
// 的 raw 点会被压入 rollup 并删除,因此这里使用 Series 走与 queryMetrics
|
||||
// 相同的 raw/rollup 混合读取路径,并保留 task_id 标签。
|
||||
func GetPingRecords(ctx context.Context, clientUUID string, taskID int, start, end time.Time) ([]models.PingRecord, error) {
|
||||
s := GetStore()
|
||||
if s == nil {
|
||||
return nil, fmt.Errorf("metric store not enabled")
|
||||
}
|
||||
|
||||
query := metric.Query{
|
||||
MetricName: MetricPingLatency,
|
||||
Start: start,
|
||||
End: end,
|
||||
Order: metric.OrderAsc,
|
||||
}
|
||||
|
||||
if clientUUID != "" {
|
||||
query.EntityID = clientUUID
|
||||
}
|
||||
|
||||
if taskID >= 0 {
|
||||
query.Tags = map[string]string{"task_id": fmt.Sprintf("%d", taskID)}
|
||||
}
|
||||
|
||||
interval := pingQueryInterval(end.Sub(start), 4000)
|
||||
interval = s.CompatibleSeriesInterval(start, time.Now().UTC(), interval)
|
||||
points, err := s.Series(ctx, metric.AggregateQuery{
|
||||
Query: query,
|
||||
Aggregation: metric.AggLast,
|
||||
Interval: interval,
|
||||
PreserveSeries: true,
|
||||
}, time.Now().UTC())
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
records := make([]models.PingRecord, 0, len(points))
|
||||
for _, p := range points {
|
||||
taskIDVal := uint(0)
|
||||
if tid, ok := p.Tags["task_id"]; ok {
|
||||
var t uint64
|
||||
fmt.Sscanf(tid, "%d", &t)
|
||||
taskIDVal = uint(t)
|
||||
}
|
||||
|
||||
records = append(records, models.PingRecord{
|
||||
Client: p.EntityID,
|
||||
TaskId: taskIDVal,
|
||||
Time: p.Bucket.UTC(),
|
||||
Value: int(p.Value),
|
||||
})
|
||||
}
|
||||
sort.Slice(records, func(i, j int) bool {
|
||||
return records[i].Time.After(records[j].Time)
|
||||
})
|
||||
|
||||
return records, nil
|
||||
}
|
||||
|
||||
func pingQueryInterval(rangeDuration time.Duration, maxPoints int) time.Duration {
|
||||
if maxPoints <= 0 {
|
||||
maxPoints = 4000
|
||||
}
|
||||
if rangeDuration <= 0 {
|
||||
return time.Second
|
||||
}
|
||||
interval := time.Duration((rangeDuration.Nanoseconds() + int64(maxPoints) - 1) / int64(maxPoints))
|
||||
if interval < time.Second {
|
||||
return time.Second
|
||||
}
|
||||
return metric.FloorStandardInterval(interval)
|
||||
}
|
||||
|
||||
// farFuture 返回一个足够远的未来时间,用于以 DeleteBefore 语义清空某指标的全部数据。
|
||||
func farFuture() time.Time {
|
||||
return time.Now().UTC().Add(24 * 365 * time.Hour)
|
||||
}
|
||||
|
||||
// DeleteAllRecords 删除所有负载/系统类记录(保留指标定义,不含 ping)。
|
||||
func DeleteAllRecords(ctx context.Context) error {
|
||||
s := GetStore()
|
||||
if s == nil {
|
||||
return fmt.Errorf("metric store not enabled")
|
||||
}
|
||||
|
||||
for _, metricName := range recordMetricNames {
|
||||
if _, err := s.DeleteBefore(ctx, metricName, farFuture()); err != nil {
|
||||
log.Printf("Failed to delete metric %s: %v", metricName, err)
|
||||
}
|
||||
}
|
||||
clearReportTrafficStates()
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// DeleteAllPingRecords 删除全部 ping 记录(保留指标定义)。
|
||||
func DeleteAllPingRecords(ctx context.Context) error {
|
||||
s := GetStore()
|
||||
if s == nil {
|
||||
return fmt.Errorf("metric store not enabled")
|
||||
}
|
||||
for _, metricName := range pingMetricNames {
|
||||
if _, err := s.DeleteBefore(ctx, metricName, farFuture()); err != nil {
|
||||
return fmt.Errorf("failed to delete ping records: %w", err)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// DeletePingRecordsByTask 删除指定任务(task_id)的全部 ping 记录。
|
||||
func DeletePingRecordsByTask(ctx context.Context, taskIDs []uint) error {
|
||||
s := GetStore()
|
||||
if s == nil {
|
||||
return fmt.Errorf("metric store not enabled")
|
||||
}
|
||||
for _, id := range taskIDs {
|
||||
for _, metricName := range pingMetricNames {
|
||||
if _, err := s.DeleteSeries(ctx, metric.Query{
|
||||
MetricName: metricName,
|
||||
Tags: map[string]string{"task_id": fmt.Sprintf("%d", id)},
|
||||
}); err != nil {
|
||||
return fmt.Errorf("failed to delete ping records for task %d: %w", id, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// DeleteEntity 删除指定 agent 在所有指标下的历史数据。
|
||||
func DeleteEntity(ctx context.Context, entityID string) error {
|
||||
s := GetStore()
|
||||
if s == nil {
|
||||
return fmt.Errorf("metric store not enabled")
|
||||
}
|
||||
if _, err := s.DeleteEntity(ctx, entityID); err != nil {
|
||||
return fmt.Errorf("failed to delete metric records for entity %s: %w", entityID, err)
|
||||
}
|
||||
deleteReportTrafficState(entityID)
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,481 @@
|
||||
package metricstore
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/komari-monitor/komari/database/models"
|
||||
"github.com/komari-monitor/komari/pkg/metric"
|
||||
v1 "github.com/komari-monitor/komari/protocol/v1"
|
||||
)
|
||||
|
||||
func TestDefaultRollupPolicy(t *testing.T) {
|
||||
policy := defaultRollupPolicy()
|
||||
if err := policy.Validate(); err != nil {
|
||||
t.Fatalf("default rollup policy should validate: %v", err)
|
||||
}
|
||||
if policy.RawRetention != DefaultRollupRawRetention {
|
||||
t.Fatalf("raw retention = %s, want %s", policy.RawRetention, DefaultRollupRawRetention)
|
||||
}
|
||||
if len(policy.Tiers) != 3 {
|
||||
t.Fatalf("expected 3 rollup tiers, got %d", len(policy.Tiers))
|
||||
}
|
||||
|
||||
wantIntervals := []time.Duration{time.Minute, 5 * time.Minute, time.Hour}
|
||||
wantRetentions := []time.Duration{48 * time.Hour, 14 * 24 * time.Hour, 14 * 24 * time.Hour}
|
||||
for i := range wantIntervals {
|
||||
if policy.Tiers[i].Interval != wantIntervals[i] {
|
||||
t.Fatalf("tier %d interval = %s, want %s", i, policy.Tiers[i].Interval, wantIntervals[i])
|
||||
}
|
||||
if policy.Tiers[i].Retention != wantRetentions[i] {
|
||||
t.Fatalf("tier %d retention = %s, want %s", i, policy.Tiers[i].Retention, wantRetentions[i])
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildMetricConfigEnablesDefaultRollupPolicy(t *testing.T) {
|
||||
cfg, err := buildMetricConfig(&MetricStoreConfig{
|
||||
Driver: "sqlite",
|
||||
DSN: ":memory:",
|
||||
DownsamplingEnabled: true,
|
||||
TablePrefix: "metric_",
|
||||
}, false)
|
||||
if err != nil {
|
||||
t.Fatalf("build metric config: %v", err)
|
||||
}
|
||||
if !cfg.RollupPolicy.Enabled() {
|
||||
t.Fatal("expected default rollup policy to be enabled")
|
||||
}
|
||||
if cfg.RollupPolicy.RawRetention != DefaultRollupRawRetention {
|
||||
t.Fatalf("raw retention = %s, want %s", cfg.RollupPolicy.RawRetention, DefaultRollupRawRetention)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildMetricConfigLeavesFinalRetentionToMetricDefinition(t *testing.T) {
|
||||
cfg, err := buildMetricConfig(&MetricStoreConfig{
|
||||
Driver: "sqlite",
|
||||
DSN: ":memory:",
|
||||
DownsamplingEnabled: true,
|
||||
}, false)
|
||||
if err != nil {
|
||||
t.Fatalf("build metric config: %v", err)
|
||||
}
|
||||
wantRollupRetention := 14 * 24 * time.Hour
|
||||
lastTier := cfg.RollupPolicy.Tiers[len(cfg.RollupPolicy.Tiers)-1]
|
||||
if lastTier.Retention != wantRollupRetention {
|
||||
t.Fatalf("rollup retention = %s, want %s", lastTier.Retention, wantRollupRetention)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildMetricConfigCanDisableDownsampling(t *testing.T) {
|
||||
cfg, err := buildMetricConfig(&MetricStoreConfig{
|
||||
Driver: "sqlite",
|
||||
DSN: ":memory:",
|
||||
DownsamplingEnabled: false,
|
||||
}, false)
|
||||
if err != nil {
|
||||
t.Fatalf("build metric config: %v", err)
|
||||
}
|
||||
if cfg.RollupPolicy.Enabled() {
|
||||
t.Fatal("expected rollup policy to be disabled")
|
||||
}
|
||||
}
|
||||
|
||||
func TestGetPingRecordsReadsRollupsAfterRawCompaction(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
now := time.Now().UTC().Truncate(time.Second)
|
||||
s, err := metric.Open(ctx, metric.SQLite(":memory:",
|
||||
metric.WithMaxOpenConns(1),
|
||||
metric.WithRollupPolicy(defaultRollupPolicy()),
|
||||
))
|
||||
if err != nil {
|
||||
t.Fatalf("open metric store: %v", err)
|
||||
}
|
||||
defer s.Close()
|
||||
if err := s.UpsertMetric(ctx, metric.Definition{
|
||||
Name: MetricPingLatency,
|
||||
Type: metric.TypeGauge,
|
||||
RetentionDays: 30,
|
||||
}); err != nil {
|
||||
t.Fatalf("create ping metric: %v", err)
|
||||
}
|
||||
if err := s.WriteBatch(ctx, []metric.Point{
|
||||
{MetricName: MetricPingLatency, EntityID: "node-a", Timestamp: now.Add(-20 * time.Minute), Value: 20, Tags: map[string]string{"task_id": "7"}},
|
||||
{MetricName: MetricPingLatency, EntityID: "node-a", Timestamp: now.Add(-10 * time.Minute), Value: 10, Tags: map[string]string{"task_id": "7"}},
|
||||
{MetricName: MetricPingLatency, EntityID: "node-a", Timestamp: now.Add(-5 * time.Minute), Value: 5, Tags: map[string]string{"task_id": "7"}},
|
||||
}); err != nil {
|
||||
t.Fatalf("write ping points: %v", err)
|
||||
}
|
||||
if _, err := s.Compact(ctx, now); err != nil {
|
||||
t.Fatalf("compact ping points: %v", err)
|
||||
}
|
||||
|
||||
storeMu.Lock()
|
||||
oldStore := store
|
||||
store = s
|
||||
storeMu.Unlock()
|
||||
defer func() {
|
||||
storeMu.Lock()
|
||||
store = oldStore
|
||||
storeMu.Unlock()
|
||||
}()
|
||||
|
||||
records, err := GetPingRecords(ctx, "node-a", 7, now.Add(-30*time.Minute), now)
|
||||
if err != nil {
|
||||
t.Fatalf("get ping records: %v", err)
|
||||
}
|
||||
if len(records) != 3 {
|
||||
t.Fatalf("expected 3 ping records across raw and rollup data, got %d: %#v", len(records), records)
|
||||
}
|
||||
if records[0].Value != 5 || records[1].Value != 10 || records[2].Value != 20 {
|
||||
t.Fatalf("unexpected ping values in descending order: %#v", records)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCreateMetricDefinitionsUsesExplicitRetentionAndPreservesOverrides(t *testing.T) {
|
||||
if defaultBuiltinMetricRetentionDays != 1 {
|
||||
t.Fatalf("default built-in metric retention = %d, want 1 day", defaultBuiltinMetricRetentionDays)
|
||||
}
|
||||
|
||||
ctx := context.Background()
|
||||
s, err := metric.Open(ctx, metric.SQLite(":memory:", metric.WithMaxOpenConns(1)))
|
||||
if err != nil {
|
||||
t.Fatalf("open metric store: %v", err)
|
||||
}
|
||||
defer s.Close()
|
||||
|
||||
if err := createMetricDefinitions(ctx, s); err != nil {
|
||||
t.Fatalf("create definitions: %v", err)
|
||||
}
|
||||
defs, err := s.ListMetrics(ctx)
|
||||
if err != nil {
|
||||
t.Fatalf("list definitions: %v", err)
|
||||
}
|
||||
if len(defs) != 25 {
|
||||
t.Fatalf("definition count = %d, want 25", len(defs))
|
||||
}
|
||||
for _, def := range defs {
|
||||
if def.RetentionDays != defaultBuiltinMetricRetentionDays {
|
||||
t.Fatalf("%s retention = %d, want %d", def.Name, def.RetentionDays, defaultBuiltinMetricRetentionDays)
|
||||
}
|
||||
}
|
||||
|
||||
cpu, err := s.GetMetric(ctx, MetricCPU)
|
||||
if err != nil {
|
||||
t.Fatalf("get cpu definition: %v", err)
|
||||
}
|
||||
cpu.RetentionDays = 60
|
||||
if err := s.UpsertMetric(ctx, cpu); err != nil {
|
||||
t.Fatalf("override cpu retention: %v", err)
|
||||
}
|
||||
if err := createMetricDefinitions(ctx, s); err != nil {
|
||||
t.Fatalf("recreate definitions: %v", err)
|
||||
}
|
||||
cpu, err = s.GetMetric(ctx, MetricCPU)
|
||||
if err != nil {
|
||||
t.Fatalf("reload cpu definition: %v", err)
|
||||
}
|
||||
if cpu.RetentionDays != 60 {
|
||||
t.Fatalf("cpu retention = %d, want preserved override 60", cpu.RetentionDays)
|
||||
}
|
||||
if _, err := s.SetMetricRetention(ctx, MetricCPU, 0); err != nil {
|
||||
t.Fatalf("disable cpu retention: %v", err)
|
||||
}
|
||||
if err := createMetricDefinitions(ctx, s); err != nil {
|
||||
t.Fatalf("refresh disabled definition: %v", err)
|
||||
}
|
||||
cpu, err = s.GetMetric(ctx, MetricCPU)
|
||||
if err != nil {
|
||||
t.Fatalf("reload disabled cpu definition: %v", err)
|
||||
}
|
||||
if cpu.RetentionDays != 0 {
|
||||
t.Fatalf("cpu retention = %d, want preserved disabled state", cpu.RetentionDays)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCreateMetricDefinitionsUsesLegacySpanOnlyForNewDefinitions(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
s, err := metric.Open(ctx, metric.SQLite(":memory:", metric.WithMaxOpenConns(1)))
|
||||
if err != nil {
|
||||
t.Fatalf("open metric store: %v", err)
|
||||
}
|
||||
defer s.Close()
|
||||
|
||||
if err := createMetricDefinitionsWithDefaultRetention(ctx, s, 10); err != nil {
|
||||
t.Fatalf("create migration definitions: %v", err)
|
||||
}
|
||||
defs, err := s.ListMetrics(ctx)
|
||||
if err != nil {
|
||||
t.Fatalf("list migration definitions: %v", err)
|
||||
}
|
||||
for _, def := range defs {
|
||||
if def.RetentionDays != 10 {
|
||||
t.Fatalf("%s retention = %d, want legacy span 10", def.Name, def.RetentionDays)
|
||||
}
|
||||
}
|
||||
|
||||
cpu, err := s.GetMetric(ctx, MetricCPU)
|
||||
if err != nil {
|
||||
t.Fatalf("get CPU definition: %v", err)
|
||||
}
|
||||
cpu.RetentionDays = 3
|
||||
if err := s.UpsertMetric(ctx, cpu); err != nil {
|
||||
t.Fatalf("override CPU retention: %v", err)
|
||||
}
|
||||
if err := createMetricDefinitionsWithDefaultRetention(ctx, s, 20); err != nil {
|
||||
t.Fatalf("refresh migration definitions: %v", err)
|
||||
}
|
||||
cpu, err = s.GetMetric(ctx, MetricCPU)
|
||||
if err != nil {
|
||||
t.Fatalf("reload CPU definition: %v", err)
|
||||
}
|
||||
if cpu.RetentionDays != 3 {
|
||||
t.Fatalf("existing CPU retention = %d, want preserved 3", cpu.RetentionDays)
|
||||
}
|
||||
}
|
||||
|
||||
func TestGetRetentionSummaryUsesAllMetricDefinitions(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
s, err := metric.Open(ctx, metric.SQLite(":memory:", metric.WithMaxOpenConns(1)))
|
||||
if err != nil {
|
||||
t.Fatalf("open metric store: %v", err)
|
||||
}
|
||||
defer s.Close()
|
||||
|
||||
storeMu.Lock()
|
||||
oldStore := store
|
||||
store = s
|
||||
storeMu.Unlock()
|
||||
defer func() {
|
||||
storeMu.Lock()
|
||||
store = oldStore
|
||||
storeMu.Unlock()
|
||||
}()
|
||||
|
||||
empty, err := GetRetentionSummary(ctx)
|
||||
if err != nil {
|
||||
t.Fatalf("summarize empty store: %v", err)
|
||||
}
|
||||
if empty.AllPositive || empty.MaxDays != 0 {
|
||||
t.Fatalf("unexpected empty summary: %#v", empty)
|
||||
}
|
||||
for _, def := range []metric.Definition{
|
||||
{Name: "short", Type: metric.TypeGauge, RetentionDays: 7},
|
||||
{Name: "long", Type: metric.TypeGauge, RetentionDays: 60},
|
||||
} {
|
||||
if err := s.UpsertMetric(ctx, def); err != nil {
|
||||
t.Fatalf("upsert %s: %v", def.Name, err)
|
||||
}
|
||||
}
|
||||
summary, err := GetRetentionSummary(ctx)
|
||||
if err != nil {
|
||||
t.Fatalf("summarize definitions: %v", err)
|
||||
}
|
||||
if !summary.AllPositive || summary.MaxDays != 60 {
|
||||
t.Fatalf("unexpected summary: %#v", summary)
|
||||
}
|
||||
if _, err := s.SetMetricRetention(ctx, "short", 0); err != nil {
|
||||
t.Fatalf("disable short metric: %v", err)
|
||||
}
|
||||
summary, err = GetRetentionSummary(ctx)
|
||||
if err != nil {
|
||||
t.Fatalf("summarize disabled metric: %v", err)
|
||||
}
|
||||
if summary.AllPositive || summary.MaxDays != 60 {
|
||||
t.Fatalf("unexpected disabled summary: %#v", summary)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSummarizeRetentionDefinitionsRequiresEveryMetricToBePositive(t *testing.T) {
|
||||
summary := summarizeRetentionDefinitions([]metric.Definition{
|
||||
{Name: "enabled", RetentionDays: 30},
|
||||
{Name: "disabled", RetentionDays: 0},
|
||||
{Name: "long", RetentionDays: 60},
|
||||
})
|
||||
if summary.AllPositive || summary.MaxDays != 60 {
|
||||
t.Fatalf("unexpected summary: %#v", summary)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCompactCleansExpiredRawPointsWhenDownsamplingDisabled(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
s, err := metric.Open(ctx, metric.SQLite(":memory:", metric.WithMaxOpenConns(1)))
|
||||
if err != nil {
|
||||
t.Fatalf("open metric store: %v", err)
|
||||
}
|
||||
if err := s.UpsertMetric(ctx, metric.Definition{
|
||||
Name: "raw.metric",
|
||||
Type: metric.TypeGauge,
|
||||
RetentionDays: 1,
|
||||
}); err != nil {
|
||||
t.Fatalf("upsert metric: %v", err)
|
||||
}
|
||||
|
||||
now := time.Date(2026, 7, 11, 12, 0, 0, 0, time.UTC)
|
||||
if err := s.WriteBatch(ctx, []metric.Point{
|
||||
{MetricName: "raw.metric", EntityID: "node", Timestamp: now.Add(-48 * time.Hour), Value: 1},
|
||||
{MetricName: "raw.metric", EntityID: "node", Timestamp: now.Add(-time.Hour), Value: 2},
|
||||
}); err != nil {
|
||||
t.Fatalf("write points: %v", err)
|
||||
}
|
||||
|
||||
storeMu.Lock()
|
||||
oldStore := store
|
||||
oldCompactAt := compactAt
|
||||
store = s
|
||||
compactAt = 0
|
||||
storeMu.Unlock()
|
||||
defer func() {
|
||||
storeMu.Lock()
|
||||
store = oldStore
|
||||
compactAt = oldCompactAt
|
||||
storeMu.Unlock()
|
||||
_ = s.Close()
|
||||
}()
|
||||
|
||||
if _, err := Compact(ctx, now); err != nil {
|
||||
t.Fatalf("compact: %v", err)
|
||||
}
|
||||
points, err := s.Query(ctx, metric.Query{
|
||||
MetricName: "raw.metric",
|
||||
EntityID: "node",
|
||||
Start: now.Add(-72 * time.Hour),
|
||||
End: now,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("query points: %v", err)
|
||||
}
|
||||
if len(points) != 1 || points[0].Value != 2 {
|
||||
t.Fatalf("expected only the retained raw point, got %#v", points)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCompactKeepsRotatingCursorAfterFullCycle(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
s, err := metric.Open(ctx, metric.SQLite(":memory:",
|
||||
metric.WithMaxOpenConns(1),
|
||||
metric.WithRollupPolicy(defaultRollupPolicy()),
|
||||
))
|
||||
if err != nil {
|
||||
t.Fatalf("open metric store: %v", err)
|
||||
}
|
||||
for _, name := range []string{"a.metric", "b.metric", "c.metric"} {
|
||||
if err := s.UpsertMetric(ctx, metric.Definition{Name: name, Type: metric.TypeGauge}); err != nil {
|
||||
t.Fatalf("upsert metric %s: %v", name, err)
|
||||
}
|
||||
}
|
||||
|
||||
storeMu.Lock()
|
||||
oldStore := store
|
||||
oldCompactAt := compactAt
|
||||
store = s
|
||||
compactAt = 1
|
||||
storeMu.Unlock()
|
||||
defer func() {
|
||||
storeMu.Lock()
|
||||
store = oldStore
|
||||
compactAt = oldCompactAt
|
||||
storeMu.Unlock()
|
||||
_ = s.Close()
|
||||
}()
|
||||
|
||||
if _, err := Compact(ctx, time.Date(2026, 7, 11, 12, 0, 0, 0, time.UTC)); err != nil {
|
||||
t.Fatalf("compact: %v", err)
|
||||
}
|
||||
if compactAt != 1 {
|
||||
t.Fatalf("compact cursor = %d, want 1 after a complete rotated cycle", compactAt)
|
||||
}
|
||||
}
|
||||
|
||||
func TestGetRecordsByClientAndTimeReadsRollupsAfterRawCompaction(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
s, err := metric.Open(ctx, metric.SQLite(":memory:",
|
||||
metric.WithMaxOpenConns(1),
|
||||
metric.WithRollupPolicy(defaultRollupPolicy()),
|
||||
))
|
||||
if err != nil {
|
||||
t.Fatalf("open metric store: %v", err)
|
||||
}
|
||||
if err := createMetricDefinitions(ctx, s); err != nil {
|
||||
t.Fatalf("create metric definitions: %v", err)
|
||||
}
|
||||
|
||||
storeMu.Lock()
|
||||
oldStore := store
|
||||
oldCompactAt := compactAt
|
||||
store = s
|
||||
compactAt = 0
|
||||
storeMu.Unlock()
|
||||
defer func() {
|
||||
storeMu.Lock()
|
||||
store = oldStore
|
||||
compactAt = oldCompactAt
|
||||
storeMu.Unlock()
|
||||
_ = s.Close()
|
||||
}()
|
||||
|
||||
now := time.Now().UTC().Truncate(time.Minute)
|
||||
ts := now.Add(-time.Hour)
|
||||
rec := models.Record{
|
||||
Client: "node-a",
|
||||
Time: ts,
|
||||
Cpu: 42.5,
|
||||
Ram: 123456,
|
||||
RamTotal: 999999,
|
||||
Disk: 456789,
|
||||
DiskTotal: 777777,
|
||||
Load: 0.75,
|
||||
Connections: 321,
|
||||
ConnectionsUdp: 12,
|
||||
}
|
||||
if _, err := WriteReport(ctx, v1.Report{
|
||||
UUID: rec.Client,
|
||||
UpdatedAt: ts,
|
||||
CPU: v1.CPUReport{Usage: float64(rec.Cpu)},
|
||||
Ram: v1.RamReport{Used: rec.Ram, Total: rec.RamTotal},
|
||||
Load: v1.LoadReport{Load1: float64(rec.Load)},
|
||||
Disk: v1.DiskReport{Used: rec.Disk, Total: rec.DiskTotal},
|
||||
Process: rec.Process,
|
||||
Connections: v1.ConnectionsReport{
|
||||
TCP: rec.Connections,
|
||||
UDP: rec.ConnectionsUdp,
|
||||
},
|
||||
}); err != nil {
|
||||
t.Fatalf("write record: %v", err)
|
||||
}
|
||||
if _, err := s.Compact(ctx, now); err != nil {
|
||||
t.Fatalf("compact raw into rollup: %v", err)
|
||||
}
|
||||
raw, err := s.Query(ctx, metric.Query{
|
||||
MetricName: MetricCPU,
|
||||
EntityID: rec.Client,
|
||||
Start: ts.Add(-time.Minute),
|
||||
End: now,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("query raw cpu: %v", err)
|
||||
}
|
||||
if len(raw) != 0 {
|
||||
t.Fatalf("expected old raw cpu point to be deleted after compaction, got %d", len(raw))
|
||||
}
|
||||
|
||||
got, err := GetRecordsByClientAndTime(ctx, rec.Client, ts.Add(-time.Minute), now)
|
||||
if err != nil {
|
||||
t.Fatalf("get records: %v", err)
|
||||
}
|
||||
if len(got) != 1 {
|
||||
t.Fatalf("expected 1 reconstructed record from rollup, got %d: %#v", len(got), got)
|
||||
}
|
||||
if got[0].Cpu == 0 || got[0].Ram == 0 || got[0].Disk == 0 || got[0].Connections == 0 {
|
||||
t.Fatalf("record was not reconstructed from rollup: %#v", got[0])
|
||||
}
|
||||
|
||||
all, err := GetRecordsByTime(ctx, ts.Add(-time.Minute), now)
|
||||
if err != nil {
|
||||
t.Fatalf("get all records: %v", err)
|
||||
}
|
||||
if len(all) != 1 || all[0].Client != rec.Client || all[0].Cpu == 0 {
|
||||
t.Fatalf("all-client records were not reconstructed from rollup: %#v", all)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,72 @@
|
||||
package metricstore
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"log"
|
||||
"strings"
|
||||
|
||||
"github.com/komari-monitor/komari/pkg/config"
|
||||
)
|
||||
|
||||
// migration.go
|
||||
//
|
||||
// 启动阶段的 metrics 数据迁移。这里只处理 metrics 存储后端切换时的数据搬运
|
||||
// (例如默认 SQLite ./data/metrics.db 切换到 MySQL/PostgreSQL)。旧 komari.db
|
||||
// 监控表导入属于一次性迁移,见 pkg/migrations.RunMetricStoreMigrations。
|
||||
|
||||
// targetFingerprint 返回当前 metrics 目标库的指纹(driver + 归一化 DSN),
|
||||
// 用于判断 metrics 存储后端是否发生变化(例如从 SQLite 切换到 MySQL/PostgreSQL)。
|
||||
func targetFingerprint(cfg *MetricStoreConfig) string {
|
||||
driver := ResolveDriverFromConfig(cfg.Driver, cfg.DSN)
|
||||
dsn := strings.TrimSpace(cfg.DSN)
|
||||
return fmt.Sprintf("%s|%s", driver, dsn)
|
||||
}
|
||||
|
||||
// RunStartupMigration 在服务启动时检查 metrics 存储后端是否发生变化,
|
||||
// 若变化则把上一个 metrics 目标库的全部数据搬运到当前目标。
|
||||
//
|
||||
// 判定依据 MigrationTargetKey(记录“数据当前完整所在的 metrics 目标指纹”):
|
||||
// - saved == current:数据已在当前目标,无需处理。
|
||||
// - saved == "":无历史目标记录。老快照的 metrics 数据固定落在默认 SQLite
|
||||
// (./data/metrics.db);若当前目标不是默认 SQLite(例如已切到 MySQL/PostgreSQL),
|
||||
// 把默认 SQLite 中可能存在的历史数据搬运过来,否则仅登记指纹。
|
||||
// - saved != current:后端已切换,把上一个目标(saved)的数据搬运到当前目标。
|
||||
//
|
||||
// 搬运以 upsert 写入,幂等,中断后重启可安全重跑。任一失败都返回错误,
|
||||
// 调用方应让启动失败并打印明确错误。
|
||||
func RunStartupMigration() error {
|
||||
s := GetStore()
|
||||
if s == nil {
|
||||
return fmt.Errorf("metric store not initialized")
|
||||
}
|
||||
|
||||
cfg, err := config.GetManyAs[MetricStoreConfig]()
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to load metric store config: %w", err)
|
||||
}
|
||||
|
||||
current := targetFingerprint(cfg)
|
||||
saved, _ := config.GetAs[string](MigrationTargetKey, "")
|
||||
|
||||
// 数据已完整位于当前目标:无需搬运。
|
||||
if saved == current {
|
||||
return nil
|
||||
}
|
||||
|
||||
// 上一个目标:优先使用已保存指纹;无记录时按老快照默认 SQLite 推断。
|
||||
prev := saved
|
||||
if prev == "" {
|
||||
prev = defaultSQLiteFingerprint()
|
||||
}
|
||||
|
||||
if prev != current {
|
||||
if err := migrateFromPreviousStore(prev, cfg, s); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
if err := config.Set(MigrationTargetKey, current); err != nil {
|
||||
log.Printf("Failed to persist migration target fingerprint: %v", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,211 @@
|
||||
package metricstore
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"log"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/komari-monitor/komari/pkg/metric"
|
||||
)
|
||||
|
||||
// storeMigrationWindow 是 store-to-store 迁移时按时间分窗查询的窗口大小。
|
||||
// 采样点按时间窗口分批拉取,避免一次性把整段序列读入内存。
|
||||
const storeMigrationWindow = 6 * time.Hour
|
||||
|
||||
// configFromFingerprint 从目标指纹(driver|dsn)重建一个 MetricStoreConfig,
|
||||
// 用于以只读方式打开“上一次的 metrics 目标库”。表前缀、保留天数、连接数等
|
||||
// 沿用当前配置(切换后端时这些通常不变)。
|
||||
func configFromFingerprint(fingerprint string, base *MetricStoreConfig) (*MetricStoreConfig, error) {
|
||||
idx := strings.Index(fingerprint, "|")
|
||||
if idx < 0 {
|
||||
return nil, fmt.Errorf("invalid target fingerprint: %q", fingerprint)
|
||||
}
|
||||
driver := fingerprint[:idx]
|
||||
dsn := fingerprint[idx+1:]
|
||||
if strings.TrimSpace(driver) == "" {
|
||||
return nil, fmt.Errorf("empty driver in target fingerprint: %q", fingerprint)
|
||||
}
|
||||
return &MetricStoreConfig{
|
||||
Driver: driver,
|
||||
DSN: dsn,
|
||||
DownsamplingEnabled: base.DownsamplingEnabled,
|
||||
TablePrefix: base.TablePrefix,
|
||||
MaxOpenConns: base.MaxOpenConns,
|
||||
MaxIdleConns: base.MaxIdleConns,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// openSourceStore 打开一个已存在的 metrics 目标库作为数据搬运的源库读取。
|
||||
//
|
||||
// 使用 autoMigrate=true:GORM 的 AutoMigrate 只新增表/列/索引、从不删除数据,
|
||||
// 因此对真实旧库幂等无害;而当源库文件/表不存在(例如老快照记录了 completed
|
||||
// 但 metrics.db 缺失)时,可创建空表让后续 ListMetrics 返回空集而非报
|
||||
// "no such table",从而把“无历史可迁移”识别为正常情况而非错误。
|
||||
func openSourceStore(ctx context.Context, cfg *MetricStoreConfig) (*metric.Store, error) {
|
||||
metricCfg, err := buildMetricConfig(cfg, true)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return metric.Open(ctx, metricCfg)
|
||||
}
|
||||
|
||||
// defaultSQLiteFingerprint 返回默认 SQLite metrics 库(./data/metrics.db)的目标指纹。
|
||||
// 老快照的 metrics 数据固定落在该 SQLite 文件,用于在缺失指纹时推断上一个源库。
|
||||
func defaultSQLiteFingerprint() string {
|
||||
return targetFingerprint(&MetricStoreConfig{Driver: "sqlite", DSN: "./data/metrics.db"})
|
||||
}
|
||||
|
||||
// migrateFromPreviousStore 打开由 prevFingerprint 指定的上一个 metrics 目标库作为源,
|
||||
// 把其中的全部指标搬运到当前目标 dst(例如 SQLite metrics.db → MySQL/PostgreSQL)。
|
||||
//
|
||||
// 该过程幂等:dst 以 (metric, entity, tags, ts) upsert 写入,中断后重启可安全重跑。
|
||||
func migrateFromPreviousStore(prevFingerprint string, cfg *MetricStoreConfig, dst *metric.Store) error {
|
||||
prevCfg, err := configFromFingerprint(prevFingerprint, cfg)
|
||||
if err != nil {
|
||||
return fmt.Errorf("parse previous metrics target %q: %w", prevFingerprint, err)
|
||||
}
|
||||
|
||||
ctx := context.Background()
|
||||
src, err := openSourceStore(ctx, prevCfg)
|
||||
if err != nil {
|
||||
return fmt.Errorf("open previous metrics store (%s): %w", prevFingerprint, err)
|
||||
}
|
||||
defer src.Close()
|
||||
|
||||
log.Printf("Migrating metrics from previous store %s to current target...", prevFingerprint)
|
||||
total, err := MigrateBetweenStores(ctx, src, dst)
|
||||
if err != nil {
|
||||
return fmt.Errorf("store-to-store metrics migration failed: %w", err)
|
||||
}
|
||||
log.Printf("Store-to-store metrics migration completed (%d points)", total)
|
||||
return nil
|
||||
}
|
||||
|
||||
// storeMigrationObserver 在 store-to-store 迁移过程中接收进度回调。
|
||||
// - currentMetric:当前正在搬运的指标名。
|
||||
// - metricIndex:该指标在全部指标中的序号(0 起),即已完成的指标数。
|
||||
// - totalMetrics:指标定义总数。
|
||||
// - addedPoints:本次新写入目标库的采样点数(用于外部累计)。
|
||||
type storeMigrationObserver func(currentMetric string, metricIndex, totalMetrics int, addedPoints int64)
|
||||
|
||||
// MigrateBetweenStores 把 src metric store 的全部指标定义与采样点搬运到 dst。
|
||||
//
|
||||
// 用于 metrics 后端切换(例如默认 SQLite metrics.db → MySQL/PostgreSQL):
|
||||
// - 先在 dst 建立/更新全部指标定义;
|
||||
// - 再按指标、按时间窗口分批读取源库采样点并写入 dst。
|
||||
//
|
||||
// 采样点在 dst 以 (metric_name, entity_id, tags, ts) upsert 写入,窗口边界重叠也
|
||||
// 幂等,因此整个过程可安全重试。返回搬运的采样点总数。
|
||||
func MigrateBetweenStores(ctx context.Context, src, dst *metric.Store) (int64, error) {
|
||||
return migrateBetweenStores(ctx, src, dst, nil)
|
||||
}
|
||||
|
||||
// migrateBetweenStores 是 MigrateBetweenStores 的内部实现,额外接受一个可选的进度
|
||||
// 观察者 observe(为 nil 时行为与旧版本完全一致)。启动迁移走 nil,WebUI/API 触发
|
||||
// 的迁移传入回调以实时更新进度。
|
||||
func migrateBetweenStores(ctx context.Context, src, dst *metric.Store, observe storeMigrationObserver) (int64, error) {
|
||||
if src == nil || dst == nil {
|
||||
return 0, fmt.Errorf("source or destination metric store is nil")
|
||||
}
|
||||
|
||||
defs, err := src.ListMetrics(ctx)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("list source metrics: %w", err)
|
||||
}
|
||||
|
||||
var total int64
|
||||
for i, def := range defs {
|
||||
if observe != nil {
|
||||
// 进入下一个指标:已完成 i 个指标。
|
||||
observe(def.Name, i, len(defs), 0)
|
||||
}
|
||||
// 目标库先建立指标定义,保证后续写入的指标存在。
|
||||
if err := dst.UpsertMetric(ctx, def); err != nil {
|
||||
return total, fmt.Errorf("upsert metric %q on target: %w", def.Name, err)
|
||||
}
|
||||
if def.RetentionDays == 0 {
|
||||
continue
|
||||
}
|
||||
|
||||
earliest, latest, ok, err := metricTimeBounds(ctx, src, def.Name)
|
||||
if err != nil {
|
||||
return total, fmt.Errorf("resolve time bounds for metric %q: %w", def.Name, err)
|
||||
}
|
||||
if !ok {
|
||||
// 该指标只有定义没有数据,跳过。
|
||||
continue
|
||||
}
|
||||
|
||||
var migrated int64
|
||||
for windowStart := earliest; !windowStart.After(latest); windowStart = windowStart.Add(storeMigrationWindow) {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return total, ctx.Err()
|
||||
default:
|
||||
}
|
||||
|
||||
windowEnd := windowStart.Add(storeMigrationWindow)
|
||||
points, err := src.Query(ctx, metric.Query{
|
||||
MetricName: def.Name,
|
||||
Start: windowStart,
|
||||
End: windowEnd,
|
||||
Order: metric.OrderAsc,
|
||||
})
|
||||
if err != nil {
|
||||
return total, fmt.Errorf("query metric %q window [%s,%s): %w", def.Name, windowStart, windowEnd, err)
|
||||
}
|
||||
if len(points) == 0 {
|
||||
continue
|
||||
}
|
||||
if err := dst.WriteBatch(ctx, points); err != nil {
|
||||
return total, fmt.Errorf("write metric %q batch to target: %w", def.Name, err)
|
||||
}
|
||||
migrated += int64(len(points))
|
||||
total += int64(len(points))
|
||||
if observe != nil {
|
||||
observe(def.Name, i, len(defs), int64(len(points)))
|
||||
}
|
||||
}
|
||||
if migrated > 0 {
|
||||
log.Printf("[store-migration] metric %q: migrated %d points", def.Name, migrated)
|
||||
}
|
||||
}
|
||||
|
||||
return total, nil
|
||||
}
|
||||
|
||||
// metricTimeBounds 返回某指标在 src 中最早/最晚采样时间。ok=false 表示无数据。
|
||||
// 通过升/降序各取一条采样点定位边界,避免把整段序列读入内存。
|
||||
func metricTimeBounds(ctx context.Context, src *metric.Store, name string) (time.Time, time.Time, bool, error) {
|
||||
wide := metric.Query{
|
||||
MetricName: name,
|
||||
Start: time.Unix(0, 0),
|
||||
End: time.Now().UTC().Add(24 * time.Hour),
|
||||
}
|
||||
|
||||
asc := wide
|
||||
asc.Order = metric.OrderAsc
|
||||
asc.Limit = 1
|
||||
first, err := src.Query(ctx, asc)
|
||||
if err != nil {
|
||||
return time.Time{}, time.Time{}, false, err
|
||||
}
|
||||
if len(first) == 0 {
|
||||
return time.Time{}, time.Time{}, false, nil
|
||||
}
|
||||
|
||||
desc := wide
|
||||
desc.Order = metric.OrderDesc
|
||||
desc.Limit = 1
|
||||
last, err := src.Query(ctx, desc)
|
||||
if err != nil {
|
||||
return time.Time{}, time.Time{}, false, err
|
||||
}
|
||||
if len(last) == 0 {
|
||||
return time.Time{}, time.Time{}, false, nil
|
||||
}
|
||||
|
||||
return first[0].Timestamp, last[0].Timestamp, true, nil
|
||||
}
|
||||
@@ -0,0 +1,38 @@
|
||||
package metricstore
|
||||
|
||||
import "context"
|
||||
|
||||
// storeOperationGate serializes operations that need a stable active Store.
|
||||
// A channel-backed gate supports both non-blocking scheduled work and
|
||||
// context-aware admin/shutdown waits.
|
||||
type storeOperationGate struct {
|
||||
token chan struct{}
|
||||
}
|
||||
|
||||
func newStoreOperationGate() *storeOperationGate {
|
||||
gate := &storeOperationGate{token: make(chan struct{}, 1)}
|
||||
gate.token <- struct{}{}
|
||||
return gate
|
||||
}
|
||||
|
||||
func (g *storeOperationGate) Acquire(ctx context.Context) error {
|
||||
select {
|
||||
case <-g.token:
|
||||
return nil
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
}
|
||||
}
|
||||
|
||||
func (g *storeOperationGate) TryAcquire() bool {
|
||||
select {
|
||||
case <-g.token:
|
||||
return true
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
func (g *storeOperationGate) Release() {
|
||||
g.token <- struct{}{}
|
||||
}
|
||||
@@ -0,0 +1,467 @@
|
||||
package metricstore
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"log"
|
||||
"strconv"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/komari-monitor/komari/database/models"
|
||||
"github.com/komari-monitor/komari/pkg/metric"
|
||||
v1 "github.com/komari-monitor/komari/protocol/v1"
|
||||
)
|
||||
|
||||
type reportTrafficState struct {
|
||||
mu sync.Mutex
|
||||
|
||||
reportTrafficValues
|
||||
}
|
||||
|
||||
type reportTrafficValues struct {
|
||||
initialized bool
|
||||
timestamp time.Time
|
||||
hasUp bool
|
||||
totalUp int64
|
||||
hasDown bool
|
||||
totalDown int64
|
||||
}
|
||||
|
||||
var reportTrafficStates sync.Map
|
||||
|
||||
const (
|
||||
reportBatchInterval = 3 * time.Second
|
||||
reportBatchQueueSize = 4096
|
||||
reportBatchMaxReports = 256 * 21
|
||||
reportBatchWriteTimeout = 10 * time.Second
|
||||
)
|
||||
|
||||
var (
|
||||
reportBatcherMu sync.Mutex
|
||||
reportBatcher *reportBatchWorker
|
||||
)
|
||||
|
||||
var (
|
||||
ErrReportBatchQueueFull = errors.New("metric report batch queue is full")
|
||||
ErrReportBatchStopped = errors.New("metric report batcher is stopped")
|
||||
)
|
||||
|
||||
type reportBatchRequest struct {
|
||||
ctx context.Context
|
||||
done chan error
|
||||
stop bool
|
||||
}
|
||||
|
||||
type reportBatchWorker struct {
|
||||
mu sync.Mutex
|
||||
queue chan v1.Report
|
||||
requests chan reportBatchRequest
|
||||
done chan struct{}
|
||||
stopping bool
|
||||
}
|
||||
|
||||
// StartReportBatcher starts the shared report writer. Reports are collected for
|
||||
// a short window so many agents can be persisted in one database transaction.
|
||||
func StartReportBatcher() {
|
||||
reportBatcherMu.Lock()
|
||||
defer reportBatcherMu.Unlock()
|
||||
if reportBatcher != nil {
|
||||
return
|
||||
}
|
||||
worker := &reportBatchWorker{
|
||||
queue: make(chan v1.Report, reportBatchQueueSize),
|
||||
requests: make(chan reportBatchRequest, 1),
|
||||
done: make(chan struct{}),
|
||||
}
|
||||
reportBatcher = worker
|
||||
go worker.run()
|
||||
}
|
||||
|
||||
// StopReportBatcher stops the report writer after flushing all queued reports.
|
||||
func StopReportBatcher(ctx context.Context) error {
|
||||
reportBatcherMu.Lock()
|
||||
worker := reportBatcher
|
||||
if worker == nil {
|
||||
reportBatcherMu.Unlock()
|
||||
return nil
|
||||
}
|
||||
worker.mu.Lock()
|
||||
worker.stopping = true
|
||||
worker.mu.Unlock()
|
||||
reportBatcherMu.Unlock()
|
||||
|
||||
request := reportBatchRequest{ctx: ctx, done: make(chan error, 1), stop: true}
|
||||
select {
|
||||
case worker.requests <- request:
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
}
|
||||
select {
|
||||
case err := <-request.done:
|
||||
<-worker.done
|
||||
reportBatcherMu.Lock()
|
||||
if reportBatcher == worker {
|
||||
reportBatcher = nil
|
||||
}
|
||||
reportBatcherMu.Unlock()
|
||||
return err
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
}
|
||||
}
|
||||
|
||||
// FlushReportBatch synchronously flushes the current queue. It is useful for
|
||||
// controlled handoff points and deterministic tests; normal operation relies
|
||||
// on the three-second ticker.
|
||||
func FlushReportBatch(ctx context.Context) error {
|
||||
reportBatcherMu.Lock()
|
||||
worker := reportBatcher
|
||||
reportBatcherMu.Unlock()
|
||||
if worker == nil {
|
||||
return nil
|
||||
}
|
||||
request := reportBatchRequest{ctx: ctx, done: make(chan error, 1)}
|
||||
worker.mu.Lock()
|
||||
stopping := worker.stopping
|
||||
worker.mu.Unlock()
|
||||
if stopping {
|
||||
return ErrReportBatchStopped
|
||||
}
|
||||
select {
|
||||
case worker.requests <- request:
|
||||
case <-worker.done:
|
||||
return ErrReportBatchStopped
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
}
|
||||
select {
|
||||
case err := <-request.done:
|
||||
return err
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
}
|
||||
}
|
||||
|
||||
// WriteReport persists one agent report as raw metric points sharing the same
|
||||
// server receive time. Traffic deltas are derived from the previous counters
|
||||
// and stored alongside the raw counters so they remain summable after rollup.
|
||||
func WriteReport(ctx context.Context, report v1.Report) (v1.Report, error) {
|
||||
if report.UUID == "" {
|
||||
return v1.Report{}, fmt.Errorf("report UUID is required")
|
||||
}
|
||||
if report.UpdatedAt.IsZero() {
|
||||
return v1.Report{}, fmt.Errorf("report receive time is required")
|
||||
}
|
||||
report.UpdatedAt = report.UpdatedAt.UTC()
|
||||
if GetStore() == nil {
|
||||
return v1.Report{}, fmt.Errorf("metric store not enabled")
|
||||
}
|
||||
|
||||
reportBatcherMu.Lock()
|
||||
worker := reportBatcher
|
||||
reportBatcherMu.Unlock()
|
||||
if worker != nil {
|
||||
if err := worker.enqueue(ctx, report); err != nil {
|
||||
return v1.Report{}, err
|
||||
}
|
||||
return report, nil
|
||||
}
|
||||
|
||||
saved, err := writeReportBatch(ctx, []v1.Report{report})
|
||||
if err != nil {
|
||||
return v1.Report{}, err
|
||||
}
|
||||
return saved[0], nil
|
||||
}
|
||||
|
||||
func (w *reportBatchWorker) enqueue(ctx context.Context, report v1.Report) error {
|
||||
w.mu.Lock()
|
||||
defer w.mu.Unlock()
|
||||
if w.stopping {
|
||||
return ErrReportBatchStopped
|
||||
}
|
||||
if err := ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
select {
|
||||
case w.queue <- report:
|
||||
return nil
|
||||
default:
|
||||
return ErrReportBatchQueueFull
|
||||
}
|
||||
}
|
||||
|
||||
func (w *reportBatchWorker) run() {
|
||||
ticker := time.NewTicker(reportBatchInterval)
|
||||
defer ticker.Stop()
|
||||
|
||||
var pending []v1.Report
|
||||
for {
|
||||
select {
|
||||
case report := <-w.queue:
|
||||
pending = append(pending, report)
|
||||
case request := <-w.requests:
|
||||
pending = append(pending, drainReportQueue(w.queue, reportBatchMaxReports-len(pending))...)
|
||||
err := writePendingReports(request.ctx, &pending)
|
||||
if request.stop {
|
||||
if err != nil {
|
||||
log.Printf("failed to flush metric report batch during shutdown: %v", err)
|
||||
}
|
||||
close(w.done)
|
||||
request.done <- err
|
||||
return
|
||||
}
|
||||
request.done <- err
|
||||
case <-ticker.C:
|
||||
pending = append(pending, drainReportQueue(w.queue, reportBatchMaxReports-len(pending))...)
|
||||
if err := writePendingReports(context.Background(), &pending); err != nil {
|
||||
log.Printf("failed to flush metric report batch: %v", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func drainReportQueue(queue <-chan v1.Report, limit int) []v1.Report {
|
||||
if limit <= 0 {
|
||||
return nil
|
||||
}
|
||||
reports := make([]v1.Report, 0, limit)
|
||||
for len(reports) < limit {
|
||||
select {
|
||||
case report := <-queue:
|
||||
reports = append(reports, report)
|
||||
default:
|
||||
return reports
|
||||
}
|
||||
}
|
||||
return reports
|
||||
}
|
||||
|
||||
func writePendingReports(ctx context.Context, pending *[]v1.Report) error {
|
||||
if len(*pending) == 0 {
|
||||
return nil
|
||||
}
|
||||
writeCtx, cancel := context.WithTimeout(ctx, reportBatchWriteTimeout)
|
||||
defer cancel()
|
||||
if err := func() error {
|
||||
reports := *pending
|
||||
_, err := writeReportBatch(writeCtx, reports)
|
||||
return err
|
||||
}(); err != nil {
|
||||
return err
|
||||
}
|
||||
*pending = (*pending)[:0]
|
||||
return nil
|
||||
}
|
||||
|
||||
func writeReportBatch(ctx context.Context, reports []v1.Report) ([]v1.Report, error) {
|
||||
if len(reports) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
if err := storeOperations.Acquire(ctx); err != nil {
|
||||
return nil, fmt.Errorf("wait for metric store operation before writing reports: %w", err)
|
||||
}
|
||||
defer storeOperations.Release()
|
||||
|
||||
s := GetStore()
|
||||
if s == nil {
|
||||
return nil, fmt.Errorf("metric store not enabled")
|
||||
}
|
||||
|
||||
prepared := make([]v1.Report, len(reports))
|
||||
copy(prepared, reports)
|
||||
points := make([]metric.Point, 0, len(reports)*20)
|
||||
pendingStates := make(map[*reportTrafficState]reportTrafficValues)
|
||||
for i, report := range prepared {
|
||||
stateValue, _ := reportTrafficStates.LoadOrStore(report.UUID, &reportTrafficState{})
|
||||
state := stateValue.(*reportTrafficState)
|
||||
values, ok := pendingStates[state]
|
||||
if !ok {
|
||||
state.mu.Lock()
|
||||
values = state.reportTrafficValues
|
||||
state.mu.Unlock()
|
||||
}
|
||||
if !values.initialized {
|
||||
var err error
|
||||
values.totalUp, values.hasUp, err = latestReportCounter(ctx, s, MetricNetTotalUp, report.UUID, report.UpdatedAt)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("load previous upload counter: %w", err)
|
||||
}
|
||||
values.totalDown, values.hasDown, err = latestReportCounter(ctx, s, MetricNetTotalDown, report.UUID, report.UpdatedAt)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("load previous download counter: %w", err)
|
||||
}
|
||||
values.initialized = true
|
||||
}
|
||||
|
||||
if !values.timestamp.IsZero() && !report.UpdatedAt.After(values.timestamp) {
|
||||
report.UpdatedAt = values.timestamp.Add(time.Nanosecond)
|
||||
}
|
||||
trafficUp := int64(0)
|
||||
if values.hasUp {
|
||||
trafficUp = TrafficCounterDelta(report.Network.TotalUp, values.totalUp)
|
||||
}
|
||||
trafficDown := int64(0)
|
||||
if values.hasDown {
|
||||
trafficDown = TrafficCounterDelta(report.Network.TotalDown, values.totalDown)
|
||||
}
|
||||
points = append(points, reportMetricPoints(report, trafficUp, trafficDown)...)
|
||||
values.timestamp = report.UpdatedAt
|
||||
values.hasUp = true
|
||||
values.totalUp = report.Network.TotalUp
|
||||
values.hasDown = true
|
||||
values.totalDown = report.Network.TotalDown
|
||||
pendingStates[state] = values
|
||||
prepared[i] = report
|
||||
}
|
||||
|
||||
if err := s.WriteBatch(ctx, points); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for state, values := range pendingStates {
|
||||
state.mu.Lock()
|
||||
state.reportTrafficValues = values
|
||||
state.mu.Unlock()
|
||||
}
|
||||
return prepared, nil
|
||||
}
|
||||
|
||||
func reportMetricPoints(report v1.Report, trafficUp, trafficDown int64) []metric.Point {
|
||||
entityID := report.UUID
|
||||
ts := report.UpdatedAt
|
||||
gpuUsage := 0.0
|
||||
if report.GPU != nil {
|
||||
gpuUsage = report.GPU.AverageUsage
|
||||
}
|
||||
points := []metric.Point{
|
||||
{MetricName: MetricCPU, EntityID: entityID, Timestamp: ts, Value: report.CPU.Usage},
|
||||
{MetricName: MetricGPU, EntityID: entityID, Timestamp: ts, Value: gpuUsage},
|
||||
{MetricName: MetricRAM, EntityID: entityID, Timestamp: ts, Value: float64(report.Ram.Used)},
|
||||
{MetricName: MetricRAMTotal, EntityID: entityID, Timestamp: ts, Value: float64(report.Ram.Total)},
|
||||
{MetricName: MetricSwap, EntityID: entityID, Timestamp: ts, Value: float64(report.Swap.Used)},
|
||||
{MetricName: MetricSwapTotal, EntityID: entityID, Timestamp: ts, Value: float64(report.Swap.Total)},
|
||||
{MetricName: MetricLoad, EntityID: entityID, Timestamp: ts, Value: report.Load.Load1},
|
||||
{MetricName: MetricTemp, EntityID: entityID, Timestamp: ts, Value: 0},
|
||||
{MetricName: MetricDisk, EntityID: entityID, Timestamp: ts, Value: float64(report.Disk.Used)},
|
||||
{MetricName: MetricDiskTotal, EntityID: entityID, Timestamp: ts, Value: float64(report.Disk.Total)},
|
||||
{MetricName: MetricNetIn, EntityID: entityID, Timestamp: ts, Value: float64(report.Network.Down)},
|
||||
{MetricName: MetricNetOut, EntityID: entityID, Timestamp: ts, Value: float64(report.Network.Up)},
|
||||
{MetricName: MetricNetTotalUp, EntityID: entityID, Timestamp: ts, Value: float64(report.Network.TotalUp)},
|
||||
{MetricName: MetricNetTotalDown, EntityID: entityID, Timestamp: ts, Value: float64(report.Network.TotalDown)},
|
||||
{MetricName: MetricTrafficUp, EntityID: entityID, Timestamp: ts, Value: float64(trafficUp)},
|
||||
{MetricName: MetricTrafficDown, EntityID: entityID, Timestamp: ts, Value: float64(trafficDown)},
|
||||
{MetricName: MetricProcess, EntityID: entityID, Timestamp: ts, Value: float64(report.Process)},
|
||||
{MetricName: MetricConnections, EntityID: entityID, Timestamp: ts, Value: float64(report.Connections.TCP)},
|
||||
{MetricName: MetricConnectionsUDP, EntityID: entityID, Timestamp: ts, Value: float64(report.Connections.UDP)},
|
||||
}
|
||||
if report.GPU == nil {
|
||||
return points
|
||||
}
|
||||
for deviceIndex, gpu := range report.GPU.DetailedInfo {
|
||||
tags := map[string]string{
|
||||
"device_index": strconv.Itoa(deviceIndex),
|
||||
"device_name": gpu.Name,
|
||||
}
|
||||
points = append(points,
|
||||
metric.Point{MetricName: MetricGPUMem, EntityID: entityID, Timestamp: ts, Value: float64(gpu.MemoryUsed), Tags: tags},
|
||||
metric.Point{MetricName: MetricGPUMemTotal, EntityID: entityID, Timestamp: ts, Value: float64(gpu.MemoryTotal), Tags: tags},
|
||||
metric.Point{MetricName: MetricGPUDeviceUsage, EntityID: entityID, Timestamp: ts, Value: gpu.Utilization, Tags: tags},
|
||||
metric.Point{MetricName: MetricGPUTemp, EntityID: entityID, Timestamp: ts, Value: float64(gpu.Temperature), Tags: tags},
|
||||
)
|
||||
}
|
||||
return points
|
||||
}
|
||||
|
||||
func latestReportCounter(ctx context.Context, s *metric.Store, metricName, entityID string, before time.Time) (int64, bool, error) {
|
||||
def, err := s.GetMetric(ctx, metricName)
|
||||
if errors.Is(err, metric.ErrNotFound) {
|
||||
return 0, false, nil
|
||||
}
|
||||
if err != nil {
|
||||
return 0, false, err
|
||||
}
|
||||
if def.RetentionDays <= 0 {
|
||||
return 0, false, nil
|
||||
}
|
||||
|
||||
end := before.UTC().Add(-time.Nanosecond)
|
||||
start := end.Add(-time.Duration(def.RetentionDays) * 24 * time.Hour)
|
||||
interval := s.CompatibleSeriesInterval(start, before, 24*time.Hour)
|
||||
points, err := s.Series(ctx, metric.AggregateQuery{
|
||||
Query: metric.Query{
|
||||
MetricName: metricName,
|
||||
EntityID: entityID,
|
||||
Start: start,
|
||||
End: end,
|
||||
Order: metric.OrderAsc,
|
||||
},
|
||||
Aggregation: metric.AggLast,
|
||||
Interval: interval,
|
||||
PreserveSeries: true,
|
||||
}, before)
|
||||
if err != nil {
|
||||
return 0, false, err
|
||||
}
|
||||
if len(points) == 0 {
|
||||
return 0, false, nil
|
||||
}
|
||||
return int64(points[len(points)-1].Value), true, nil
|
||||
}
|
||||
|
||||
// GetLatestTrafficBefore returns the latest retained upload/download counters
|
||||
// before a boundary, transparently reading raw points or rollups.
|
||||
func GetLatestTrafficBefore(ctx context.Context, entityIDs []string, before time.Time) (map[string]models.Record, error) {
|
||||
s := GetStore()
|
||||
if s == nil {
|
||||
return nil, fmt.Errorf("metric store not enabled")
|
||||
}
|
||||
result := make(map[string]models.Record, len(entityIDs))
|
||||
for _, entityID := range entityIDs {
|
||||
if entityID == "" {
|
||||
continue
|
||||
}
|
||||
up, hasUp, err := latestReportCounter(ctx, s, MetricNetTotalUp, entityID, before)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
down, hasDown, err := latestReportCounter(ctx, s, MetricNetTotalDown, entityID, before)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if !hasUp && !hasDown {
|
||||
continue
|
||||
}
|
||||
result[entityID] = models.Record{
|
||||
Client: entityID,
|
||||
Time: before.UTC().Add(-time.Nanosecond),
|
||||
NetTotalUp: up,
|
||||
NetTotalDown: down,
|
||||
}
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
// TrafficCounterDelta returns a reset-aware increase between two cumulative
|
||||
// traffic counters. After a reset, the current counter is the new increase.
|
||||
func TrafficCounterDelta(current, previous int64) int64 {
|
||||
if current < 0 || previous < 0 {
|
||||
return 0
|
||||
}
|
||||
if current >= previous {
|
||||
return current - previous
|
||||
}
|
||||
return current
|
||||
}
|
||||
|
||||
func deleteReportTrafficState(entityID string) {
|
||||
reportTrafficStates.Delete(entityID)
|
||||
}
|
||||
|
||||
func clearReportTrafficStates() {
|
||||
reportTrafficStates.Range(func(key, _ any) bool {
|
||||
reportTrafficStates.Delete(key)
|
||||
return true
|
||||
})
|
||||
}
|
||||
@@ -0,0 +1,277 @@
|
||||
package metricstore
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/komari-monitor/komari/pkg/metric"
|
||||
v1 "github.com/komari-monitor/komari/protocol/v1"
|
||||
)
|
||||
|
||||
func useReportTestStore(t *testing.T, policy *metric.RollupPolicy) *metric.Store {
|
||||
t.Helper()
|
||||
ctx := context.Background()
|
||||
opts := []metric.Option{metric.WithMaxOpenConns(1)}
|
||||
if policy != nil {
|
||||
opts = append(opts, metric.WithRollupPolicy(*policy))
|
||||
}
|
||||
dsn := fmt.Sprintf("file:report-%d?mode=memory&cache=shared", time.Now().UnixNano())
|
||||
s, err := metric.Open(ctx, metric.SQLite(dsn, opts...))
|
||||
if err != nil {
|
||||
t.Fatalf("open metric store: %v", err)
|
||||
}
|
||||
if err := createMetricDefinitions(ctx, s); err != nil {
|
||||
_ = s.Close()
|
||||
t.Fatalf("create metric definitions: %v", err)
|
||||
}
|
||||
|
||||
storeMu.Lock()
|
||||
previous := store
|
||||
store = s
|
||||
storeMu.Unlock()
|
||||
clearReportTrafficStates()
|
||||
t.Cleanup(func() {
|
||||
clearReportTrafficStates()
|
||||
storeMu.Lock()
|
||||
store = previous
|
||||
storeMu.Unlock()
|
||||
_ = s.Close()
|
||||
})
|
||||
return s
|
||||
}
|
||||
|
||||
func TestWriteReportStoresRawMetricsAndResetAwareTraffic(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
policy := defaultRollupPolicy()
|
||||
s := useReportTestStore(t, &policy)
|
||||
now := time.Now().UTC().Truncate(time.Minute)
|
||||
base := now.Add(-30 * time.Minute)
|
||||
|
||||
report := v1.Report{
|
||||
UUID: "node-a",
|
||||
UpdatedAt: base,
|
||||
CPU: v1.CPUReport{Usage: 12.5},
|
||||
Ram: v1.RamReport{Used: 100, Total: 1000},
|
||||
Swap: v1.RamReport{Used: 20, Total: 200},
|
||||
Load: v1.LoadReport{Load1: 0.5},
|
||||
Disk: v1.DiskReport{Used: 300, Total: 3000},
|
||||
Network: v1.NetworkReport{Up: 3, Down: 4, TotalUp: 100, TotalDown: 200},
|
||||
Process: 7,
|
||||
Connections: v1.ConnectionsReport{TCP: 8, UDP: 9},
|
||||
GPU: &v1.GPUDetailReport{
|
||||
AverageUsage: 25,
|
||||
DetailedInfo: []v1.GPUDeviceInfo{{
|
||||
Name: "GPU 0", MemoryUsed: 400, MemoryTotal: 800, Utilization: 30, Temperature: 55,
|
||||
}},
|
||||
},
|
||||
}
|
||||
if _, err := WriteReport(ctx, report); err != nil {
|
||||
t.Fatalf("write first report: %v", err)
|
||||
}
|
||||
|
||||
report.UpdatedAt = base.Add(3 * time.Second)
|
||||
report.Network.TotalUp = 150
|
||||
report.Network.TotalDown = 260
|
||||
if _, err := WriteReport(ctx, report); err != nil {
|
||||
t.Fatalf("write second report: %v", err)
|
||||
}
|
||||
|
||||
report.UpdatedAt = base.Add(6 * time.Second)
|
||||
report.Network.TotalUp = 20
|
||||
report.Network.TotalDown = 30
|
||||
if _, err := WriteReport(ctx, report); err != nil {
|
||||
t.Fatalf("write reset report: %v", err)
|
||||
}
|
||||
|
||||
assertMetricValues(t, s, MetricTrafficUp, report.UUID, base.Add(-time.Second), base.Add(time.Minute), []float64{0, 50, 20})
|
||||
assertMetricValues(t, s, MetricTrafficDown, report.UUID, base.Add(-time.Second), base.Add(time.Minute), []float64{0, 60, 30})
|
||||
assertMetricValues(t, s, MetricNetTotalUp, report.UUID, base.Add(-time.Second), base.Add(time.Minute), []float64{100, 150, 20})
|
||||
|
||||
gpuPoints, err := s.Query(ctx, metric.Query{
|
||||
MetricName: MetricGPUDeviceUsage,
|
||||
EntityID: report.UUID,
|
||||
Start: base.Add(-time.Second),
|
||||
End: base.Add(time.Minute),
|
||||
Tags: map[string]string{"device_index": "0"},
|
||||
Order: metric.OrderAsc,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("query GPU points: %v", err)
|
||||
}
|
||||
if len(gpuPoints) != 3 || gpuPoints[0].Timestamp != base || gpuPoints[0].Tags["device_name"] != "GPU 0" {
|
||||
t.Fatalf("unexpected GPU points: %#v", gpuPoints)
|
||||
}
|
||||
|
||||
if _, err := s.Compact(ctx, now); err != nil {
|
||||
t.Fatalf("compact reports: %v", err)
|
||||
}
|
||||
deleteReportTrafficState(report.UUID)
|
||||
report.UpdatedAt = now
|
||||
report.Network.TotalUp = 35
|
||||
report.Network.TotalDown = 50
|
||||
if _, err := WriteReport(ctx, report); err != nil {
|
||||
t.Fatalf("write after restoring rollup baseline: %v", err)
|
||||
}
|
||||
assertMetricValues(t, s, MetricTrafficUp, report.UUID, now.Add(-time.Second), now.Add(time.Second), []float64{15})
|
||||
assertMetricValues(t, s, MetricTrafficDown, report.UUID, now.Add(-time.Second), now.Add(time.Second), []float64{20})
|
||||
}
|
||||
|
||||
func TestReportBatcherFlushesQueuedReports(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
s := useReportTestStore(t, nil)
|
||||
StartReportBatcher()
|
||||
t.Cleanup(func() {
|
||||
if err := StopReportBatcher(ctx); err != nil {
|
||||
t.Errorf("stop report batcher: %v", err)
|
||||
}
|
||||
})
|
||||
|
||||
base := time.Now().UTC().Truncate(time.Second)
|
||||
first := v1.Report{
|
||||
UUID: "batched-node",
|
||||
UpdatedAt: base,
|
||||
CPU: v1.CPUReport{Usage: 10},
|
||||
Network: v1.NetworkReport{TotalUp: 100, TotalDown: 200},
|
||||
}
|
||||
second := first
|
||||
second.UpdatedAt = base.Add(3 * time.Second)
|
||||
second.CPU.Usage = 20
|
||||
second.Network.TotalUp = 150
|
||||
second.Network.TotalDown = 260
|
||||
|
||||
if _, err := WriteReport(ctx, first); err != nil {
|
||||
t.Fatalf("queue first report: %v", err)
|
||||
}
|
||||
if _, err := WriteReport(ctx, second); err != nil {
|
||||
t.Fatalf("queue second report: %v", err)
|
||||
}
|
||||
points, err := s.Query(ctx, metric.Query{
|
||||
MetricName: MetricCPU,
|
||||
EntityID: first.UUID,
|
||||
Start: base.Add(-time.Second),
|
||||
End: base.Add(time.Minute),
|
||||
Order: metric.OrderAsc,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("query before flush: %v", err)
|
||||
}
|
||||
if len(points) != 0 {
|
||||
t.Fatalf("queued reports were written before flush: %#v", points)
|
||||
}
|
||||
|
||||
if err := FlushReportBatch(ctx); err != nil {
|
||||
t.Fatalf("flush report batch: %v", err)
|
||||
}
|
||||
assertMetricValues(t, s, MetricCPU, first.UUID, base.Add(-time.Second), base.Add(time.Minute), []float64{10, 20})
|
||||
assertMetricValues(t, s, MetricTrafficUp, first.UUID, base.Add(-time.Second), base.Add(time.Minute), []float64{0, 50})
|
||||
assertMetricValues(t, s, MetricTrafficDown, first.UUID, base.Add(-time.Second), base.Add(time.Minute), []float64{0, 60})
|
||||
}
|
||||
|
||||
func TestRecordReconstructionUsesMetricSpecificAggregation(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
s := useReportTestStore(t, nil)
|
||||
base := time.Now().UTC().Truncate(time.Minute)
|
||||
entityID := "node-aggregation"
|
||||
points := []metric.Point{
|
||||
{MetricName: MetricCPU, EntityID: entityID, Timestamp: base.Add(time.Second), Value: 10},
|
||||
{MetricName: MetricCPU, EntityID: entityID, Timestamp: base.Add(2 * time.Second), Value: 30},
|
||||
{MetricName: MetricNetTotalUp, EntityID: entityID, Timestamp: base.Add(time.Second), Value: 100},
|
||||
{MetricName: MetricNetTotalUp, EntityID: entityID, Timestamp: base.Add(2 * time.Second), Value: 200},
|
||||
{MetricName: MetricTrafficUp, EntityID: entityID, Timestamp: base.Add(time.Second), Value: 10},
|
||||
{MetricName: MetricTrafficUp, EntityID: entityID, Timestamp: base.Add(2 * time.Second), Value: 20},
|
||||
}
|
||||
if err := s.WriteBatch(ctx, points); err != nil {
|
||||
t.Fatalf("write points: %v", err)
|
||||
}
|
||||
|
||||
records, err := GetRecordsByClientAndTime(ctx, entityID, base, base.Add(time.Hour))
|
||||
if err != nil {
|
||||
t.Fatalf("reconstruct records: %v", err)
|
||||
}
|
||||
if len(records) != 1 {
|
||||
t.Fatalf("records = %#v, want one bucket", records)
|
||||
}
|
||||
if records[0].Cpu != 20 || records[0].NetTotalUp != 200 || records[0].TrafficUp != 30 {
|
||||
t.Fatalf("unexpected aggregation result: %#v", records[0])
|
||||
}
|
||||
}
|
||||
|
||||
func TestTrafficCounterDelta(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
current int64
|
||||
previous int64
|
||||
want int64
|
||||
}{
|
||||
{name: "previous zero", current: 120, previous: 0, want: 120},
|
||||
{name: "monotonic counter", current: 250, previous: 200, want: 50},
|
||||
{name: "unchanged counter", current: 100, previous: 100, want: 0},
|
||||
{name: "counter reset", current: 15, previous: 250, want: 15},
|
||||
{name: "negative current", current: -1, previous: 100, want: 0},
|
||||
{name: "negative previous", current: 15, previous: -1, want: 0},
|
||||
}
|
||||
for _, test := range tests {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
if got := TrafficCounterDelta(test.current, test.previous); got != test.want {
|
||||
t.Fatalf("TrafficCounterDelta(%d, %d) = %d, want %d", test.current, test.previous, got, test.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestWriteReportNormalizesReceiveTimeToUTC(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
s := useReportTestStore(t, nil)
|
||||
local := time.FixedZone("UTC+8", 8*60*60)
|
||||
receiveTime := time.Date(2026, 7, 17, 9, 30, 0, 123456789, local)
|
||||
report := v1.Report{
|
||||
UUID: "utc-report",
|
||||
UpdatedAt: receiveTime,
|
||||
CPU: v1.CPUReport{Usage: 10},
|
||||
Network: v1.NetworkReport{TotalUp: 1, TotalDown: 2},
|
||||
}
|
||||
|
||||
saved, err := WriteReport(ctx, report)
|
||||
if err != nil {
|
||||
t.Fatalf("write report: %v", err)
|
||||
}
|
||||
if !saved.UpdatedAt.Equal(receiveTime) || saved.UpdatedAt.Location() != time.UTC {
|
||||
t.Fatalf("saved receive time = %s (%s), want UTC", saved.UpdatedAt, saved.UpdatedAt.Location())
|
||||
}
|
||||
points, err := s.Query(ctx, metric.Query{
|
||||
MetricName: MetricCPU,
|
||||
EntityID: report.UUID,
|
||||
Start: receiveTime.Add(-time.Nanosecond),
|
||||
End: receiveTime.Add(time.Nanosecond),
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("query stored point: %v", err)
|
||||
}
|
||||
if len(points) != 1 || points[0].Timestamp.Location() != time.UTC || points[0].Timestamp.Nanosecond() != 123456789 {
|
||||
t.Fatalf("stored points = %#v, want one UTC nanosecond point", points)
|
||||
}
|
||||
}
|
||||
|
||||
func assertMetricValues(t *testing.T, s *metric.Store, metricName, entityID string, start, end time.Time, want []float64) {
|
||||
t.Helper()
|
||||
points, err := s.Query(context.Background(), metric.Query{
|
||||
MetricName: metricName,
|
||||
EntityID: entityID,
|
||||
Start: start,
|
||||
End: end,
|
||||
Order: metric.OrderAsc,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("query %s: %v", metricName, err)
|
||||
}
|
||||
if len(points) != len(want) {
|
||||
t.Fatalf("%s point count = %d, want %d: %#v", metricName, len(points), len(want), points)
|
||||
}
|
||||
for i := range want {
|
||||
if points[i].Value != want[i] {
|
||||
t.Fatalf("%s point %d = %v, want %v", metricName, i, points[i].Value, want[i])
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,312 @@
|
||||
package metricstore
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"log"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/komari-monitor/komari/pkg/config"
|
||||
"github.com/komari-monitor/komari/pkg/metric"
|
||||
)
|
||||
|
||||
// store_migration.go
|
||||
//
|
||||
// 面向 WebUI/API 的“存储后端迁移”支撑:把一个 metrics 源库(默认本地 SQLite
|
||||
// ./data/metrics.db)中的全部指标数据搬运到当前正在运行的 metrics 目标库
|
||||
// (通常是用户新配置并热重载后的 MySQL/PostgreSQL)。
|
||||
//
|
||||
// 与启动迁移(RunStartupMigration)的区别:
|
||||
// - 启动迁移在进程启动时自动、同步执行(komari.db → 当前 metrics 目标,或上一个
|
||||
// metrics 目标 → 当前目标)。
|
||||
// - 此处的 store 迁移由管理员在 WebUI 中「切换数据库后」手动触发,异步执行并可
|
||||
// 查询进度 / 取消。数据以 upsert 写入,可安全重复执行。
|
||||
//
|
||||
// 典型流程:
|
||||
// 1. 管理员在设置中把 metric_db_dsn 改为 MySQL/PostgreSQL,保存 → 后端连接测试
|
||||
// 通过后热重载,当前 store 切到远端(此时远端为空)。
|
||||
// 2. 管理员调用 startMetricMigration(不带 source 参数),系统以上一个 metrics
|
||||
// 目标(默认 SQLite metrics.db)为源,把历史数据搬运到远端。
|
||||
// 3. 完成后登记目标指纹,下次启动不再重复搬运。
|
||||
|
||||
// StoreMigrationProgress 描述一次 store-to-store 迁移的实时进度。
|
||||
type StoreMigrationProgress struct {
|
||||
Status string `json:"status"` // idle, running, completed, failed, canceled
|
||||
SourceDriver string `json:"source_driver"` // 源库驱动
|
||||
SourceDSN string `json:"source_dsn"` // 源库 DSN(脱敏后)
|
||||
TargetDriver string `json:"target_driver"` // 目标库驱动
|
||||
TargetDSN string `json:"target_dsn"` // 目标库 DSN(脱敏后)
|
||||
TotalMetrics int `json:"total_metrics"` // 指标定义总数
|
||||
CurrentMetric string `json:"current_metric"` // 当前正在搬运的指标名
|
||||
MetricsDone int `json:"metrics_done"` // 已完成的指标数
|
||||
MigratedPoints int64 `json:"migrated_points"` // 已搬运的采样点数
|
||||
StartTime time.Time `json:"start_time"` // 开始时间
|
||||
EndTime time.Time `json:"end_time"` // 结束时间
|
||||
Error string `json:"error,omitempty"` // 错误信息
|
||||
}
|
||||
|
||||
var (
|
||||
storeMigMu sync.Mutex
|
||||
storeMigProgress StoreMigrationProgress
|
||||
storeMigCancel context.CancelFunc
|
||||
storeMigDone chan struct{}
|
||||
storeClosing bool
|
||||
)
|
||||
|
||||
// IsStoreMigrationRunning 报告是否有 store-to-store 迁移正在运行。
|
||||
func IsStoreMigrationRunning() bool {
|
||||
storeMigMu.Lock()
|
||||
defer storeMigMu.Unlock()
|
||||
return storeMigCancel != nil
|
||||
}
|
||||
|
||||
// GetStoreMigrationProgress 返回当前 store-to-store 迁移进度快照。
|
||||
func GetStoreMigrationProgress() StoreMigrationProgress {
|
||||
storeMigMu.Lock()
|
||||
defer storeMigMu.Unlock()
|
||||
return storeMigProgress
|
||||
}
|
||||
|
||||
// ResolveStoreMigrationSourceFingerprint 推断本次 store 迁移的源库指纹。
|
||||
//
|
||||
// 优先级:
|
||||
// 1. 显式传入的 driver+dsn(driver 可留空,交由 DSN 推断)。
|
||||
// 2. 已保存的上一个 metrics 目标指纹(MigrationTargetKey)。
|
||||
// 3. 默认 SQLite(./data/metrics.db)。
|
||||
func ResolveStoreMigrationSourceFingerprint(driver, dsn string) string {
|
||||
dsn = strings.TrimSpace(dsn)
|
||||
if dsn != "" {
|
||||
d := ResolveDriverFromConfig(driver, dsn)
|
||||
return fmt.Sprintf("%s|%s", d, dsn)
|
||||
}
|
||||
if saved, _ := config.GetAs[string](MigrationTargetKey, ""); saved != "" {
|
||||
return saved
|
||||
}
|
||||
return defaultSQLiteFingerprint()
|
||||
}
|
||||
|
||||
// StartStoreMigration 异步启动一次 store-to-store 迁移:把源库(由 sourceDriver +
|
||||
// sourceDSN 指定,留空则自动推断)的全部指标数据搬运到当前运行中的 metrics 目标库。
|
||||
//
|
||||
// 返回错误表示“未能启动”(例如已有迁移在跑、源与目标相同、目标未初始化等);
|
||||
// 迁移过程中的错误通过 GetStoreMigrationProgress().Status == "failed" 与 Error 暴露。
|
||||
func StartStoreMigration(sourceDriver, sourceDSN string) error {
|
||||
cfg, err := config.GetManyAs[MetricStoreConfig]()
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to load metric store config: %w", err)
|
||||
}
|
||||
targetFP := targetFingerprint(cfg)
|
||||
sourceFP := ResolveStoreMigrationSourceFingerprint(sourceDriver, sourceDSN)
|
||||
|
||||
if sourceFP == targetFP {
|
||||
return fmt.Errorf("source and target metrics database are the same; nothing to migrate")
|
||||
}
|
||||
|
||||
srcCfg, err := configFromFingerprint(sourceFP, cfg)
|
||||
if err != nil {
|
||||
return fmt.Errorf("resolve source metrics store: %w", err)
|
||||
}
|
||||
if !storeOperations.TryAcquire() {
|
||||
return ErrStoreBusy
|
||||
}
|
||||
exclusiveLeaseHandedOff := false
|
||||
defer func() {
|
||||
if !exclusiveLeaseHandedOff {
|
||||
storeOperations.Release()
|
||||
}
|
||||
}()
|
||||
|
||||
storeMu.RLock()
|
||||
dst := store
|
||||
activeFingerprint := storeFingerprint
|
||||
storeMu.RUnlock()
|
||||
if dst == nil {
|
||||
return ErrStoreNotInitialized
|
||||
}
|
||||
if activeFingerprint != targetFP {
|
||||
return fmt.Errorf("active metric store does not match the configured migration target; wait for hot reload and retry")
|
||||
}
|
||||
|
||||
storeMigMu.Lock()
|
||||
if storeClosing {
|
||||
storeMigMu.Unlock()
|
||||
return ErrStoreBusy
|
||||
}
|
||||
if storeMigCancel != nil {
|
||||
storeMigMu.Unlock()
|
||||
return fmt.Errorf("a store migration is already in progress")
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
done := make(chan struct{})
|
||||
storeMigCancel = cancel
|
||||
storeMigDone = done
|
||||
storeMigProgress = StoreMigrationProgress{
|
||||
Status: "running",
|
||||
SourceDriver: string(ResolveDriverFromConfig(srcCfg.Driver, srcCfg.DSN)),
|
||||
SourceDSN: maskDSN(srcCfg.DSN),
|
||||
TargetDriver: string(ResolveDriverFromConfig(cfg.Driver, cfg.DSN)),
|
||||
TargetDSN: maskDSN(cfg.DSN),
|
||||
StartTime: time.Now().UTC(),
|
||||
}
|
||||
storeMigMu.Unlock()
|
||||
|
||||
exclusiveLeaseHandedOff = true
|
||||
go runStoreMigration(ctx, cancel, done, srcCfg, cfg, dst, targetFP)
|
||||
return nil
|
||||
}
|
||||
|
||||
// runStoreMigration 执行实际的搬运逻辑(在独立 goroutine 中)。
|
||||
func runStoreMigration(ctx context.Context, cancel context.CancelFunc, done chan struct{}, srcCfg *MetricStoreConfig, cfg *MetricStoreConfig, dst *metric.Store, targetFP string) {
|
||||
defer func() {
|
||||
cancel()
|
||||
storeOperations.Release()
|
||||
|
||||
storeMigMu.Lock()
|
||||
storeMigCancel = nil
|
||||
storeMigDone = nil
|
||||
close(done)
|
||||
storeMigMu.Unlock()
|
||||
}()
|
||||
|
||||
src, err := openSourceStore(ctx, srcCfg)
|
||||
if err != nil {
|
||||
finishStoreMigration("failed", fmt.Errorf("open source metrics store: %w", err))
|
||||
return
|
||||
}
|
||||
defer src.Close()
|
||||
|
||||
observe := func(currentMetric string, metricIndex, totalMetrics int, addedPoints int64) {
|
||||
storeMigMu.Lock()
|
||||
storeMigProgress.CurrentMetric = currentMetric
|
||||
storeMigProgress.MetricsDone = metricIndex
|
||||
storeMigProgress.TotalMetrics = totalMetrics
|
||||
storeMigProgress.MigratedPoints += addedPoints
|
||||
storeMigMu.Unlock()
|
||||
}
|
||||
|
||||
total, err := migrateBetweenStores(ctx, src, dst, observe)
|
||||
if err != nil {
|
||||
if ctx.Err() != nil {
|
||||
finishStoreMigration("canceled", nil)
|
||||
log.Printf("[store-migration] canceled after %d points", total)
|
||||
return
|
||||
}
|
||||
finishStoreMigration("failed", err)
|
||||
return
|
||||
}
|
||||
|
||||
// 搬运成功:登记当前目标指纹,避免下次启动重复搬运。
|
||||
if err := config.Set(MigrationTargetKey, targetFP); err != nil {
|
||||
log.Printf("[store-migration] failed to persist migration target fingerprint: %v", err)
|
||||
}
|
||||
finishStoreMigration("completed", nil)
|
||||
log.Printf("[store-migration] completed via API (%d points) target_driver=%s", total, ResolveDriverFromConfig(cfg.Driver, cfg.DSN))
|
||||
}
|
||||
|
||||
// finishStoreMigration 统一收尾:设置终态状态、结束时间与错误信息。
|
||||
func finishStoreMigration(status string, err error) {
|
||||
storeMigMu.Lock()
|
||||
defer storeMigMu.Unlock()
|
||||
storeMigProgress.Status = status
|
||||
storeMigProgress.EndTime = time.Now().UTC()
|
||||
if err != nil {
|
||||
storeMigProgress.Error = err.Error()
|
||||
}
|
||||
// 完成/取消时把当前指标标记为已全部处理,便于前端进度条归位。
|
||||
if status == "completed" && storeMigProgress.TotalMetrics > 0 {
|
||||
storeMigProgress.MetricsDone = storeMigProgress.TotalMetrics
|
||||
}
|
||||
}
|
||||
|
||||
// CancelStoreMigration 请求取消正在运行的 store-to-store 迁移,并等待其退出。
|
||||
// 由于写入是幂等 upsert,取消后可安全重新发起,不会产生重复数据。
|
||||
func CancelStoreMigration() error {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||||
defer cancel()
|
||||
return stopStoreMigration(ctx, true)
|
||||
}
|
||||
|
||||
func stopStoreMigration(ctx context.Context, requireRunning bool) error {
|
||||
storeMigMu.Lock()
|
||||
cancel := storeMigCancel
|
||||
done := storeMigDone
|
||||
storeMigMu.Unlock()
|
||||
|
||||
if cancel == nil {
|
||||
if requireRunning {
|
||||
return fmt.Errorf("no store migration is currently running")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
cancel()
|
||||
|
||||
select {
|
||||
case <-done:
|
||||
return nil
|
||||
case <-ctx.Done():
|
||||
return fmt.Errorf("wait for store migration to cancel: %w", ctx.Err())
|
||||
}
|
||||
}
|
||||
|
||||
func stopStoreMigrationForClose(ctx context.Context) error {
|
||||
storeMigMu.Lock()
|
||||
storeClosing = true
|
||||
cancel := storeMigCancel
|
||||
done := storeMigDone
|
||||
storeMigMu.Unlock()
|
||||
|
||||
if cancel == nil {
|
||||
return nil
|
||||
}
|
||||
cancel()
|
||||
select {
|
||||
case <-done:
|
||||
return nil
|
||||
case <-ctx.Done():
|
||||
return fmt.Errorf("wait for store migration to cancel: %w", ctx.Err())
|
||||
}
|
||||
}
|
||||
|
||||
func clearStoreClosing() {
|
||||
storeMigMu.Lock()
|
||||
storeClosing = false
|
||||
storeMigMu.Unlock()
|
||||
}
|
||||
|
||||
func isStoreClosing() bool {
|
||||
storeMigMu.Lock()
|
||||
defer storeMigMu.Unlock()
|
||||
return storeClosing
|
||||
}
|
||||
|
||||
// maskDSN 对 DSN 做粗粒度脱敏,避免在 API 响应/日志中泄露密码。
|
||||
// 处理两类常见格式:URL(scheme://user:pass@host/...)与 key=value(password=...)。
|
||||
func maskDSN(dsn string) string {
|
||||
dsn = strings.TrimSpace(dsn)
|
||||
if dsn == "" {
|
||||
return ""
|
||||
}
|
||||
// key=value 形式:屏蔽 password=... 段。
|
||||
if strings.Contains(dsn, "password=") {
|
||||
parts := strings.Fields(dsn)
|
||||
for i, p := range parts {
|
||||
if strings.HasPrefix(strings.ToLower(p), "password=") {
|
||||
parts[i] = "password=***"
|
||||
}
|
||||
}
|
||||
return strings.Join(parts, " ")
|
||||
}
|
||||
// URL / user:pass@host 形式:屏蔽 user:pass@ 中的密码。
|
||||
if at := strings.LastIndex(dsn, "@"); at > 0 {
|
||||
head := dsn[:at]
|
||||
tail := dsn[at:]
|
||||
if colon := strings.LastIndex(head, ":"); colon >= 0 {
|
||||
// 保留 scheme://user,屏蔽密码。
|
||||
return head[:colon] + ":***" + tail
|
||||
}
|
||||
}
|
||||
return dsn
|
||||
}
|
||||
@@ -1,121 +0,0 @@
|
||||
package models
|
||||
|
||||
import (
|
||||
"database/sql/driver"
|
||||
"fmt"
|
||||
"log"
|
||||
"os"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
// LocalTime is a custom time type for GORM to handle time.Time correctly.
|
||||
// It ensures that all times are stored and retrieved based on the application's configured timezone.
|
||||
type LocalTime time.Time
|
||||
|
||||
// Value implements the driver.Valuer interface.
|
||||
// It converts UTCTime to a TEXT format that the database understands.
|
||||
// The time is formatted as a string in the application's local timezone, without timezone information.
|
||||
func (t LocalTime) Value() (driver.Value, error) {
|
||||
if time.Time(t).IsZero() {
|
||||
return nil, nil
|
||||
}
|
||||
return time.Time(t).In(GetAppLocation()).Format("2006-01-02 15:04:05.0000000"), nil
|
||||
}
|
||||
|
||||
// Scan implements the sql.Scanner interface.
|
||||
// It scans a value from the database into a UTCTime object.
|
||||
func (t *LocalTime) Scan(v interface{}) error {
|
||||
if v == nil {
|
||||
*t = LocalTime(time.Time{})
|
||||
return nil
|
||||
}
|
||||
|
||||
loc := GetAppLocation()
|
||||
|
||||
switch val := v.(type) {
|
||||
case time.Time:
|
||||
// CRITICAL FIX: When the driver reads a timezone-less string (e.g., "16:00:00"),
|
||||
// it often incorrectly assumes it's UTC. We must correct this by re-interpreting
|
||||
// the date and clock values in the application's actual timezone.
|
||||
year, month, day := val.Date()
|
||||
hour, min, sec := val.Clock()
|
||||
nanosec := val.Nanosecond()
|
||||
*t = LocalTime(time.Date(year, month, day, hour, min, sec, nanosec, loc))
|
||||
return nil
|
||||
case []byte:
|
||||
return t.parseTime(string(val), loc)
|
||||
case string:
|
||||
return t.parseTime(val, loc)
|
||||
default:
|
||||
return fmt.Errorf("UTCTime scan source was not string, []byte or time.Time: %T (%v)", v, v)
|
||||
}
|
||||
}
|
||||
|
||||
// parseTime handles parsing a string into UTCTime.
|
||||
func (t *LocalTime) parseTime(timeStr string, loc *time.Location) error {
|
||||
timeStr = strings.TrimSpace(timeStr)
|
||||
if timeStr == "" {
|
||||
*t = LocalTime(time.Time{})
|
||||
return nil
|
||||
}
|
||||
|
||||
layouts := []string{
|
||||
time.RFC3339Nano, time.RFC3339,
|
||||
"2006-01-02 15:04:05.0000000-07:00", "2006-01-02 15:04:05-07:00",
|
||||
"2006-01-02 15:04:05.0000000", "2006-01-02 15:04:05", "2006-01-02",
|
||||
}
|
||||
|
||||
for _, layout := range layouts {
|
||||
if parsedTime, err := time.ParseInLocation(layout, timeStr, loc); err == nil {
|
||||
*t = LocalTime(parsedTime)
|
||||
return nil
|
||||
}
|
||||
}
|
||||
return fmt.Errorf("unable to parse time string '%s' into UTCTime", timeStr)
|
||||
}
|
||||
|
||||
// MarshalJSON implements the json.Marshaler interface.
|
||||
// Serializes UTCTime to JSON in RFC3339 format with the correct timezone offset.
|
||||
func (t LocalTime) MarshalJSON() ([]byte, error) {
|
||||
if time.Time(t).IsZero() {
|
||||
return []byte("null"), nil
|
||||
}
|
||||
formattedTime := time.Time(t).In(GetAppLocation()).Format(time.RFC3339)
|
||||
return []byte(fmt.Sprintf(`"%s"`, formattedTime)), nil
|
||||
}
|
||||
|
||||
// ToTime converts UTCTime to Go's native time.Time type.
|
||||
func (t LocalTime) ToTime() time.Time { return time.Time(t) }
|
||||
|
||||
// FromTime converts Go's native time.Time type to UTCTime.
|
||||
func FromTime(t time.Time) LocalTime { return LocalTime(t) }
|
||||
|
||||
// Now returns the current time in the application's configured location.
|
||||
func Now() LocalTime { return LocalTime(time.Now().In(GetAppLocation())) }
|
||||
|
||||
var (
|
||||
appLocation *time.Location
|
||||
locationOnce sync.Once
|
||||
)
|
||||
|
||||
// GetAppLocation retrieves the application's global timezone from the "TZ" environment variable.
|
||||
func GetAppLocation() *time.Location {
|
||||
locationOnce.Do(func() {
|
||||
tz := os.Getenv("TZ")
|
||||
if tz == "" {
|
||||
tz = "UTC"
|
||||
}
|
||||
loc, err := time.LoadLocation(tz)
|
||||
if err != nil {
|
||||
log.Printf("Warning: Failed to load timezone '%s', falling back to UTC. Error: %v", tz, err)
|
||||
appLocation = time.UTC
|
||||
} else {
|
||||
appLocation = loc
|
||||
}
|
||||
time.Local = appLocation
|
||||
log.Printf("Application timezone is set to '%s'.", appLocation.String())
|
||||
})
|
||||
return appLocation
|
||||
}
|
||||
@@ -1,11 +1,13 @@
|
||||
package models
|
||||
|
||||
import "time"
|
||||
|
||||
type Clipboard struct {
|
||||
Id int `json:"id" gorm:"primaryKey;autoIncrement;unique"`
|
||||
Text string `json:"text" gorm:"type:longtext"`
|
||||
Name string `json:"name" gorm:"type:varchar(255)"`
|
||||
Weight int `json:"weight" gorm:"type:int"`
|
||||
Remark string `json:"remark" gorm:"type:text"`
|
||||
CreatedAt LocalTime `json:"created_at"`
|
||||
UpdatedAt LocalTime `json:"updated_at"`
|
||||
CreatedAt time.Time `json:"created_at"`
|
||||
UpdatedAt time.Time `json:"updated_at"`
|
||||
}
|
||||
@@ -1,10 +1,12 @@
|
||||
package models
|
||||
|
||||
import "time"
|
||||
|
||||
type Log struct {
|
||||
ID uint `json:"id,omitempty" gorm:"primaryKey;autoIncrement"`
|
||||
IP string `json:"ip" gorm:"type:varchar(45);"` // IPv4 or IPv6
|
||||
UUID string `json:"uuid" gorm:"type:varchar(36);"`
|
||||
Message string `json:"message" gorm:"type:text;not null"`
|
||||
MsgType string `json:"msg_type" gorm:"type:varchar(20);not null"`
|
||||
Time LocalTime `json:"time" gorm:"autoCreateTime;not null"`
|
||||
MsgType string `json:"msg_type" gorm:"type:varchar(20);not null;index:idx_logs_msg_type_time,priority:1"`
|
||||
Time time.Time `json:"time" gorm:"autoCreateTime;not null;index:idx_logs_time;index:idx_logs_msg_type_time,priority:2"`
|
||||
}
|
||||
+41
-40
@@ -4,43 +4,44 @@ import (
|
||||
"database/sql/driver"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Client represents a registered client device
|
||||
type Client struct {
|
||||
UUID string `json:"uuid,omitempty" gorm:"type:varchar(36);primaryKey"`
|
||||
Token string `json:"token,omitempty" gorm:"type:varchar(255);unique;not null"`
|
||||
Name string `json:"name" gorm:"type:varchar(100)"`
|
||||
CpuName string `json:"cpu_name" gorm:"type:varchar(100)"`
|
||||
Virtualization string `json:"virtualization" gorm:"type:varchar(50)"`
|
||||
Arch string `json:"arch" gorm:"type:varchar(50)"`
|
||||
CpuCores int `json:"cpu_cores" gorm:"type:int"`
|
||||
CpuPhysicalCores int `json:"cpu_physical_cores" gorm:"type:int"`
|
||||
OS string `json:"os" gorm:"type:varchar(100)"`
|
||||
KernelVersion string `json:"kernel_version" gorm:"type:varchar(100)"`
|
||||
GpuName string `json:"gpu_name" gorm:"type:varchar(100)"`
|
||||
IPv4 string `json:"ipv4,omitempty" gorm:"type:varchar(100)"`
|
||||
IPv6 string `json:"ipv6,omitempty" gorm:"type:varchar(100)"`
|
||||
Region string `json:"region" gorm:"type:varchar(100)"`
|
||||
Remark string `json:"remark,omitempty" gorm:"type:longtext"`
|
||||
PublicRemark string `json:"public_remark,omitempty" gorm:"type:longtext"`
|
||||
MemTotal int64 `json:"mem_total" gorm:"type:bigint"`
|
||||
SwapTotal int64 `json:"swap_total" gorm:"type:bigint"`
|
||||
DiskTotal int64 `json:"disk_total" gorm:"type:bigint"`
|
||||
Version string `json:"version,omitempty" gorm:"type:varchar(100)"`
|
||||
Weight int `json:"weight" gorm:"type:int"`
|
||||
Price float64 `json:"price"`
|
||||
BillingCycle int `json:"billing_cycle"`
|
||||
AutoRenewal bool `json:"auto_renewal" gorm:"default:false"` // 是否自动续费
|
||||
Currency string `json:"currency" gorm:"type:varchar(20);default:'$'"`
|
||||
ExpiredAt LocalTime `json:"expired_at" gorm:"type:timestamp"`
|
||||
Group string `json:"group" gorm:"type:varchar(100)"`
|
||||
Tags string `json:"tags" gorm:"type:text"` // split by ';'
|
||||
Hidden bool `json:"hidden" gorm:"default:false"`
|
||||
TrafficLimit int64 `json:"traffic_limit" gorm:"type:bigint"`
|
||||
TrafficLimitType string `json:"traffic_limit_type" gorm:"type:varchar(10);default:'max'"` // 流量阈值类型:sum max min up down
|
||||
CreatedAt LocalTime `json:"created_at"`
|
||||
UpdatedAt LocalTime `json:"updated_at"`
|
||||
UUID string `json:"uuid,omitempty" gorm:"type:varchar(36);primaryKey"`
|
||||
Token string `json:"token,omitempty" gorm:"type:varchar(255);unique;not null"`
|
||||
Name string `json:"name" gorm:"type:varchar(100)"`
|
||||
CpuName string `json:"cpu_name" gorm:"type:varchar(100)"`
|
||||
Virtualization string `json:"virtualization" gorm:"type:varchar(50)"`
|
||||
Arch string `json:"arch" gorm:"type:varchar(50)"`
|
||||
CpuCores int `json:"cpu_cores" gorm:"type:int"`
|
||||
CpuPhysicalCores int `json:"cpu_physical_cores" gorm:"type:int"`
|
||||
OS string `json:"os" gorm:"type:varchar(100)"`
|
||||
KernelVersion string `json:"kernel_version" gorm:"type:varchar(100)"`
|
||||
GpuName string `json:"gpu_name" gorm:"type:varchar(100)"`
|
||||
IPv4 string `json:"ipv4,omitempty" gorm:"type:varchar(100)"`
|
||||
IPv6 string `json:"ipv6,omitempty" gorm:"type:varchar(100)"`
|
||||
Region string `json:"region" gorm:"type:varchar(100)"`
|
||||
Remark string `json:"remark,omitempty" gorm:"type:longtext"`
|
||||
PublicRemark string `json:"public_remark,omitempty" gorm:"type:longtext"`
|
||||
MemTotal int64 `json:"mem_total" gorm:"type:bigint"`
|
||||
SwapTotal int64 `json:"swap_total" gorm:"type:bigint"`
|
||||
DiskTotal int64 `json:"disk_total" gorm:"type:bigint"`
|
||||
Version string `json:"version,omitempty" gorm:"type:varchar(100)"`
|
||||
Weight int `json:"weight" gorm:"type:int"`
|
||||
Price float64 `json:"price"`
|
||||
BillingCycle int `json:"billing_cycle"`
|
||||
AutoRenewal bool `json:"auto_renewal" gorm:"default:false"` // 是否自动续费
|
||||
Currency string `json:"currency" gorm:"type:varchar(20);default:'$'"`
|
||||
ExpiredAt *time.Time `json:"expired_at" gorm:"type:timestamp"`
|
||||
Group string `json:"group" gorm:"type:varchar(100)"`
|
||||
Tags string `json:"tags" gorm:"type:text"` // split by ';'
|
||||
Hidden bool `json:"hidden" gorm:"default:false"`
|
||||
TrafficLimit int64 `json:"traffic_limit" gorm:"type:bigint"`
|
||||
TrafficLimitType string `json:"traffic_limit_type" gorm:"type:varchar(10);default:'max'"` // 流量阈值类型:sum max min up down
|
||||
CreatedAt time.Time `json:"created_at"`
|
||||
UpdatedAt time.Time `json:"updated_at"`
|
||||
}
|
||||
|
||||
// User represents an authenticated user
|
||||
@@ -52,8 +53,8 @@ type User struct {
|
||||
SSOID string `json:"sso_id" gorm:"type:varchar(100)"` // OAuth provider's user ID
|
||||
TwoFactor string `json:"two_factor,omitempty" gorm:"type:varchar(255)"` // 2FA secret
|
||||
Sessions []Session `json:"sessions,omitempty" gorm:"foreignKey:UUID;references:UUID;constraint:OnDelete:CASCADE,OnUpdate:CASCADE"`
|
||||
CreatedAt LocalTime `json:"created_at"`
|
||||
UpdatedAt LocalTime `json:"updated_at"`
|
||||
CreatedAt time.Time `json:"created_at"`
|
||||
UpdatedAt time.Time `json:"updated_at"`
|
||||
}
|
||||
|
||||
// Session manages user sessions
|
||||
@@ -63,17 +64,17 @@ type Session struct {
|
||||
UserAgent string `json:"user_agent" gorm:"type:text"`
|
||||
Ip string `json:"ip" gorm:"type:varchar(100)"`
|
||||
LoginMethod string `json:"login_method" gorm:"type:varchar(50)"`
|
||||
LatestOnline LocalTime `json:"latest_online" gorm:"type:timestamp"`
|
||||
LatestOnline time.Time `json:"latest_online" gorm:"type:timestamp"`
|
||||
LatestUserAgent string `json:"latest_user_agent" gorm:"type:text"`
|
||||
LatestIp string `json:"latest_ip" gorm:"type:varchar(100)"`
|
||||
Expires LocalTime `json:"expires" gorm:"not null"`
|
||||
CreatedAt LocalTime `json:"created_at"`
|
||||
Expires time.Time `json:"expires" gorm:"not null"`
|
||||
CreatedAt time.Time `json:"created_at"`
|
||||
}
|
||||
|
||||
// Record logs client metrics over time
|
||||
type Record struct {
|
||||
Client string `json:"client" gorm:"type:varchar(36);index"`
|
||||
Time LocalTime `json:"time" gorm:"index"`
|
||||
Time time.Time `json:"time" gorm:"index"`
|
||||
Cpu float32 `json:"cpu" gorm:"type:decimal(5,2)"` // e.g., 75.50%
|
||||
Gpu float32 `json:"gpu" gorm:"type:decimal(5,2)"`
|
||||
Ram int64 `json:"ram" gorm:"type:bigint"`
|
||||
@@ -99,7 +100,7 @@ type Record struct {
|
||||
// GPURecord logs individual GPU metrics over time
|
||||
type GPURecord struct {
|
||||
Client string `json:"client" gorm:"type:varchar(36);index"` // 客户端UUID
|
||||
Time LocalTime `json:"time" gorm:"index"` // 记录时间
|
||||
Time time.Time `json:"time" gorm:"index"` // 记录时间
|
||||
DeviceIndex int `json:"device_index" gorm:"index"` // GPU设备索引 (0,1,2...)
|
||||
DeviceName string `json:"device_name" gorm:"type:varchar(100)"` // GPU型号
|
||||
MemTotal int64 `json:"mem_total" gorm:"type:bigint"` // 显存总量(字节)
|
||||
|
||||
@@ -1,13 +1,15 @@
|
||||
package models
|
||||
|
||||
import "time"
|
||||
|
||||
// Notification 定义了通知相关的数据库模型
|
||||
type OfflineNotification struct {
|
||||
Client string `json:"client" gorm:"type:varchar(36);not null;index;unique;constraint:OnDelete:CASCADE,OnUpdate:CASCADE;foreignKey:client;references:UUID"`
|
||||
ClientInfo Client `json:"client_info,omitempty" gorm:"foreignKey:Client;references:UUID"`
|
||||
Enable bool `json:"enable" gorm:"type:boolean;default:false"`
|
||||
//Cooldown int `json:"cooldown" gorm:"type:int;not null;default:1800"` // 冷却时间(秒),默认 30 分钟
|
||||
GracePeriod int `json:"grace_period" gorm:"type:int;not null;default:180"` // 宽限期(秒),默认 3 分钟
|
||||
LastNotified LocalTime `json:"last_notified"` // 上次通知时间
|
||||
GracePeriod int `json:"grace_period" gorm:"type:int;not null;default:180"` // 宽限期(秒),默认 3 分钟
|
||||
LastNotified *time.Time `json:"last_notified"` // 上次通知时间
|
||||
}
|
||||
|
||||
// LoadNotification 定义了基于资源占用达标时间比的负载通知规则
|
||||
@@ -19,7 +21,7 @@ type LoadNotification struct {
|
||||
Threshold float32 `json:"threshold" gorm:"type:decimal(5,2);not null;default:80.00"` // 阈值百分比
|
||||
Ratio float32 `json:"ratio" gorm:"type:decimal(5,2);not null;default:0.80"` // 达标时间比
|
||||
Interval int `json:"interval" gorm:"type:int;not null;default:15"` // 监测间隔(分钟)
|
||||
LastNotified LocalTime `json:"last_notified"` // 上次通知时间
|
||||
LastNotified *time.Time `json:"last_notified"` // 上次通知时间
|
||||
}
|
||||
|
||||
// TrafficReportNotification 定义了流量定时报告的数据库模型
|
||||
@@ -27,7 +29,7 @@ type TrafficReportNotification struct {
|
||||
Client string `json:"client" gorm:"type:varchar(36);not null;index;unique;constraint:OnDelete:CASCADE,OnUpdate:CASCADE;foreignKey:client;references:UUID"`
|
||||
ClientInfo Client `json:"client_info,omitempty" gorm:"foreignKey:Client;references:UUID"`
|
||||
Enable bool `json:"enable" gorm:"type:boolean;default:false"`
|
||||
Daily bool `json:"daily" gorm:"type:boolean;default:false"` // 日报
|
||||
Weekly bool `json:"weekly" gorm:"type:boolean;default:false"` // 周报
|
||||
Daily bool `json:"daily" gorm:"type:boolean;default:false"` // 日报
|
||||
Weekly bool `json:"weekly" gorm:"type:boolean;default:false"` // 周报
|
||||
Monthly bool `json:"monthly" gorm:"type:boolean;default:false"` // 月报
|
||||
}
|
||||
@@ -1,11 +1,13 @@
|
||||
package models
|
||||
|
||||
import "time"
|
||||
|
||||
type PingRecord struct {
|
||||
Client string `json:"client" gorm:"type:varchar(36);not null;index"`
|
||||
ClientInfo Client `json:"client_info" gorm:"foreignKey:Client;references:UUID;constraint:OnDelete:CASCADE,OnUpdate:CASCADE"`
|
||||
TaskId uint `json:"task_id" gorm:"not null;index"`
|
||||
Task PingTask `json:"task" gorm:"foreignKey:TaskId;references:Id;constraint:OnDelete:CASCADE,OnUpdate:CASCADE;"`
|
||||
Time LocalTime `json:"time" gorm:"index;not null"`
|
||||
Time time.Time `json:"time" gorm:"index;not null"`
|
||||
Value int `json:"value" gorm:"type:int;not null"` // Ping 值,单位毫秒
|
||||
}
|
||||
|
||||
|
||||
@@ -1,5 +1,7 @@
|
||||
package models
|
||||
|
||||
import "time"
|
||||
|
||||
type Task struct {
|
||||
TaskId string `json:"task_id" gorm:"type:varchar(36);primaryKey;unique"`
|
||||
Clients StringArray `json:"clients" gorm:"type:longtext"`
|
||||
@@ -13,6 +15,6 @@ type TaskResult struct {
|
||||
ClientInfo Client `json:"client_info" gorm:"foreignKey:Client;references:UUID;constraint:OnDelete:CASCADE,OnUpdate:CASCADE"`
|
||||
Result string `json:"result" gorm:"type:longtext"`
|
||||
ExitCode *int `json:"exit_code" gorm:"type:int"`
|
||||
FinishedAt *LocalTime `json:"finished_at" gorm:"type:timestamp"`
|
||||
CreatedAt LocalTime `json:"created_at" gorm:"type:timestamp"`
|
||||
FinishedAt *time.Time `json:"finished_at" gorm:"type:timestamp"`
|
||||
CreatedAt time.Time `json:"created_at" gorm:"type:timestamp"`
|
||||
}
|
||||
@@ -0,0 +1,77 @@
|
||||
package models
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"gorm.io/driver/sqlite"
|
||||
"gorm.io/gorm"
|
||||
"gorm.io/gorm/logger"
|
||||
)
|
||||
|
||||
func TestTimeFieldsStoreAndSerializeAsUTCWithNanoseconds(t *testing.T) {
|
||||
dsn := fmt.Sprintf("file:%s?mode=memory&cache=shared", strings.ReplaceAll(t.Name(), "/", "_"))
|
||||
db, err := gorm.Open(sqlite.Open(dsn), &gorm.Config{
|
||||
Logger: logger.Default.LogMode(logger.Silent),
|
||||
NowFunc: func() time.Time { return time.Now().UTC() },
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("open database: %v", err)
|
||||
}
|
||||
if err := db.AutoMigrate(&Log{}); err != nil {
|
||||
t.Fatalf("migrate log: %v", err)
|
||||
}
|
||||
|
||||
stamp := time.Date(2026, 7, 17, 1, 2, 3, 123456789, time.UTC)
|
||||
entry := Log{IP: "127.0.0.1", UUID: "test", Message: "test", MsgType: "info", Time: stamp}
|
||||
if err := db.Create(&entry).Error; err != nil {
|
||||
t.Fatalf("create log: %v", err)
|
||||
}
|
||||
var loaded Log
|
||||
if err := db.First(&loaded, entry.ID).Error; err != nil {
|
||||
t.Fatalf("load log: %v", err)
|
||||
}
|
||||
if !loaded.Time.Equal(stamp) || loaded.Time.Nanosecond() != stamp.Nanosecond() {
|
||||
t.Fatalf("loaded time = %s, want %s", loaded.Time, stamp)
|
||||
}
|
||||
|
||||
encoded, err := json.Marshal(loaded)
|
||||
if err != nil {
|
||||
t.Fatalf("marshal log: %v", err)
|
||||
}
|
||||
if !strings.Contains(string(encoded), `"time":"2026-07-17T01:02:03.123456789Z"`) {
|
||||
t.Fatalf("unexpected JSON: %s", encoded)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNullableTimeFieldReadsAndSerializesNull(t *testing.T) {
|
||||
dsn := fmt.Sprintf("file:%s?mode=memory&cache=shared", strings.ReplaceAll(t.Name(), "/", "_"))
|
||||
db, err := gorm.Open(sqlite.Open(dsn), &gorm.Config{Logger: logger.Default.LogMode(logger.Silent)})
|
||||
if err != nil {
|
||||
t.Fatalf("open database: %v", err)
|
||||
}
|
||||
if err := db.AutoMigrate(&Client{}); err != nil {
|
||||
t.Fatalf("migrate client: %v", err)
|
||||
}
|
||||
if err := db.Exec(`INSERT INTO clients (uuid, token, expired_at) VALUES (?, ?, NULL)`, "nullable", "token").Error; err != nil {
|
||||
t.Fatalf("insert client: %v", err)
|
||||
}
|
||||
|
||||
var client Client
|
||||
if err := db.First(&client, "uuid = ?", "nullable").Error; err != nil {
|
||||
t.Fatalf("load client: %v", err)
|
||||
}
|
||||
if client.ExpiredAt != nil {
|
||||
t.Fatalf("expired_at = %s, want nil", client.ExpiredAt)
|
||||
}
|
||||
encoded, err := json.Marshal(client)
|
||||
if err != nil {
|
||||
t.Fatalf("marshal client: %v", err)
|
||||
}
|
||||
if !strings.Contains(string(encoded), `"expired_at":null`) {
|
||||
t.Fatalf("unexpected JSON: %s", encoded)
|
||||
}
|
||||
}
|
||||
@@ -23,6 +23,11 @@ func GetOidcConfigByName(name string) (*models.OidcProvider, error) {
|
||||
return &config, nil
|
||||
}
|
||||
|
||||
func DeleteOidcConfigByName(name string) error {
|
||||
db := dbcore.GetDBInstance()
|
||||
return db.Delete(&models.OidcProvider{}, "name = ?", name).Error
|
||||
}
|
||||
|
||||
func SaveOidcConfig(config *models.OidcProvider) error {
|
||||
db := dbcore.GetDBInstance()
|
||||
if err := db.Save(config).Error; err != nil {
|
||||
|
||||
+15
-505
@@ -1,524 +1,34 @@
|
||||
package records
|
||||
|
||||
import (
|
||||
"log"
|
||||
"sort"
|
||||
"strings"
|
||||
"context"
|
||||
"time"
|
||||
|
||||
"gorm.io/gorm"
|
||||
|
||||
"github.com/komari-monitor/komari/cmd/flags"
|
||||
"github.com/komari-monitor/komari/database/dbcore"
|
||||
"github.com/komari-monitor/komari/database/metricstore"
|
||||
"github.com/komari-monitor/komari/database/models"
|
||||
)
|
||||
|
||||
func RecordOne(rec models.Record) error {
|
||||
db := dbcore.GetDBInstance()
|
||||
return db.Create(&rec).Error
|
||||
}
|
||||
|
||||
func RecordGPU(rec models.GPURecord) error {
|
||||
db := dbcore.GetDBInstance()
|
||||
return db.Create(&rec).Error
|
||||
}
|
||||
// 历史监控数据已完全迁移到 metric store(默认 SQLite ./data/metrics.db,或配置的
|
||||
// MySQL/PostgreSQL)。旧的 records / records_long_term / gpu_records / ping_records
|
||||
// 表仅在启动一次性迁移时作为数据源读取,运行期的读写全部走 metric store。
|
||||
//
|
||||
// 因此本文件不再保留对旧表的任何读写、压缩(rollup)与流量增量修复逻辑:
|
||||
// metric store 自行管理 rollup 与保留策略,旧表压缩已无意义。
|
||||
|
||||
func DeleteAll() error {
|
||||
db := dbcore.GetDBInstance()
|
||||
if err := db.Exec("DELETE FROM records_long_term").Error; err != nil {
|
||||
return err
|
||||
}
|
||||
if err := db.Exec("DELETE FROM gpu_records_long_term").Error; err != nil {
|
||||
return err
|
||||
}
|
||||
if err := db.Exec("DELETE FROM gpu_records").Error; err != nil {
|
||||
return err
|
||||
}
|
||||
return db.Exec("DELETE FROM records").Error
|
||||
return metricstore.DeleteAllRecords(context.Background())
|
||||
}
|
||||
|
||||
// GetGPURecordsByClientAndTime 获取GPU记录数据
|
||||
// GetGPURecordsByClientAndTime 获取 GPU 记录数据。
|
||||
func GetGPURecordsByClientAndTime(uuid string, start, end time.Time) ([]models.GPURecord, error) {
|
||||
db := dbcore.GetDBInstance()
|
||||
var records []models.GPURecord
|
||||
|
||||
fourHoursAgo := time.Now().Add(-4*time.Hour - time.Minute)
|
||||
|
||||
var recentRecords []models.GPURecord
|
||||
recentStart := start
|
||||
if end.After(fourHoursAgo) {
|
||||
if recentStart.Before(fourHoursAgo) {
|
||||
recentStart = fourHoursAgo
|
||||
}
|
||||
err := db.Where("client = ? AND time >= ? AND time <= ?", uuid, recentStart, end).
|
||||
Order("time ASC, device_index ASC").Find(&recentRecords).Error
|
||||
if err != nil {
|
||||
log.Printf("Error fetching recent GPU records for client %s between %s and %s: %v", uuid, recentStart, end, err)
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
|
||||
var longTermRecords []models.GPURecord
|
||||
err := db.Table("gpu_records_long_term").Where("client = ? AND time >= ? AND time <= ?", uuid, start, end).
|
||||
Order("time ASC, device_index ASC").Find(&longTermRecords).Error
|
||||
if err != nil {
|
||||
log.Printf("Error fetching long-term GPU records for client %s between %s and %s: %v", uuid, start, end, err)
|
||||
return recentRecords, nil
|
||||
}
|
||||
|
||||
// 合并结果 - 不再需要类型转换
|
||||
records = append(records, recentRecords...)
|
||||
records = append(records, longTermRecords...)
|
||||
|
||||
return records, nil
|
||||
}
|
||||
|
||||
func GetLatestRecord(uuid string) (Record []models.Record, err error) {
|
||||
db := dbcore.GetDBInstance()
|
||||
err = db.Where("client = ?", uuid).Order("time DESC").Limit(1).Find(&Record).Error
|
||||
return
|
||||
}
|
||||
|
||||
func DeleteRecordBefore(before time.Time) error {
|
||||
db := dbcore.GetDBInstance()
|
||||
db.Table("records_long_term").Where("time < ?", before).Delete(&models.Record{})
|
||||
db.Table("gpu_records_long_term").Where("time < ?", before).Delete(&models.GPURecord{})
|
||||
db.Where("time < ?", before).Delete(&models.GPURecord{})
|
||||
return db.Where("time < ?", before).Delete(&models.Record{}).Error
|
||||
return metricstore.GetGPURecordsByClientAndTime(context.Background(), uuid, start, end)
|
||||
}
|
||||
|
||||
func GetRecordsByClientAndTime(uuid string, start, end time.Time) ([]models.Record, error) {
|
||||
db := dbcore.GetDBInstance()
|
||||
var records []models.Record
|
||||
|
||||
fourHoursAgo := time.Now().Add(-4*time.Hour - time.Minute)
|
||||
|
||||
var recentRecords []models.Record
|
||||
recentStart := start
|
||||
if end.After(fourHoursAgo) {
|
||||
if recentStart.Before(fourHoursAgo) {
|
||||
recentStart = fourHoursAgo
|
||||
}
|
||||
err := db.Where("client = ? AND time >= ? AND time <= ?", uuid, recentStart, end).Order("time ASC").Find(&recentRecords).Error
|
||||
if err != nil {
|
||||
log.Printf("Error fetching recent records for client %s between %s and %s: %v", uuid, recentStart, end, err)
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
|
||||
var long_term []models.Record
|
||||
err := db.Table("records_long_term").Where("client = ? AND time >= ? AND time <= ?", uuid, start, end).Order("time ASC").Find(&long_term).Error
|
||||
if err != nil {
|
||||
log.Printf("Error fetching long-term records for client %s between %s and %s: %v", uuid, start, end, err)
|
||||
return recentRecords, nil
|
||||
}
|
||||
|
||||
if len(long_term) == 0 {
|
||||
// 没有查到long_term,返回全部recentRecords
|
||||
records = append(records, recentRecords...)
|
||||
return records, nil
|
||||
}
|
||||
|
||||
// 查到了long_term,recentRecords按15分钟分组,每组只保留一条(取最新一条)
|
||||
grouped := make(map[string]models.Record)
|
||||
for _, rec := range recentRecords {
|
||||
key := rec.Time.ToTime().Truncate(15 * time.Minute).Format(time.RFC3339)
|
||||
if old, ok := grouped[key]; !ok || rec.Time.ToTime().After(old.Time.ToTime()) {
|
||||
grouped[key] = rec
|
||||
}
|
||||
}
|
||||
var groupedList []models.Record
|
||||
for _, rec := range grouped {
|
||||
groupedList = append(groupedList, rec)
|
||||
}
|
||||
sort.Slice(groupedList, func(i, j int) bool {
|
||||
return groupedList[i].Time.ToTime().Before(groupedList[j].Time.ToTime())
|
||||
})
|
||||
records = append(records, groupedList...)
|
||||
records = append(records, long_term...)
|
||||
return records, nil
|
||||
return metricstore.GetRecordsByClientAndTime(context.Background(), uuid, start, end)
|
||||
}
|
||||
|
||||
func GetAllRecords() ([]models.Record, error) {
|
||||
db := dbcore.GetDBInstance()
|
||||
var records []models.Record
|
||||
var long_term []models.Record
|
||||
err := db.Table("records").Order("time ASC").Find(&records).Error
|
||||
if err != nil {
|
||||
log.Printf("Error fetching all records: %v", err)
|
||||
return nil, err
|
||||
}
|
||||
err = db.Table("records_long_term").Order("time ASC").Find(&long_term).Error
|
||||
if err != nil {
|
||||
log.Printf("Error fetching long-term records: %v", err)
|
||||
return records, nil
|
||||
}
|
||||
records = append(records, long_term...)
|
||||
return records, nil
|
||||
}
|
||||
|
||||
// 压缩数据库
|
||||
func CompactRecord() error {
|
||||
db := dbcore.GetDBInstance()
|
||||
err := migrateOldRecords(db)
|
||||
if err != nil {
|
||||
log.Printf("Error migrating old records: %v", err)
|
||||
return err
|
||||
}
|
||||
|
||||
err = migrateGPURecords(db)
|
||||
if err != nil {
|
||||
log.Printf("Error migrating GPU records: %v", err)
|
||||
return err
|
||||
}
|
||||
|
||||
if flags.IsSQLite() {
|
||||
db.Exec("PRAGMA wal_checkpoint(PASSIVE);")
|
||||
}
|
||||
//log.Printf("Record compaction completed")
|
||||
return nil
|
||||
}
|
||||
|
||||
func migrateOldRecords(db *gorm.DB) error {
|
||||
return migrateOldRecordsAt(db, time.Now())
|
||||
}
|
||||
|
||||
func compactRecordCutoff(now time.Time) time.Time {
|
||||
return now.Add(-4 * time.Hour).Truncate(15 * time.Minute)
|
||||
}
|
||||
|
||||
func migrateOldRecordsAt(db *gorm.DB, now time.Time) error {
|
||||
cutoff := compactRecordCutoff(now)
|
||||
|
||||
// 查询 records 表中超过 4 小时的记录
|
||||
var records []models.Record
|
||||
if err := db.Table("records").Where("time < ?", cutoff).Find(&records).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if len(records) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
// 按 Client 和 15 分钟时间段分组,并存储所有记录以计算分位数
|
||||
type groupData struct {
|
||||
Cpu []float32
|
||||
Gpu []float32
|
||||
Load []float32
|
||||
Temp []float32
|
||||
Ram []int64
|
||||
RamTotal []int64
|
||||
Swap []int64
|
||||
SwapTotal []int64
|
||||
Disk []int64
|
||||
DiskTotal []int64
|
||||
NetIn []int64
|
||||
NetOut []int64
|
||||
NetTotalUp []int64
|
||||
NetTotalDown []int64
|
||||
TrafficUp int64
|
||||
TrafficDown int64
|
||||
LatestTime time.Time
|
||||
LatestTotalUp int64
|
||||
LatestTotalDown int64
|
||||
Process []int
|
||||
Connections []int
|
||||
ConnectionsUdp []int
|
||||
Uptime []int64
|
||||
}
|
||||
|
||||
groupedRecords := make(map[string]*groupData)
|
||||
for _, record := range records {
|
||||
key := record.Client + "_" + record.Time.ToTime().Truncate(15*time.Minute).Format(time.RFC3339)
|
||||
if _, ok := groupedRecords[key]; !ok {
|
||||
groupedRecords[key] = &groupData{}
|
||||
}
|
||||
data := groupedRecords[key]
|
||||
data.Cpu = append(data.Cpu, record.Cpu)
|
||||
data.Gpu = append(data.Gpu, record.Gpu)
|
||||
data.Load = append(data.Load, record.Load)
|
||||
data.Temp = append(data.Temp, record.Temp)
|
||||
data.Ram = append(data.Ram, record.Ram)
|
||||
data.RamTotal = append(data.RamTotal, record.RamTotal)
|
||||
data.Swap = append(data.Swap, record.Swap)
|
||||
data.SwapTotal = append(data.SwapTotal, record.SwapTotal)
|
||||
data.Disk = append(data.Disk, record.Disk)
|
||||
data.DiskTotal = append(data.DiskTotal, record.DiskTotal)
|
||||
data.NetIn = append(data.NetIn, record.NetIn)
|
||||
data.NetOut = append(data.NetOut, record.NetOut)
|
||||
data.NetTotalUp = append(data.NetTotalUp, record.NetTotalUp)
|
||||
data.NetTotalDown = append(data.NetTotalDown, record.NetTotalDown)
|
||||
data.TrafficUp += record.TrafficUp
|
||||
data.TrafficDown += record.TrafficDown
|
||||
if data.LatestTime.IsZero() || record.Time.ToTime().After(data.LatestTime) {
|
||||
data.LatestTime = record.Time.ToTime()
|
||||
data.LatestTotalUp = record.NetTotalUp
|
||||
data.LatestTotalDown = record.NetTotalDown
|
||||
}
|
||||
data.Process = append(data.Process, record.Process)
|
||||
data.Connections = append(data.Connections, record.Connections)
|
||||
data.ConnectionsUdp = append(data.ConnectionsUdp, record.ConnectionsUdp)
|
||||
//data.Uptime = append(data.Uptime, record.Uptime)
|
||||
}
|
||||
|
||||
getPercentile := func(values []float64, percentile float64) float64 {
|
||||
if len(values) == 0 {
|
||||
return 0
|
||||
}
|
||||
sortedValues := make([]float64, len(values))
|
||||
copy(sortedValues, values)
|
||||
sort.Float64s(sortedValues)
|
||||
index := float64(len(sortedValues)-1) * percentile
|
||||
lowerIndex := int(index)
|
||||
if lowerIndex >= len(sortedValues)-1 {
|
||||
return sortedValues[len(sortedValues)-1]
|
||||
}
|
||||
frac := index - float64(lowerIndex)
|
||||
return sortedValues[lowerIndex] + frac*(sortedValues[lowerIndex+1]-sortedValues[lowerIndex])
|
||||
}
|
||||
|
||||
getIntPercentile := func(values []int64, percentile float64) int64 {
|
||||
if len(values) == 0 {
|
||||
return 0
|
||||
}
|
||||
floats := make([]float64, len(values))
|
||||
for i, v := range values {
|
||||
floats[i] = float64(v)
|
||||
}
|
||||
return int64(getPercentile(floats, percentile))
|
||||
}
|
||||
|
||||
getInt32Percentile := func(values []int, percentile float64) int {
|
||||
if len(values) == 0 {
|
||||
return 0
|
||||
}
|
||||
floats := make([]float64, len(values))
|
||||
for i, v := range values {
|
||||
floats[i] = float64(v)
|
||||
}
|
||||
return int(getPercentile(floats, percentile))
|
||||
}
|
||||
|
||||
return db.Transaction(func(tx *gorm.DB) error {
|
||||
for key, data := range groupedRecords {
|
||||
// 解析 Client 和时间
|
||||
parts := strings.Split(key, "_")
|
||||
clientUUID := parts[0]
|
||||
timeSlot, err := time.Parse(time.RFC3339, strings.Join(parts[1:], "_"))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
cpuFloats := make([]float64, len(data.Cpu))
|
||||
for i, v := range data.Cpu {
|
||||
cpuFloats[i] = float64(v)
|
||||
}
|
||||
gpuFloats := make([]float64, len(data.Gpu))
|
||||
for i, v := range data.Gpu {
|
||||
gpuFloats[i] = float64(v)
|
||||
}
|
||||
loadFloats := make([]float64, len(data.Load))
|
||||
for i, v := range data.Load {
|
||||
loadFloats[i] = float64(v)
|
||||
}
|
||||
tempFloats := make([]float64, len(data.Temp))
|
||||
for i, v := range data.Temp {
|
||||
tempFloats[i] = float64(v)
|
||||
}
|
||||
// 取高位
|
||||
high_percentile := 0.7
|
||||
// 检查 records_long_term 表中是否已存在相同的记录
|
||||
var existingCount int64
|
||||
if err := tx.Table("records_long_term").Where("client = ? AND time = ?", clientUUID, timeSlot).Count(&existingCount).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
newRec := models.Record{
|
||||
Client: clientUUID,
|
||||
Time: models.FromTime(timeSlot),
|
||||
Cpu: float32(getPercentile(cpuFloats, high_percentile)),
|
||||
Gpu: float32(getPercentile(gpuFloats, high_percentile)),
|
||||
Load: float32(getPercentile(loadFloats, high_percentile)),
|
||||
Temp: float32(getPercentile(tempFloats, high_percentile)),
|
||||
Ram: getIntPercentile(data.Ram, high_percentile),
|
||||
RamTotal: getIntPercentile(data.RamTotal, high_percentile),
|
||||
Swap: getIntPercentile(data.Swap, high_percentile),
|
||||
SwapTotal: getIntPercentile(data.SwapTotal, high_percentile),
|
||||
Disk: getIntPercentile(data.Disk, high_percentile),
|
||||
DiskTotal: getIntPercentile(data.DiskTotal, high_percentile),
|
||||
NetIn: getIntPercentile(data.NetIn, 0.2),
|
||||
NetOut: getIntPercentile(data.NetOut, 0.2),
|
||||
NetTotalUp: data.LatestTotalUp,
|
||||
NetTotalDown: data.LatestTotalDown,
|
||||
TrafficUp: data.TrafficUp,
|
||||
TrafficDown: data.TrafficDown,
|
||||
Process: getInt32Percentile(data.Process, high_percentile),
|
||||
Connections: getInt32Percentile(data.Connections, high_percentile),
|
||||
ConnectionsUdp: getInt32Percentile(data.ConnectionsUdp, high_percentile),
|
||||
//Uptime: getIntPercentile(data.Uptime, high_percentile),
|
||||
}
|
||||
|
||||
// 如果记录已存在则更新,否则创建新记录
|
||||
if existingCount > 0 {
|
||||
if err := tx.Table("records_long_term").Where("client = ? AND time = ?", clientUUID, timeSlot).Updates(&newRec).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
} else {
|
||||
if err := tx.Table("records_long_term").Create(&newRec).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 删除 records 表中的旧数据
|
||||
if err := tx.Table("records").Where("time < ?", cutoff.Add(-1*time.Hour)).Delete(&models.Record{}).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
})
|
||||
}
|
||||
|
||||
// migrateGPURecords 压缩GPU记录数据
|
||||
func migrateGPURecords(db *gorm.DB) error {
|
||||
cutoff := compactRecordCutoff(time.Now())
|
||||
|
||||
// 查询超过4小时的GPU记录
|
||||
var gpuRecords []models.GPURecord
|
||||
if err := db.Where("time < ?", cutoff).Find(&gpuRecords).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if len(gpuRecords) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
// 按Client + DeviceIndex + 15分钟时间窗口分组
|
||||
type gpuGroupKey struct {
|
||||
Client string
|
||||
DeviceIndex int
|
||||
TimeSlot time.Time
|
||||
DeviceName string
|
||||
}
|
||||
|
||||
type gpuGroupData struct {
|
||||
MemTotal []int64
|
||||
MemUsed []int64
|
||||
Utilization []float32
|
||||
Temperature []int
|
||||
}
|
||||
|
||||
groupedGPUs := make(map[gpuGroupKey]*gpuGroupData)
|
||||
|
||||
for _, record := range gpuRecords {
|
||||
key := gpuGroupKey{
|
||||
Client: record.Client,
|
||||
DeviceIndex: record.DeviceIndex,
|
||||
TimeSlot: record.Time.ToTime().Truncate(15 * time.Minute),
|
||||
DeviceName: record.DeviceName,
|
||||
}
|
||||
|
||||
if _, ok := groupedGPUs[key]; !ok {
|
||||
groupedGPUs[key] = &gpuGroupData{}
|
||||
}
|
||||
|
||||
data := groupedGPUs[key]
|
||||
data.MemTotal = append(data.MemTotal, record.MemTotal)
|
||||
data.MemUsed = append(data.MemUsed, record.MemUsed)
|
||||
data.Utilization = append(data.Utilization, record.Utilization)
|
||||
data.Temperature = append(data.Temperature, record.Temperature)
|
||||
}
|
||||
|
||||
// 百分位数计算函数 (复用传统Record压缩逻辑)
|
||||
getPercentile := func(values []float64, percentile float64) float64 {
|
||||
if len(values) == 0 {
|
||||
return 0
|
||||
}
|
||||
sortedValues := make([]float64, len(values))
|
||||
copy(sortedValues, values)
|
||||
sort.Float64s(sortedValues)
|
||||
index := float64(len(sortedValues)-1) * percentile
|
||||
lowerIndex := int(index)
|
||||
if lowerIndex >= len(sortedValues)-1 {
|
||||
return sortedValues[len(sortedValues)-1]
|
||||
}
|
||||
frac := index - float64(lowerIndex)
|
||||
return sortedValues[lowerIndex] + frac*(sortedValues[lowerIndex+1]-sortedValues[lowerIndex])
|
||||
}
|
||||
|
||||
getIntPercentile := func(values []int64, percentile float64) int64 {
|
||||
if len(values) == 0 {
|
||||
return 0
|
||||
}
|
||||
floats := make([]float64, len(values))
|
||||
for i, v := range values {
|
||||
floats[i] = float64(v)
|
||||
}
|
||||
return int64(getPercentile(floats, percentile))
|
||||
}
|
||||
|
||||
// 温度数据转换辅助函数
|
||||
convertIntToInt64 := func(values []int) []int64 {
|
||||
result := make([]int64, len(values))
|
||||
for i, v := range values {
|
||||
result[i] = int64(v)
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
getFloat32Percentile := func(values []float32, percentile float64) float32 {
|
||||
if len(values) == 0 {
|
||||
return 0
|
||||
}
|
||||
floats := make([]float64, len(values))
|
||||
for i, v := range values {
|
||||
floats[i] = float64(v)
|
||||
}
|
||||
return float32(getPercentile(floats, percentile))
|
||||
}
|
||||
|
||||
// 保持与传统Record压缩的一致性
|
||||
high_percentile := 0.7
|
||||
|
||||
return db.Transaction(func(tx *gorm.DB) error {
|
||||
for key, data := range groupedGPUs {
|
||||
// 检查是否已存在记录
|
||||
var existingCount int64
|
||||
if err := tx.Table("gpu_records_long_term").Where("client = ? AND device_index = ? AND time = ?",
|
||||
key.Client, key.DeviceIndex, key.TimeSlot).Count(&existingCount).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
compressedGPU := models.GPURecord{
|
||||
Client: key.Client,
|
||||
DeviceIndex: key.DeviceIndex,
|
||||
Time: models.FromTime(key.TimeSlot),
|
||||
DeviceName: key.DeviceName,
|
||||
MemTotal: getIntPercentile(data.MemTotal, high_percentile),
|
||||
MemUsed: getIntPercentile(data.MemUsed, high_percentile),
|
||||
Utilization: getFloat32Percentile(data.Utilization, high_percentile),
|
||||
Temperature: int(getIntPercentile(convertIntToInt64(data.Temperature), high_percentile)),
|
||||
}
|
||||
|
||||
if existingCount > 0 {
|
||||
// 更新已存在记录
|
||||
if err := tx.Table("gpu_records_long_term").Where("client = ? AND device_index = ? AND time = ?",
|
||||
key.Client, key.DeviceIndex, key.TimeSlot).Updates(&compressedGPU).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
} else {
|
||||
// 创建新记录
|
||||
if err := tx.Table("gpu_records_long_term").Create(&compressedGPU).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 删除已压缩的原始GPU数据
|
||||
if err := tx.Where("time < ?", cutoff.Add(-1*time.Hour)).Delete(&models.GPURecord{}).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
})
|
||||
// GetRecordsByTime 获取所有客户端在时间范围内的记录。
|
||||
func GetRecordsByTime(start, end time.Time) ([]models.Record, error) {
|
||||
return metricstore.GetRecordsByTime(context.Background(), start, end)
|
||||
}
|
||||
@@ -1,234 +0,0 @@
|
||||
package records
|
||||
|
||||
import (
|
||||
"encoding/csv"
|
||||
"os"
|
||||
"strconv"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"gorm.io/driver/sqlite"
|
||||
"gorm.io/gorm"
|
||||
|
||||
"github.com/komari-monitor/komari/database/models"
|
||||
)
|
||||
|
||||
var uuid = "7901508c-304f-49aa-b84f-957c33ae6f8a"
|
||||
|
||||
var _ = func() bool {
|
||||
// 确保 Test 环境中使用 sqlite 内存数据库
|
||||
return true
|
||||
}()
|
||||
|
||||
// TestCompactRecord tests the database compaction logic by inserting 4h30m of data (one record per minute),
|
||||
// then running migrateOldRecords and verifying the aggregation and cleanup.
|
||||
func TestCompactRecord(t *testing.T) {
|
||||
const totalMinutes = 12*60 + 30
|
||||
now := time.Now().UTC().Truncate(time.Minute)
|
||||
threshold := compactRecordCutoff(now)
|
||||
overlapCutoff := threshold.Add(-1 * time.Hour)
|
||||
|
||||
// 使用 sqlite 内存数据库并迁移表结构
|
||||
db, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{})
|
||||
assert.NoError(t, err)
|
||||
assert.NoError(t, db.AutoMigrate(&models.Record{}))
|
||||
assert.NoError(t, db.Table("records_long_term").AutoMigrate(&models.Record{}))
|
||||
|
||||
expectedGroups := make(map[time.Time]struct{})
|
||||
expectedRawRemain := 0
|
||||
|
||||
// 插入数据
|
||||
for i := 0; i < totalMinutes; i++ {
|
||||
recTime := now.Add(-time.Duration(i) * time.Minute)
|
||||
rec := models.Record{Client: uuid, Time: models.FromTime(recTime), Cpu: float32(i), Gpu: float32(i), Load: float32(i), Temp: float32(i), Ram: int64(i)}
|
||||
err := db.Create(&rec).Error
|
||||
assert.NoError(t, err)
|
||||
|
||||
if recTime.Before(threshold) {
|
||||
slot := recTime.Truncate(15 * time.Minute)
|
||||
expectedGroups[slot] = struct{}{}
|
||||
}
|
||||
if !recTime.Before(overlapCutoff) {
|
||||
expectedRawRemain++
|
||||
}
|
||||
}
|
||||
|
||||
// 导出原始数据到 CSV
|
||||
os.MkdirAll("../../data", 0755)
|
||||
var origRecs []models.Record
|
||||
db.Order("time desc").Find(&origRecs)
|
||||
fOrig, err := os.Create("../../data/original.csv")
|
||||
assert.NoError(t, err)
|
||||
defer fOrig.Close()
|
||||
wOrig := csv.NewWriter(fOrig)
|
||||
defer wOrig.Flush()
|
||||
wOrig.Write([]string{"Client", "Time", "Cpu", "Gpu", "Load", "Temp", "Ram"})
|
||||
for _, r := range origRecs {
|
||||
wOrig.Write([]string{
|
||||
r.Client,
|
||||
r.Time.ToTime().Format(time.RFC3339),
|
||||
strconv.FormatFloat(float64(r.Cpu), 'f', -1, 32),
|
||||
strconv.FormatFloat(float64(r.Gpu), 'f', -1, 32),
|
||||
strconv.FormatFloat(float64(r.Load), 'f', -1, 32),
|
||||
strconv.FormatFloat(float64(r.Temp), 'f', -1, 32),
|
||||
strconv.FormatInt(r.Ram, 10),
|
||||
})
|
||||
}
|
||||
|
||||
// 运行压缩(迁移)逻辑
|
||||
err = migrateOldRecordsAt(db, now)
|
||||
assert.NoError(t, err)
|
||||
|
||||
// 验证 long-term 表中的聚合记录数
|
||||
var longCount int64
|
||||
assert.NoError(t, db.Table("records_long_term").Count(&longCount).Error)
|
||||
assert.Equal(t, int64(len(expectedGroups)), longCount)
|
||||
|
||||
// 验证原始表中剩余记录数
|
||||
var remainCount int64
|
||||
assert.NoError(t, db.Table("records").Count(&remainCount).Error)
|
||||
assert.Equal(t, int64(expectedRawRemain), remainCount)
|
||||
|
||||
// 导出压缩后的数据到 CSV
|
||||
var compRecs []models.Record
|
||||
db.Table("records_long_term").Order("time desc").Find(&compRecs)
|
||||
fComp, err := os.Create("../../data/compressed.csv")
|
||||
assert.NoError(t, err)
|
||||
defer fComp.Close()
|
||||
wComp := csv.NewWriter(fComp)
|
||||
defer wComp.Flush()
|
||||
wComp.Write([]string{"Client", "Time", "Cpu", "Gpu", "Load", "Temp", "Ram"})
|
||||
for _, r := range compRecs {
|
||||
wComp.Write([]string{
|
||||
r.Client,
|
||||
r.Time.ToTime().Format(time.RFC3339),
|
||||
strconv.FormatFloat(float64(r.Cpu), 'f', -1, 32),
|
||||
strconv.FormatFloat(float64(r.Gpu), 'f', -1, 32),
|
||||
strconv.FormatFloat(float64(r.Load), 'f', -1, 32),
|
||||
strconv.FormatFloat(float64(r.Temp), 'f', -1, 32),
|
||||
strconv.FormatInt(r.Ram, 10),
|
||||
})
|
||||
}
|
||||
|
||||
db.Table("records").Order("time desc").Find(&compRecs)
|
||||
fComp, err = os.Create("../../data/compressed_records.csv")
|
||||
assert.NoError(t, err)
|
||||
defer fComp.Close()
|
||||
wComp = csv.NewWriter(fComp)
|
||||
defer wComp.Flush()
|
||||
wComp.Write([]string{"Client", "Time", "Cpu", "Gpu", "Load", "Temp", "Ram"})
|
||||
for _, r := range compRecs {
|
||||
wComp.Write([]string{
|
||||
r.Client,
|
||||
r.Time.ToTime().Format(time.RFC3339),
|
||||
strconv.FormatFloat(float64(r.Cpu), 'f', -1, 32),
|
||||
strconv.FormatFloat(float64(r.Gpu), 'f', -1, 32),
|
||||
strconv.FormatFloat(float64(r.Load), 'f', -1, 32),
|
||||
strconv.FormatFloat(float64(r.Temp), 'f', -1, 32),
|
||||
strconv.FormatInt(r.Ram, 10),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestCompactRecordPreservesExactTrafficDelta(t *testing.T) {
|
||||
currentTime := time.Now().UTC().Truncate(time.Minute)
|
||||
now := currentTime.Truncate(15 * time.Minute).Add(-5*time.Hour + time.Minute)
|
||||
db, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{})
|
||||
assert.NoError(t, err)
|
||||
assert.NoError(t, db.AutoMigrate(&models.Record{}))
|
||||
assert.NoError(t, db.Table("records_long_term").AutoMigrate(&models.Record{}))
|
||||
|
||||
records := []models.Record{
|
||||
{
|
||||
Client: uuid,
|
||||
Time: models.FromTime(now),
|
||||
NetTotalUp: 100,
|
||||
NetTotalDown: 200,
|
||||
TrafficUp: 0,
|
||||
TrafficDown: 0,
|
||||
},
|
||||
{
|
||||
Client: uuid,
|
||||
Time: models.FromTime(now.Add(5 * time.Minute)),
|
||||
NetTotalUp: 150,
|
||||
NetTotalDown: 260,
|
||||
TrafficUp: 50,
|
||||
TrafficDown: 60,
|
||||
},
|
||||
{
|
||||
Client: uuid,
|
||||
Time: models.FromTime(now.Add(10 * time.Minute)),
|
||||
NetTotalUp: 10,
|
||||
NetTotalDown: 30,
|
||||
TrafficUp: 10,
|
||||
TrafficDown: 30,
|
||||
},
|
||||
}
|
||||
|
||||
for _, rec := range records {
|
||||
assert.NoError(t, db.Create(&rec).Error)
|
||||
}
|
||||
|
||||
assert.NoError(t, migrateOldRecordsAt(db, currentTime))
|
||||
|
||||
var compacted []models.Record
|
||||
assert.NoError(t, db.Table("records_long_term").Find(&compacted).Error)
|
||||
assert.Len(t, compacted, 1)
|
||||
assert.Equal(t, int64(60), compacted[0].TrafficUp)
|
||||
assert.Equal(t, int64(90), compacted[0].TrafficDown)
|
||||
assert.Equal(t, int64(10), compacted[0].NetTotalUp)
|
||||
assert.Equal(t, int64(30), compacted[0].NetTotalDown)
|
||||
assert.True(t, compacted[0].Time.ToTime().Equal(records[2].Time.ToTime().Truncate(15*time.Minute)))
|
||||
}
|
||||
|
||||
func TestCompactRecordRetainsOneHourOverlapWindow(t *testing.T) {
|
||||
now := time.Date(2026, 6, 7, 12, 7, 0, 0, time.UTC)
|
||||
cutoff := compactRecordCutoff(now)
|
||||
db, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{})
|
||||
assert.NoError(t, err)
|
||||
assert.NoError(t, db.AutoMigrate(&models.Record{}))
|
||||
assert.NoError(t, db.Table("records_long_term").AutoMigrate(&models.Record{}))
|
||||
|
||||
records := []models.Record{
|
||||
{Client: uuid, Time: models.FromTime(cutoff.Add(-time.Hour - time.Minute)), TrafficUp: 5},
|
||||
{Client: uuid, Time: models.FromTime(cutoff.Add(-30 * time.Minute)), TrafficUp: 7},
|
||||
{Client: uuid, Time: models.FromTime(now.Add(-3 * time.Hour)), TrafficUp: 11},
|
||||
}
|
||||
for _, rec := range records {
|
||||
assert.NoError(t, db.Create(&rec).Error)
|
||||
}
|
||||
|
||||
assert.NoError(t, migrateOldRecordsAt(db, now))
|
||||
|
||||
var remainTimes []models.Record
|
||||
assert.NoError(t, db.Table("records").Order("time ASC").Find(&remainTimes).Error)
|
||||
assert.Len(t, remainTimes, 2)
|
||||
assert.True(t, remainTimes[0].Time.ToTime().Equal(records[1].Time.ToTime()))
|
||||
assert.True(t, remainTimes[1].Time.ToTime().Equal(records[2].Time.ToTime()))
|
||||
}
|
||||
|
||||
func TestCompactRecordOnlyMigratesCompleteFifteenMinuteBuckets(t *testing.T) {
|
||||
now := time.Date(2026, 6, 7, 12, 7, 0, 0, time.UTC)
|
||||
cutoff := compactRecordCutoff(now)
|
||||
db, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{})
|
||||
assert.NoError(t, err)
|
||||
assert.NoError(t, db.AutoMigrate(&models.Record{}))
|
||||
assert.NoError(t, db.Table("records_long_term").AutoMigrate(&models.Record{}))
|
||||
|
||||
compactable := models.Record{Client: uuid, Time: models.FromTime(cutoff.Add(-time.Minute)), TrafficUp: 5}
|
||||
partialSlot := models.Record{Client: uuid, Time: models.FromTime(cutoff.Add(time.Minute)), TrafficUp: 7}
|
||||
assert.NoError(t, db.Create(&compactable).Error)
|
||||
assert.NoError(t, db.Create(&partialSlot).Error)
|
||||
|
||||
assert.NoError(t, migrateOldRecordsAt(db, now))
|
||||
|
||||
var compacted []models.Record
|
||||
assert.NoError(t, db.Table("records_long_term").Order("time ASC").Find(&compacted).Error)
|
||||
assert.Len(t, compacted, 1)
|
||||
assert.True(t, compacted[0].Time.ToTime().Equal(compactable.Time.ToTime().Truncate(15*time.Minute)))
|
||||
|
||||
var rawCount int64
|
||||
assert.NoError(t, db.Table("records").Where("time = ?", partialSlot.Time).Count(&rawCount).Error)
|
||||
assert.Equal(t, int64(1), rawCount)
|
||||
}
|
||||
+16
-33
@@ -1,9 +1,11 @@
|
||||
package tasks
|
||||
|
||||
import (
|
||||
"context"
|
||||
"time"
|
||||
|
||||
"github.com/komari-monitor/komari/database/dbcore"
|
||||
"github.com/komari-monitor/komari/database/metricstore"
|
||||
"github.com/komari-monitor/komari/database/models"
|
||||
"github.com/komari-monitor/komari/utils"
|
||||
"gorm.io/gorm"
|
||||
@@ -45,6 +47,12 @@ func AddPingTask(clients []string, defaultOn bool, name string, target, task_typ
|
||||
}
|
||||
|
||||
func DeletePingTask(id []uint) error {
|
||||
// The metric store is independent from the main database, so clean it first
|
||||
// to avoid leaving history that can no longer be addressed through the task.
|
||||
if err := DeletePingRecords(id); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
db := dbcore.GetDBInstance()
|
||||
result := db.Where("id IN ?", id).Delete(&models.PingTask{})
|
||||
if result.RowsAffected == 0 {
|
||||
@@ -125,34 +133,21 @@ func UpdatePingTaskOrder(order map[uint]int) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func SavePingRecord(record models.PingRecord) error {
|
||||
db := dbcore.GetDBInstance()
|
||||
return db.Create(&record).Error
|
||||
}
|
||||
// ping 记录已完全迁移到 metric store(指标 ping.latency_ms),运行期读写全部走
|
||||
// metric store,旧 ping_records 表不再参与。
|
||||
|
||||
func DeletePingRecordsBefore(time time.Time) error {
|
||||
db := dbcore.GetDBInstance()
|
||||
err := db.Where("time < ?", time).Delete(&models.PingRecord{}).Error
|
||||
return err
|
||||
func SavePingRecord(record models.PingRecord) error {
|
||||
return metricstore.WritePingRecord(context.Background(), record)
|
||||
}
|
||||
|
||||
func DeletePingRecords(id []uint) error {
|
||||
db := dbcore.GetDBInstance()
|
||||
result := db.Where("task_id IN ?", id).Delete(&models.PingRecord{})
|
||||
if result.RowsAffected == 0 {
|
||||
return gorm.ErrRecordNotFound
|
||||
}
|
||||
return result.Error
|
||||
return metricstore.DeletePingRecordsByTask(context.Background(), id)
|
||||
}
|
||||
|
||||
func DeleteAllPingRecords() error {
|
||||
db := dbcore.GetDBInstance()
|
||||
result := db.Exec("DELETE FROM ping_records")
|
||||
if result.RowsAffected == 0 {
|
||||
return gorm.ErrRecordNotFound
|
||||
}
|
||||
return result.Error
|
||||
return metricstore.DeleteAllPingRecords(context.Background())
|
||||
}
|
||||
|
||||
func ReloadPingSchedule() error {
|
||||
db := dbcore.GetDBInstance()
|
||||
var pingTasks []models.PingTask
|
||||
@@ -201,17 +196,5 @@ func AddDefaultOnClientUUID(uuid string) error {
|
||||
}
|
||||
|
||||
func GetPingRecords(uuid string, taskId int, start, end time.Time) ([]models.PingRecord, error) {
|
||||
db := dbcore.GetDBInstance()
|
||||
var records []models.PingRecord
|
||||
dbQuery := db.Model(&models.PingRecord{})
|
||||
if uuid != "" {
|
||||
dbQuery = dbQuery.Where("client = ?", uuid)
|
||||
}
|
||||
if taskId >= 0 {
|
||||
dbQuery = dbQuery.Where("task_id = ?", uint(taskId))
|
||||
}
|
||||
if err := dbQuery.Where("time >= ? AND time <= ?", start, end).Order("time DESC").Find(&records).Error; err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return records, nil
|
||||
return metricstore.GetPingRecords(context.Background(), uuid, taskId, start, end)
|
||||
}
|
||||
+4
-15
@@ -26,7 +26,7 @@ func CreateTask(taskId string, clients []string, command string) error {
|
||||
Result: "",
|
||||
ExitCode: nil,
|
||||
FinishedAt: nil,
|
||||
CreatedAt: models.FromTime(time.Now()),
|
||||
CreatedAt: time.Now().UTC(),
|
||||
})
|
||||
}
|
||||
if len(taskResults) > 0 {
|
||||
@@ -57,13 +57,6 @@ func GetSpecificTaskResult(taskId, clientId string) (*models.TaskResult, error)
|
||||
return &result, nil
|
||||
}
|
||||
|
||||
func GetAllTasksResultByUUID(uuid string) ([]models.TaskResult, error) {
|
||||
var results []models.TaskResult
|
||||
if err := dbcore.GetDBInstance().Where("client = ?", uuid).Find(&results).Error; err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return results, nil
|
||||
}
|
||||
func GetAllTasks() ([]models.Task, error) {
|
||||
var tasks []models.Task
|
||||
if err := dbcore.GetDBInstance().Find(&tasks).Error; err != nil {
|
||||
@@ -79,21 +72,17 @@ func GetTaskResultsByTaskId(taskId string) ([]models.TaskResult, error) {
|
||||
}
|
||||
return results, nil
|
||||
}
|
||||
func DeleteTaskByTaskId(taskId string) error {
|
||||
return dbcore.GetDBInstance().Where("task_id = ?", taskId).Delete(&models.Task{}).Error
|
||||
}
|
||||
|
||||
func SaveTaskResult(taskId, clientId, result string, exitCode int, timestamp models.LocalTime) error {
|
||||
func SaveTaskResult(taskId, clientId, result string, exitCode int, timestamp time.Time) error {
|
||||
return dbcore.GetDBInstance().
|
||||
Model(&models.TaskResult{}).
|
||||
Where("task_id = ? AND client = ?", taskId, clientId).
|
||||
Updates(map[string]interface{}{
|
||||
"result": result,
|
||||
"exit_code": exitCode,
|
||||
"finished_at": timestamp,
|
||||
"finished_at": timestamp.UTC(),
|
||||
}).Error
|
||||
}
|
||||
|
||||
func ClearTaskResultsByTimeBefore(before time.Time) error {
|
||||
return dbcore.GetDBInstance().Where("created_at < ?", before.Format(time.RFC3339)).Delete(&models.TaskResult{}).Error
|
||||
return dbcore.GetDBInstance().Where("created_at < ?", before.UTC()).Delete(&models.TaskResult{}).Error
|
||||
}
|
||||
@@ -0,0 +1,42 @@
|
||||
package tasks
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/komari-monitor/komari/cmd/flags"
|
||||
"github.com/komari-monitor/komari/database/dbcore"
|
||||
"github.com/komari-monitor/komari/database/models"
|
||||
)
|
||||
|
||||
func TestClearTaskResultsByTimeBeforeUsesUTCTimeValue(t *testing.T) {
|
||||
flags.DatabaseType = flags.DatabaseTypeSQLite
|
||||
flags.DatabaseFile = "file:task_cleanup_time?mode=memory&cache=shared"
|
||||
db := dbcore.GetDBInstance()
|
||||
|
||||
taskID := "task-cleanup-time"
|
||||
if err := db.Create(&models.Task{TaskId: taskID, Command: "true"}).Error; err != nil {
|
||||
t.Fatalf("create task: %v", err)
|
||||
}
|
||||
cutoff := time.Date(2026, 7, 17, 12, 0, 0, 123456789, time.UTC)
|
||||
results := []models.TaskResult{
|
||||
{TaskId: taskID, Client: "old", CreatedAt: cutoff.Add(-time.Nanosecond)},
|
||||
{TaskId: taskID, Client: "boundary", CreatedAt: cutoff},
|
||||
{TaskId: taskID, Client: "new", CreatedAt: cutoff.Add(time.Nanosecond)},
|
||||
}
|
||||
if err := db.Create(&results).Error; err != nil {
|
||||
t.Fatalf("create task results: %v", err)
|
||||
}
|
||||
|
||||
localCutoff := cutoff.In(time.FixedZone("UTC+8", 8*60*60))
|
||||
if err := ClearTaskResultsByTimeBefore(localCutoff); err != nil {
|
||||
t.Fatalf("clear task results: %v", err)
|
||||
}
|
||||
var remaining []models.TaskResult
|
||||
if err := db.Where("task_id = ?", taskID).Order("created_at").Find(&remaining).Error; err != nil {
|
||||
t.Fatalf("load remaining results: %v", err)
|
||||
}
|
||||
if len(remaining) != 2 || remaining[0].Client != "boundary" || remaining[1].Client != "new" {
|
||||
t.Fatalf("remaining results = %#v, want boundary and new", remaining)
|
||||
}
|
||||
}
|
||||
+10
-13
@@ -1,6 +1,7 @@
|
||||
package database
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"log"
|
||||
"os"
|
||||
@@ -9,6 +10,7 @@ import (
|
||||
|
||||
"github.com/gin-gonic/gin"
|
||||
"github.com/komari-monitor/komari/database/dbcore"
|
||||
"github.com/komari-monitor/komari/database/metricstore"
|
||||
"github.com/komari-monitor/komari/database/models"
|
||||
"github.com/komari-monitor/komari/pkg/config"
|
||||
)
|
||||
@@ -42,15 +44,6 @@ func GetPublicInfo() (map[string]interface{}, error) {
|
||||
if !hasKey("o_auth_provider") {
|
||||
cst.OAuthProvider = "github"
|
||||
}
|
||||
if !hasKey("record_enabled") {
|
||||
cst.RecordEnabled = true
|
||||
}
|
||||
if !hasKey("record_preserve_time") {
|
||||
cst.RecordPreserveTime = 720
|
||||
}
|
||||
if !hasKey("ping_record_preserve_time") {
|
||||
cst.PingRecordPreserveTime = 24
|
||||
}
|
||||
|
||||
// Fallback defaults if we couldn't enumerate keys.
|
||||
if allErr != nil {
|
||||
@@ -61,7 +54,10 @@ func GetPublicInfo() (map[string]interface{}, error) {
|
||||
cst.Description = "Komari Monitor, a simple server monitoring tool."
|
||||
}
|
||||
}
|
||||
|
||||
retention, err := metricstore.GetRetentionSummary(context.Background())
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
db := dbcore.GetDBInstance()
|
||||
tc := models.ThemeConfiguration{}
|
||||
err = db.Model(&models.ThemeConfiguration{}).Where("short = ?", cst.Theme).First(&tc).Error
|
||||
@@ -135,10 +131,11 @@ func GetPublicInfo() (map[string]interface{}, error) {
|
||||
"oauth_provider": cst.OAuthProvider,
|
||||
"disable_password_login": cst.DisablePasswordLogin,
|
||||
"cors_origin_check_enabled": cst.CorsOriginCheckEnabled,
|
||||
"record_enabled": cst.RecordEnabled,
|
||||
"record_preserve_time": cst.RecordPreserveTime,
|
||||
"ping_record_preserve_time": cst.PingRecordPreserveTime,
|
||||
"record_enabled": retention.AllPositive, // 兼容旧版本主题
|
||||
"record_preserve_time": retention.MaxDays * 24,
|
||||
"ping_record_preserve_time": retention.MaxDays * 24,
|
||||
"private_site": cst.PrivateSite,
|
||||
"visitor_audit_enabled": cst.VisitorAuditEnabled,
|
||||
"theme": cst.Theme,
|
||||
"theme_settings": tc_data,
|
||||
}, nil
|
||||
|
||||
@@ -6,6 +6,11 @@
|
||||
|
||||
Komariは、サーバーのパフォーマンスを監視するためのシンプルで効率的なソリューションを提供することを目的とした、軽量の自己ホスト型サーバー監視ツールです。Webインターフェースを介してサーバーのステータスを表示し、軽量エージェントを介してデータを収集します。
|
||||
|
||||
> [!WARNING]
|
||||
> Komariは、自己ホスト型の監視/制御プログラムであり、所有しているシステム、または管理権限を得ているシステムでのみ使用してください。Komariを武器化したり、許可なく展開、アクセス、永続化、コマンド実行、その他の不正利用に使用したりしないでください。実際の悪用リスクについては、Huntressの分析をご参照ください:[Komari C2 agent abuse](https://www.huntress.com/blog/komari-c2-agent-abuse)。
|
||||
> Komariの展開および運用方法については、利用者自身が責任を負います。開発者は、無許可または不正な利用、およびその結果について責任を負いません。
|
||||
> Windows端末でリモートコントロールを有効にした場合、クライアントはユーザーがログインするたびに、KomariがリモートコントロールソフトウェアであることをWindows通知で知らせます。
|
||||
|
||||
[ドキュメント](https://komari-document.pages.dev/) | [Telegramグループ](https://t.me/komari_monitor)
|
||||
|
||||
## 特徴
|
||||
|
||||
@@ -6,6 +6,11 @@
|
||||
|
||||
Komari 是一款輕量級的自託管伺服器監控工具,旨在提供簡單、高效的伺服器性能監控解決方案。它支援透過 Web 介面查看伺服器狀態,並透過輕量級 Agent 收集數據。
|
||||
|
||||
> [!WARNING]
|
||||
> Komari 是一款自託管的監控/控制程式,僅應部署在你擁有或已獲授權管理的系統上。請勿將 Komari 武器化,或在未獲授權的情況下部署、存取、持久化、執行命令及從事其他濫用行為。關於現實中的濫用風險,可參考 Huntress 的分析:[Komari C2 agent abuse](https://www.huntress.com/blog/komari-c2-agent-abuse)。
|
||||
> 使用者需要自行承擔部署和使用 Komari 的責任。開發者不對未經授權或濫用行為及其後果承擔責任。
|
||||
> 在 Windows 端啟用遠端控制後,客戶端會在每次使用者登入時透過 Windows 通知提醒使用者 Komari 是一款遠端控制軟體。
|
||||
|
||||
[文檔](https://komari-document.pages.dev/) | [Telegram 群組](https://t.me/komari_monitor)
|
||||
|
||||
## 特性
|
||||
|
||||
@@ -6,6 +6,11 @@
|
||||
|
||||
Komari 是一款轻量级的自托管服务器监控工具,旨在提供简单、高效的服务器性能监控解决方案。它支持通过 Web 界面查看服务器状态,并通过轻量级 Agent 收集数据。
|
||||
|
||||
> [!WARNING]
|
||||
> Komari 是一款自托管的监控/控制程序,仅应部署在你拥有或已获得授权管理的系统上。请勿将 Komari 武器化,或在未获授权的情况下部署、访问、持久化、执行命令及从事其他滥用行为。关于现实中的滥用风险,可参考 Huntress 的分析:[Komari C2 agent abuse](https://www.huntress.com/blog/komari-c2-agent-abuse)。
|
||||
> 用户需要自行承担部署和使用 Komari 的责任。开发者不对未经授权或滥用行为及其后果承担责任。
|
||||
> 在 Windows 端开启远程控制后,客户端会在每次用户登录时通过 Windows 通知提醒用户 Komari 是一款远程控制软件。
|
||||
|
||||
[文档](https://komari-document.pages.dev/) | [文档(镜像站 By Geekertao)](https://www.komari.wiki) | [Telegram 群组](https://t.me/komari_monitor)
|
||||
|
||||
## 特性
|
||||
|
||||
@@ -0,0 +1,327 @@
|
||||
# Visitor Audit RPC 使用说明
|
||||
|
||||
本文档说明 `public:recordVisitorEvent` RPC2 接口的调用方式。该接口用于让主题前端把访客访问和前端操作记录写入 Komari 后端审计日志。
|
||||
|
||||
## 接口概览
|
||||
|
||||
- RPC 方法:`public:recordVisitorEvent`
|
||||
- RPC 路径:`/api/rpc2`
|
||||
- HTTP 方法:`POST`
|
||||
- 写入位置:现有审计日志表 `models.Log`
|
||||
- 管理端读取:复用现有 `admin:getLogs`
|
||||
- 日志类型:`msg_type = "visitor"`
|
||||
- 默认状态:关闭,需设置 `visitor_audit_enabled = true`
|
||||
|
||||
该接口不会信任前端传入的 IP。来源 IP 和 User-Agent 由后端从请求上下文记录。
|
||||
|
||||
## 权限
|
||||
|
||||
`public:recordVisitorEvent` 属于 `public` 命名空间,访客可调用。
|
||||
|
||||
私有站点模式下,该方法也在登录页白名单内,便于记录登录前/前台访问事件。
|
||||
|
||||
这是有意保留的行为;总开关关闭时,白名单中的方法仍可调用,但不会写库。
|
||||
|
||||
## 启用
|
||||
|
||||
存量实例和新实例默认都不会开放匿名日志写入。管理员可通过
|
||||
`admin:editSettings` 将 `visitor_audit_enabled` 设为 `true`。公开设置
|
||||
`public:getPublicSettings` 也会返回这个字段,主题可据此决定是否上报。
|
||||
|
||||
未启用时接口返回 `status = "disabled"`。启用后,每个来源 IP 使用内存令牌桶限流:
|
||||
平均每分钟 30 次,允许短时突发 10 次;超限时返回 `status = "rate_limited"`,不写库。
|
||||
|
||||
## 请求参数
|
||||
|
||||
| 参数 | 类型 | 必填 | 说明 |
|
||||
| --- | --- | --- | --- |
|
||||
| `event` | `string` | 是 | 事件名,例如 `page_view`、`node_open`、`search` |
|
||||
| `action` | `string` | 否 | `event` 的别名 |
|
||||
| `operation` | `string` | 否 | `event` 的别名 |
|
||||
| `path` | `string` | 否 | 前端路径,例如 `/`、`/instance/<uuid>` |
|
||||
| `route` | `string` | 否 | 前端路由名,例如 `home`、`instance-detail` |
|
||||
| `target` | `string` | 否 | 操作目标,例如节点 UUID、工具 key、按钮 key |
|
||||
| `detail` | `object` | 否 | 额外元数据,大小受限 |
|
||||
|
||||
`event`、`action`、`operation` 三者任选一个即可,优先级为:
|
||||
|
||||
```text
|
||||
event > action > operation
|
||||
```
|
||||
|
||||
## 后端自动记录的信息
|
||||
|
||||
| 字段 | 来源 |
|
||||
| --- | --- |
|
||||
| IP | `rpc.ContextMeta.RemoteIP`,来自服务端 `c.ClientIP()` |
|
||||
| User-Agent | `rpc.ContextMeta.UserAgent`,来自请求头 |
|
||||
| 用户 UUID | 登录状态下使用 `rpc.ContextMeta.UserUUID`;访客为空 |
|
||||
| 时间 | 后端写入审计日志时生成 |
|
||||
|
||||
## 字段限制
|
||||
|
||||
字符串字段按 Unicode 字符计数,`detail` 和最终 message 按序列化后的字节数计数。
|
||||
|
||||
| 字段 | 最大长度 |
|
||||
| --- | ---: |
|
||||
| `event` | 64 |
|
||||
| `path` | 512 |
|
||||
| `route` | 128 |
|
||||
| `target` | 128 |
|
||||
| User-Agent | 512 |
|
||||
| `detail` JSON | 2048 |
|
||||
| 最终日志 message | 4096 |
|
||||
|
||||
如果 `detail` 超过限制,不会完整保存,而会写入截断标记:
|
||||
|
||||
```json
|
||||
{
|
||||
"truncated": true,
|
||||
"size": 4096
|
||||
}
|
||||
```
|
||||
|
||||
## 事件名规范化
|
||||
|
||||
服务端会规范化事件名:
|
||||
|
||||
- 转小写
|
||||
- 空格转 `_`
|
||||
- 仅保留字母、数字、`_`、`-`、`:`、`.`
|
||||
- 最大长度 64
|
||||
|
||||
示例:
|
||||
|
||||
```text
|
||||
Page View -> page_view
|
||||
node:open.detail -> node:open.detail
|
||||
```
|
||||
|
||||
## 请求示例:页面访问
|
||||
|
||||
```bash
|
||||
curl -X POST http://localhost:25774/api/rpc2 \
|
||||
-H 'Content-Type: application/json' \
|
||||
--data '{
|
||||
"jsonrpc": "2.0",
|
||||
"method": "public:recordVisitorEvent",
|
||||
"params": {
|
||||
"event": "page_view",
|
||||
"path": "/",
|
||||
"route": "home",
|
||||
"detail": {
|
||||
"theme": "glassmorphism"
|
||||
}
|
||||
},
|
||||
"id": 1
|
||||
}'
|
||||
```
|
||||
|
||||
写入成功返回:
|
||||
|
||||
```json
|
||||
{
|
||||
"jsonrpc": "2.0",
|
||||
"result": {
|
||||
"status": "success"
|
||||
},
|
||||
"id": 1
|
||||
}
|
||||
```
|
||||
|
||||
`disabled` 和 `rate_limited` 也属于正常响应,不会返回 JSON-RPC 错误;主题上报不应影响主流程。
|
||||
|
||||
## 请求示例:打开节点详情
|
||||
|
||||
```json
|
||||
{
|
||||
"jsonrpc": "2.0",
|
||||
"method": "public:recordVisitorEvent",
|
||||
"params": {
|
||||
"event": "node_open",
|
||||
"path": "/instance/6f0b-example",
|
||||
"route": "instance-detail",
|
||||
"target": "6f0b-example",
|
||||
"detail": {
|
||||
"source": "node_card"
|
||||
}
|
||||
},
|
||||
"id": 2
|
||||
}
|
||||
```
|
||||
|
||||
## 请求示例:搜索
|
||||
|
||||
```json
|
||||
{
|
||||
"jsonrpc": "2.0",
|
||||
"method": "public:recordVisitorEvent",
|
||||
"params": {
|
||||
"event": "search",
|
||||
"path": "/",
|
||||
"route": "home",
|
||||
"detail": {
|
||||
"keyword_length": 6,
|
||||
"result_count": 3
|
||||
}
|
||||
},
|
||||
"id": 3
|
||||
}
|
||||
```
|
||||
|
||||
建议不要记录完整搜索关键词,避免把敏感内容写入审计日志。
|
||||
|
||||
## 前端封装示例
|
||||
|
||||
```ts
|
||||
interface VisitorAuditEvent {
|
||||
event?: string
|
||||
action?: string
|
||||
operation?: string
|
||||
path?: string
|
||||
route?: string
|
||||
target?: string
|
||||
detail?: Record<string, unknown>
|
||||
}
|
||||
|
||||
let auditRequestId = 0
|
||||
|
||||
export async function recordVisitorEvent(event: VisitorAuditEvent): Promise<void> {
|
||||
try {
|
||||
await fetch('/api/rpc2', {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
'Content-Type': 'application/json',
|
||||
},
|
||||
credentials: 'include',
|
||||
body: JSON.stringify({
|
||||
jsonrpc: '2.0',
|
||||
method: 'public:recordVisitorEvent',
|
||||
params: event,
|
||||
id: ++auditRequestId,
|
||||
}),
|
||||
})
|
||||
}
|
||||
catch {
|
||||
// 审计上报不应影响用户正常浏览
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
## Vue Router 访问记录示例
|
||||
|
||||
```ts
|
||||
router.afterEach((to) => {
|
||||
void recordVisitorEvent({
|
||||
event: 'page_view',
|
||||
path: to.fullPath,
|
||||
route: String(to.name ?? ''),
|
||||
detail: {
|
||||
params: Object.keys(to.params),
|
||||
query_keys: Object.keys(to.query),
|
||||
},
|
||||
})
|
||||
})
|
||||
```
|
||||
|
||||
建议只记录 query key,不记录完整 query value,避免泄露 token 或敏感参数。
|
||||
|
||||
## 推荐事件名
|
||||
|
||||
| 场景 | event |
|
||||
| --- | --- |
|
||||
| 页面访问 | `page_view` |
|
||||
| 打开节点详情 | `node_open` |
|
||||
| 搜索 | `search` |
|
||||
| 切换分组 | `group_change` |
|
||||
| 切换视图模式 | `view_mode_change` |
|
||||
| 点击管理入口 | `admin_entry_click` |
|
||||
| 打开高级工具 | `home_tool_open` |
|
||||
| 审计日志刷新 | `audit_refresh` |
|
||||
| 审计日志翻页 | `audit_page_change` |
|
||||
| 导出 JSON | `export_json` |
|
||||
| 导出 CSV | `export_csv` |
|
||||
| WebRTC 自检开始 | `webrtc_check_start` |
|
||||
| WebRTC 自检完成 | `webrtc_check_done` |
|
||||
|
||||
## 审计日志保存形态
|
||||
|
||||
写入现有 `models.Log`:
|
||||
|
||||
| 字段 | 值 |
|
||||
| --- | --- |
|
||||
| `ip` | 服务端看到的来源 IP |
|
||||
| `uuid` | 登录用户 UUID;访客为空 |
|
||||
| `msg_type` | `visitor` |
|
||||
| `message` | `visitor event: {...}` |
|
||||
| `time` | 后端写入时间 |
|
||||
|
||||
示例 `message`:
|
||||
|
||||
```text
|
||||
visitor event: {"event":"page_view","path":"/","route":"home","user_agent":"Mozilla/5.0 ...","detail":{"theme":"glassmorphism"}}
|
||||
```
|
||||
|
||||
## 管理端读取
|
||||
|
||||
复用现有接口:
|
||||
|
||||
```text
|
||||
admin:getLogs
|
||||
```
|
||||
|
||||
请求示例:
|
||||
|
||||
```json
|
||||
{
|
||||
"jsonrpc": "2.0",
|
||||
"method": "admin:getLogs",
|
||||
"params": {
|
||||
"limit": "100",
|
||||
"page": "1",
|
||||
"msg_type": "visitor"
|
||||
},
|
||||
"id": 10
|
||||
}
|
||||
```
|
||||
|
||||
`msg_type` 为可选的精确匹配过滤参数。传入 `visitor` 后,计数和分页都会在 SQL 查询层过滤,
|
||||
无需先拉取所有日志再由前端筛选。
|
||||
|
||||
## 安全注意事项
|
||||
|
||||
### 不要从前端传 IP
|
||||
|
||||
前端 IP 不可信。该接口会自动使用服务端看到的来源 IP。
|
||||
|
||||
### 不要记录敏感值
|
||||
|
||||
不建议记录:
|
||||
|
||||
- 密码
|
||||
- token
|
||||
- cookie
|
||||
- 完整 URL query
|
||||
- 完整搜索关键词
|
||||
- 导出内容
|
||||
- WebSSH 命令内容
|
||||
- 剪贴板内容
|
||||
|
||||
建议只记录:
|
||||
|
||||
- 操作类型
|
||||
- 路由
|
||||
- 目标 ID
|
||||
- 结果数量
|
||||
- 布尔状态
|
||||
- 非敏感摘要
|
||||
|
||||
### 上报不要阻塞主流程
|
||||
|
||||
前端调用建议使用:
|
||||
|
||||
```ts
|
||||
void recordVisitorEvent(...)
|
||||
```
|
||||
|
||||
即使上报失败,也不影响页面功能。
|
||||
@@ -1,24 +1,22 @@
|
||||
module github.com/komari-monitor/komari
|
||||
|
||||
go 1.24.0
|
||||
|
||||
toolchain go1.24.11
|
||||
go 1.25.0
|
||||
|
||||
require (
|
||||
github.com/coreos/go-oidc/v3 v3.15.0
|
||||
github.com/dop251/goja v0.0.0-20251008123653-cf18d89f3cf6
|
||||
github.com/dop251/goja_nodejs v0.0.0-20251015164255-5e94316bedaf
|
||||
github.com/gin-gonic/gin v1.10.0
|
||||
github.com/go-sql-driver/mysql v1.10.0
|
||||
github.com/google/uuid v1.6.0
|
||||
github.com/gorilla/websocket v1.5.3
|
||||
github.com/jackc/pgx/v5 v5.10.0
|
||||
github.com/mattn/go-sqlite3 v1.14.27
|
||||
github.com/oschwald/maxminddb-golang v1.13.1
|
||||
github.com/patrickmn/go-cache v2.1.0+incompatible
|
||||
github.com/pquerna/otp v1.5.0
|
||||
github.com/spf13/cobra v1.9.1
|
||||
github.com/stretchr/testify v1.10.0
|
||||
github.com/stretchr/testify v1.11.1
|
||||
golang.org/x/image v0.35.0
|
||||
google.golang.org/grpc v1.75.0
|
||||
google.golang.org/protobuf v1.36.6
|
||||
gorm.io/driver/sqlite v1.5.7
|
||||
gorm.io/gorm v1.26.1
|
||||
)
|
||||
@@ -32,6 +30,7 @@ require golang.org/x/crypto v0.39.0 // indirect
|
||||
require golang.org/x/net v0.41.0 // indirect
|
||||
|
||||
require (
|
||||
filippo.io/edwards25519 v1.2.0 // indirect
|
||||
github.com/boombuler/barcode v1.0.1-0.20190219062509-6c824513bacc // indirect
|
||||
github.com/bytedance/sonic v1.13.2 // indirect
|
||||
github.com/bytedance/sonic/loader v0.2.4 // indirect
|
||||
@@ -40,14 +39,17 @@ require (
|
||||
github.com/dlclark/regexp2 v1.11.4 // indirect
|
||||
github.com/gabriel-vasile/mimetype v1.4.8 // indirect
|
||||
github.com/gin-contrib/sse v1.0.0 // indirect
|
||||
github.com/go-jose/go-jose/v4 v4.1.1 // indirect
|
||||
github.com/go-playground/locales v0.14.1 // indirect
|
||||
github.com/go-playground/universal-translator v0.18.1 // indirect
|
||||
github.com/go-playground/validator/v10 v10.26.0 // indirect
|
||||
github.com/go-sourcemap/sourcemap v2.1.4+incompatible // indirect
|
||||
github.com/goccy/go-json v0.10.5 // indirect
|
||||
github.com/google/go-cmp v0.7.0 // indirect
|
||||
github.com/google/pprof v0.0.0-20240727154555-813a5fbdbec8 // indirect
|
||||
github.com/inconshreveable/mousetrap v1.1.0 // indirect
|
||||
github.com/jackc/pgpassfile v1.0.0 // indirect
|
||||
github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect
|
||||
github.com/jackc/puddle/v2 v2.2.2 // indirect
|
||||
github.com/jinzhu/inflection v1.0.0 // indirect
|
||||
github.com/jinzhu/now v1.1.5 // indirect
|
||||
github.com/json-iterator/go v1.1.12 // indirect
|
||||
@@ -55,7 +57,6 @@ require (
|
||||
github.com/kr/text v0.2.0 // indirect
|
||||
github.com/leodido/go-urn v1.4.0 // indirect
|
||||
github.com/mattn/go-isatty v0.0.20 // indirect
|
||||
github.com/mattn/go-sqlite3 v1.14.27 // indirect
|
||||
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
|
||||
github.com/modern-go/reflect2 v1.0.2 // indirect
|
||||
github.com/pelletier/go-toml/v2 v2.2.3 // indirect
|
||||
@@ -65,10 +66,9 @@ require (
|
||||
github.com/twitchyliquid64/golang-asm v0.15.1 // indirect
|
||||
github.com/ugorji/go/codec v1.2.12 // indirect
|
||||
golang.org/x/arch v0.15.0 // indirect
|
||||
golang.org/x/oauth2 v0.30.0 // indirect
|
||||
golang.org/x/sync v0.19.0 // indirect
|
||||
golang.org/x/sys v0.33.0 // indirect
|
||||
golang.org/x/text v0.33.0 // indirect
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20250707201910-8d1bb00bc6a7 // indirect
|
||||
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect
|
||||
google.golang.org/protobuf v1.36.6 // indirect
|
||||
gopkg.in/yaml.v3 v3.0.1 // indirect
|
||||
)
|
||||
@@ -1,3 +1,5 @@
|
||||
filippo.io/edwards25519 v1.2.0 h1:crnVqOiS4jqYleHd9vaKZ+HKtHfllngJIiOpNpoJsjo=
|
||||
filippo.io/edwards25519 v1.2.0/go.mod h1:xzAOLCNug/yB62zG1bQ8uziwrIqIuxhctzJT18Q77mc=
|
||||
github.com/Masterminds/semver/v3 v3.2.1 h1:RN9w6+7QoMeJVGyfmbcgs28Br8cvmnucEXnY0rYXWg0=
|
||||
github.com/Masterminds/semver/v3 v3.2.1/go.mod h1:qvl/7zhW3nngYb5+80sSMF+FG2BjYrf8m9wsX0PNOMQ=
|
||||
github.com/boombuler/barcode v1.0.1-0.20190219062509-6c824513bacc h1:biVzkmvwrH8WK8raXaxBx6fRVTlJILwEwQGL1I/ByEI=
|
||||
@@ -10,8 +12,6 @@ github.com/bytedance/sonic/loader v0.2.4/go.mod h1:N8A3vUdtUebEY2/VQC0MyhYeKUFos
|
||||
github.com/cloudwego/base64x v0.1.5 h1:XPciSp1xaq2VCSt6lF0phncD4koWyULpl5bUxbfCyP4=
|
||||
github.com/cloudwego/base64x v0.1.5/go.mod h1:0zlkT4Wn5C6NdauXdJRhSKRlJvmclQ1hhJgA0rcu/8w=
|
||||
github.com/cloudwego/iasm v0.2.0/go.mod h1:8rXZaNYT2n95jn+zTI1sDr+IgcD2GVs0nlbbQPiEFhY=
|
||||
github.com/coreos/go-oidc/v3 v3.15.0 h1:R6Oz8Z4bqWR7VFQ+sPSvZPQv4x8M+sJkDO5ojgwlyAg=
|
||||
github.com/coreos/go-oidc/v3 v3.15.0/go.mod h1:HaZ3szPaZ0e4r6ebqvsLWlk2Tn+aejfmrfah6hnSYEU=
|
||||
github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g=
|
||||
github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
|
||||
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
@@ -29,12 +29,6 @@ github.com/gin-contrib/sse v1.0.0 h1:y3bT1mUWUxDpW4JLQg/HnTqV4rozuW4tC9eFKTxYI9E
|
||||
github.com/gin-contrib/sse v1.0.0/go.mod h1:zNuFdwarAygJBht0NTKiSi3jRf6RbqeILZ9Sp6Slhe0=
|
||||
github.com/gin-gonic/gin v1.10.0 h1:nTuyha1TYqgedzytsKYqna+DfLos46nTv2ygFy86HFU=
|
||||
github.com/gin-gonic/gin v1.10.0/go.mod h1:4PMNQiOhvDRa013RKVbsiNwoyezlm2rm0uX/T7kzp5Y=
|
||||
github.com/go-jose/go-jose/v4 v4.1.1 h1:JYhSgy4mXXzAdF3nUx3ygx347LRXJRrpgyU3adRmkAI=
|
||||
github.com/go-jose/go-jose/v4 v4.1.1/go.mod h1:BdsZGqgdO3b6tTc6LSE56wcDbMMLuPsw5d4ZD5f94kA=
|
||||
github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI=
|
||||
github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
|
||||
github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag=
|
||||
github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE=
|
||||
github.com/go-playground/assert/v2 v2.2.0 h1:JvknZsQTYeFEAhQwI4qEt9cyV5ONwRHC+lYKSsYSR8s=
|
||||
github.com/go-playground/assert/v2 v2.2.0/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4=
|
||||
github.com/go-playground/locales v0.14.1 h1:EWaQ/wswjilfKLTECiXz7Rh+3BjFhfDFKv/oXslEjJA=
|
||||
@@ -45,10 +39,10 @@ github.com/go-playground/validator/v10 v10.26.0 h1:SP05Nqhjcvz81uJaRfEV0YBSSSGMc
|
||||
github.com/go-playground/validator/v10 v10.26.0/go.mod h1:I5QpIEbmr8On7W0TktmJAumgzX4CA1XNl4ZmDuVHKKo=
|
||||
github.com/go-sourcemap/sourcemap v2.1.4+incompatible h1:a+iTbH5auLKxaNwQFg0B+TCYl6lbukKPc7b5x0n1s6Q=
|
||||
github.com/go-sourcemap/sourcemap v2.1.4+incompatible/go.mod h1:F8jJfvm2KbVjc5NqelyYJmf/v5J0dwNLS2mL4sNA1Jg=
|
||||
github.com/go-sql-driver/mysql v1.10.0 h1:Q+1LV8DkHJvSYAdR83XzuhDaTykuDx0l6fkXxoWCWfw=
|
||||
github.com/go-sql-driver/mysql v1.10.0/go.mod h1:M+cqaI7+xxXGG9swrdeUIoPG3Y3KCkF0pZej+SK+nWk=
|
||||
github.com/goccy/go-json v0.10.5 h1:Fq85nIqj+gXn/S5ahsiTlK3TmC85qgirsdTP/+DeaC4=
|
||||
github.com/goccy/go-json v0.10.5/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M=
|
||||
github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek=
|
||||
github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps=
|
||||
github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=
|
||||
github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU=
|
||||
github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
|
||||
@@ -60,6 +54,14 @@ github.com/gorilla/websocket v1.5.3 h1:saDtZ6Pbx/0u+bgYQ3q96pZgCzfhKXGPqt7kZ72aN
|
||||
github.com/gorilla/websocket v1.5.3/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
|
||||
github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8=
|
||||
github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
|
||||
github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM=
|
||||
github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg=
|
||||
github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 h1:iCEnooe7UlwOQYpKFhBabPMi4aNAfoODPEFNiAnClxo=
|
||||
github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM=
|
||||
github.com/jackc/pgx/v5 v5.10.0 h1:VhSvgU2jSli8o3AqIEOTJr7rZwAEUVo4E4XhR94Zfr0=
|
||||
github.com/jackc/pgx/v5 v5.10.0/go.mod h1:mal1tBGAFfLHvZzaYh77YS/eC6IX9OWbRV1QIIM0Jn4=
|
||||
github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo=
|
||||
github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4=
|
||||
github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E=
|
||||
github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc=
|
||||
github.com/jinzhu/now v1.1.5 h1:/o9tlHleP7gOFmsnYNz3RGnqzefHA47wQpKrrdTIwXQ=
|
||||
@@ -71,7 +73,6 @@ github.com/klauspost/cpuid/v2 v2.2.10 h1:tBs3QSyvjDyFTq3uoc/9xFpCuOsJQFNPiAhYdw2
|
||||
github.com/klauspost/cpuid/v2 v2.2.10/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0=
|
||||
github.com/knz/go-libedit v1.10.1/go.mod h1:MZTVkCWyz0oBc7JOWP3wNAzd002ZbM/5hgShxwh4x8M=
|
||||
github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
|
||||
github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
|
||||
github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0=
|
||||
github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk=
|
||||
github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
|
||||
@@ -117,24 +118,13 @@ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/
|
||||
github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
|
||||
github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
|
||||
github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
|
||||
github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA=
|
||||
github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
|
||||
github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U=
|
||||
github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U=
|
||||
github.com/twitchyliquid64/golang-asm v0.15.1 h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS4MhqMhdFk5YI=
|
||||
github.com/twitchyliquid64/golang-asm v0.15.1/go.mod h1:a1lVb/DtPvCB8fslRZhAngC2+aY1QWCk3Cedj/Gdt08=
|
||||
github.com/ugorji/go/codec v1.2.12 h1:9LC83zGrHhuUA9l16C9AHXAqEV/2wBQ4nkvumAE65EE=
|
||||
github.com/ugorji/go/codec v1.2.12/go.mod h1:UNopzCgEMSXjBc6AOMqYvWC1ktqTAfzJZUZgYf6w6lg=
|
||||
go.opentelemetry.io/auto/sdk v1.1.0 h1:cH53jehLUN6UFLY71z+NDOiNJqDdPRaXzTel0sJySYA=
|
||||
go.opentelemetry.io/auto/sdk v1.1.0/go.mod h1:3wSPjt5PWp2RhlCcmmOial7AvC4DQqZb7a7wCow3W8A=
|
||||
go.opentelemetry.io/otel v1.37.0 h1:9zhNfelUvx0KBfu/gb+ZgeAfAgtWrfHJZcAqFC228wQ=
|
||||
go.opentelemetry.io/otel v1.37.0/go.mod h1:ehE/umFRLnuLa/vSccNq9oS1ErUlkkK71gMcN34UG8I=
|
||||
go.opentelemetry.io/otel/metric v1.37.0 h1:mvwbQS5m0tbmqML4NqK+e3aDiO02vsf/WgbsdpcPoZE=
|
||||
go.opentelemetry.io/otel/metric v1.37.0/go.mod h1:04wGrZurHYKOc+RKeye86GwKiTb9FKm1WHtO+4EVr2E=
|
||||
go.opentelemetry.io/otel/sdk v1.37.0 h1:ItB0QUqnjesGRvNcmAcU0LyvkVyGJ2xftD29bWdDvKI=
|
||||
go.opentelemetry.io/otel/sdk v1.37.0/go.mod h1:VredYzxUvuo2q3WRcDnKDjbdvmO0sCzOvVAiY+yUkAg=
|
||||
go.opentelemetry.io/otel/sdk/metric v1.37.0 h1:90lI228XrB9jCMuSdA0673aubgRobVZFhbjxHHspCPc=
|
||||
go.opentelemetry.io/otel/sdk/metric v1.37.0/go.mod h1:cNen4ZWfiD37l5NhS+Keb5RXVWZWpRE+9WyVCpbo5ps=
|
||||
go.opentelemetry.io/otel/trace v1.37.0 h1:HLdcFNbRQBE2imdSEgm/kwqmQj1Or1l/7bW6mxVK7z4=
|
||||
go.opentelemetry.io/otel/trace v1.37.0/go.mod h1:TlgrlQ+PtQO5XFerSPUYG0JSgGyryXewPGyayAWSBS0=
|
||||
golang.org/x/arch v0.15.0 h1:QtOrQd0bTUnhNVNndMpLHNWrDmYzZ2KDqSrEymqInZw=
|
||||
golang.org/x/arch v0.15.0/go.mod h1:JmwW7aLIoRUKgaTzhkiEFxvcEiQGyOg9BMonBJUS7EE=
|
||||
golang.org/x/crypto v0.39.0 h1:SHs+kF4LP+f+p14esP5jAoDpHU8Gu/v9lFRK6IT5imM=
|
||||
@@ -143,19 +133,13 @@ golang.org/x/image v0.35.0 h1:LKjiHdgMtO8z7Fh18nGY6KDcoEtVfsgLDPeLyguqb7I=
|
||||
golang.org/x/image v0.35.0/go.mod h1:MwPLTVgvxSASsxdLzKrl8BRFuyqMyGhLwmC+TO1Sybk=
|
||||
golang.org/x/net v0.41.0 h1:vBTly1HeNPEn3wtREYfy4GZ/NECgw2Cnl+nK6Nz3uvw=
|
||||
golang.org/x/net v0.41.0/go.mod h1:B/K4NNqkfmg07DQYrbwvSluqCJOOXwUjeb/5lOisjbA=
|
||||
golang.org/x/oauth2 v0.30.0 h1:dnDm7JmhM45NNpd8FDDeLhK6FwqbOf4MLCM9zb1BOHI=
|
||||
golang.org/x/oauth2 v0.30.0/go.mod h1:B++QgG3ZKulg6sRPGD/mqlHQs5rB3Ml9erfeDY7xKlU=
|
||||
golang.org/x/sync v0.19.0 h1:vV+1eWNmZ5geRlYjzm2adRgW2/mcpevXNg50YZtPCE4=
|
||||
golang.org/x/sync v0.19.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI=
|
||||
golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.33.0 h1:q3i8TbbEz+JRD9ywIRlyRAQbM0qF7hu24q3teo2hbuw=
|
||||
golang.org/x/sys v0.33.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k=
|
||||
golang.org/x/text v0.33.0 h1:B3njUFyqtHDUI5jMn1YIr5B0IE2U0qck04r6d4KPAxE=
|
||||
golang.org/x/text v0.33.0/go.mod h1:LuMebE6+rBincTi9+xWTY8TztLzKHc/9C1uBCG27+q8=
|
||||
gonum.org/v1/gonum v0.16.0 h1:5+ul4Swaf3ESvrOnidPp4GZbzf0mxVQpDCYUQE7OJfk=
|
||||
gonum.org/v1/gonum v0.16.0/go.mod h1:fef3am4MQ93R2HHpKnLk4/Tbh/s0+wqD5nfa6Pnwy4E=
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20250707201910-8d1bb00bc6a7 h1:pFyd6EwwL2TqFf8emdthzeX+gZE1ElRq3iM8pui4KBY=
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20250707201910-8d1bb00bc6a7/go.mod h1:qQ0YXyHHx3XkvlzUtpXDkS29lDSafHMZBAZDc03LQ3A=
|
||||
google.golang.org/grpc v1.75.0 h1:+TW+dqTd2Biwe6KKfhE5JpiYIBWq865PhKGSXiivqt4=
|
||||
google.golang.org/grpc v1.75.0/go.mod h1:JtPAzKiq4v1xcAB2hydNlWI2RnF85XXcV0mhKXr2ecQ=
|
||||
google.golang.org/protobuf v1.36.6 h1:z1NpPI8ku2WgiWnf+t9wTPsn6eP1L7ksHUlkfLvd9xY=
|
||||
google.golang.org/protobuf v1.36.6/go.mod h1:jduwjTPXsFjZGTmRluh+L6NjiWu7pchiJ2/5YcXBHnY=
|
||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
|
||||
+300
-77
@@ -31,9 +31,135 @@ SERVICE_NAME="komari"
|
||||
BINARY_PATH="$INSTALL_DIR/komari"
|
||||
DEFAULT_PORT="25774"
|
||||
LISTEN_PORT=""
|
||||
REPO="komari-monitor/komari"
|
||||
# 发布通道: stable(稳定版)或 snapshot(快照版)
|
||||
CHANNEL="stable"
|
||||
# TUI 工具: whiptail / dialog / 空(回退纯文本)
|
||||
TUI_TOOL=""
|
||||
|
||||
# Show banner
|
||||
# ==========================================================
|
||||
# TUI / 交互层
|
||||
# ==========================================================
|
||||
|
||||
# 检测可用的 TUI 工具
|
||||
detect_tui() {
|
||||
if command -v whiptail >/dev/null 2>&1; then
|
||||
TUI_TOOL="whiptail"
|
||||
elif command -v dialog >/dev/null 2>&1; then
|
||||
TUI_TOOL="dialog"
|
||||
else
|
||||
TUI_TOOL=""
|
||||
fi
|
||||
}
|
||||
|
||||
# 是否启用 TUI
|
||||
tui_enabled() {
|
||||
[ -n "$TUI_TOOL" ]
|
||||
}
|
||||
|
||||
# 菜单选择
|
||||
# 用法: ui_menu "标题" "提示" tag1 "item1" tag2 "item2" ...
|
||||
# 返回: 选中的 tag(输出到 stdout),取消返回非零
|
||||
ui_menu() {
|
||||
local title="$1"; shift
|
||||
local prompt="$1"; shift
|
||||
|
||||
if tui_enabled; then
|
||||
$TUI_TOOL --title "$title" --menu "$prompt" 20 70 10 "$@" 3>&1 1>&2 2>&3
|
||||
return $?
|
||||
fi
|
||||
|
||||
# 纯文本回退
|
||||
{
|
||||
echo
|
||||
echo "=============================================================="
|
||||
echo " $title"
|
||||
echo "=============================================================="
|
||||
echo "$prompt"
|
||||
echo
|
||||
local tag item
|
||||
local args=("$@")
|
||||
local i=0
|
||||
while [ $i -lt ${#args[@]} ]; do
|
||||
tag="${args[$i]}"
|
||||
item="${args[$((i + 1))]}"
|
||||
echo " $tag) $item"
|
||||
i=$((i + 2))
|
||||
done
|
||||
echo
|
||||
} >&2
|
||||
local choice
|
||||
read -r -p "输入选项: " choice >&2
|
||||
echo "$choice"
|
||||
}
|
||||
|
||||
# 输入框
|
||||
# 用法: ui_input "标题" "提示" "默认值"
|
||||
# 返回: 输入内容(输出到 stdout),取消返回非零
|
||||
ui_input() {
|
||||
local title="$1"
|
||||
local prompt="$2"
|
||||
local default="$3"
|
||||
|
||||
if tui_enabled; then
|
||||
$TUI_TOOL --title "$title" --inputbox "$prompt" 12 70 "$default" 3>&1 1>&2 2>&3
|
||||
return $?
|
||||
fi
|
||||
|
||||
local input
|
||||
read -r -p "$prompt [默认: $default]: " input >&2
|
||||
if [ -z "$input" ]; then
|
||||
echo "$default"
|
||||
else
|
||||
echo "$input"
|
||||
fi
|
||||
}
|
||||
|
||||
# 是/否确认
|
||||
# 用法: ui_yesno "标题" "提示"
|
||||
# 返回: 0 表示 是,1 表示 否
|
||||
ui_yesno() {
|
||||
local title="$1"
|
||||
local prompt="$2"
|
||||
|
||||
if tui_enabled; then
|
||||
$TUI_TOOL --title "$title" --yesno "$prompt" 12 70
|
||||
return $?
|
||||
fi
|
||||
|
||||
local confirm
|
||||
read -r -p "$prompt (Y/n): " confirm >&2
|
||||
if [[ $confirm =~ ^[Nn]$ ]]; then
|
||||
return 1
|
||||
fi
|
||||
return 0
|
||||
}
|
||||
|
||||
# 信息提示框
|
||||
# 用法: ui_msgbox "标题" "内容"
|
||||
ui_msgbox() {
|
||||
local title="$1"
|
||||
local content="$2"
|
||||
|
||||
if tui_enabled; then
|
||||
$TUI_TOOL --title "$title" --msgbox "$content" 20 72
|
||||
return
|
||||
fi
|
||||
|
||||
echo
|
||||
echo "=============================================================="
|
||||
echo " $title"
|
||||
echo "=============================================================="
|
||||
echo -e "$content"
|
||||
echo "=============================================================="
|
||||
read -r -p "按回车键继续..." _
|
||||
}
|
||||
|
||||
# 显示横幅(仅纯文本模式)
|
||||
show_banner() {
|
||||
if tui_enabled; then
|
||||
return
|
||||
fi
|
||||
clear
|
||||
echo "=============================================================="
|
||||
echo " Komari Monitoring System Installer"
|
||||
@@ -42,6 +168,31 @@ show_banner() {
|
||||
echo
|
||||
}
|
||||
|
||||
# 选择发布通道,结果写入全局变量 CHANNEL
|
||||
select_channel() {
|
||||
local choice
|
||||
choice=$(ui_menu "选择发布通道" "请选择要使用的发布通道:" \
|
||||
"stable" "稳定版 (推荐)" \
|
||||
"snapshot" "快照版 (最新功能)")
|
||||
|
||||
case "$choice" in
|
||||
snapshot|2)
|
||||
CHANNEL="snapshot"
|
||||
;;
|
||||
stable|1|"")
|
||||
CHANNEL="stable"
|
||||
;;
|
||||
*)
|
||||
CHANNEL="stable"
|
||||
;;
|
||||
esac
|
||||
log_info "已选择通道: $CHANNEL"
|
||||
}
|
||||
|
||||
# ==========================================================
|
||||
# 基础检查
|
||||
# ==========================================================
|
||||
|
||||
# Check if running as root
|
||||
check_root() {
|
||||
if [ "$EUID" -ne 0 ]; then
|
||||
@@ -75,6 +226,9 @@ detect_arch() {
|
||||
riscv64)
|
||||
echo "riscv64"
|
||||
;;
|
||||
loongarch64|loong64)
|
||||
echo "loong64"
|
||||
;;
|
||||
*)
|
||||
log_error "不支持的架构: $arch"
|
||||
exit 1
|
||||
@@ -113,19 +267,54 @@ install_dependencies() {
|
||||
fi
|
||||
}
|
||||
|
||||
# Get download URL based on channel
|
||||
get_download_url() {
|
||||
local arch=$1
|
||||
local file_name="komari-linux-${arch}"
|
||||
|
||||
if [ "$CHANNEL" = "snapshot" ]; then
|
||||
# 获取最新的 snapshot 预发布版本
|
||||
log_info "获取最新 snapshot 版本..." >&2
|
||||
local latest_snapshot=$(curl -s "https://api.github.com/repos/${REPO}/releases" | grep '"tag_name"' | grep 'Snapshot-' | head -1 | sed -e 's/.*"tag_name": *"//' -e 's/".*//')
|
||||
|
||||
if [ -z "$latest_snapshot" ]; then
|
||||
log_error "未找到 snapshot 版本" >&2
|
||||
return 1
|
||||
fi
|
||||
|
||||
log_info "最新 snapshot 版本: $latest_snapshot" >&2
|
||||
echo "https://github.com/${REPO}/releases/download/${latest_snapshot}/${file_name}"
|
||||
else
|
||||
# 稳定版:使用 latest
|
||||
echo "https://github.com/${REPO}/releases/latest/download/${file_name}"
|
||||
fi
|
||||
}
|
||||
|
||||
# ==========================================================
|
||||
# 业务操作
|
||||
# ==========================================================
|
||||
|
||||
# Binary installation
|
||||
install_binary() {
|
||||
log_step "开始二进制安装..."
|
||||
|
||||
if is_installed; then
|
||||
log_info "Komari 已安装。要升级,请使用升级选项。"
|
||||
ui_msgbox "提示" "Komari 已安装。\n如需升级,请使用主菜单中的升级选项。"
|
||||
return
|
||||
fi
|
||||
|
||||
# 选择发布通道
|
||||
select_channel
|
||||
|
||||
# 监听端口输入,校验范围 1-65535
|
||||
while true; do
|
||||
read -p "请输入监听端口 [默认: $DEFAULT_PORT]: " input_port
|
||||
local input_port
|
||||
input_port=$(ui_input "监听端口" "请输入 Komari 的监听端口 (1-65535):" "$DEFAULT_PORT")
|
||||
# 取消输入
|
||||
if [ $? -ne 0 ]; then
|
||||
log_info "安装已取消"
|
||||
return
|
||||
fi
|
||||
if [[ -z "$input_port" ]]; then
|
||||
LISTEN_PORT="$DEFAULT_PORT"
|
||||
break
|
||||
@@ -133,7 +322,7 @@ install_binary() {
|
||||
LISTEN_PORT="$input_port"
|
||||
break
|
||||
else
|
||||
log_error "端口号无效,请输入 1-65535 之间的数字。"
|
||||
ui_msgbox "错误" "端口号无效,请输入 1-65535 之间的数字。"
|
||||
fi
|
||||
done
|
||||
|
||||
@@ -148,14 +337,17 @@ install_binary() {
|
||||
log_step "创建数据目录: $DATA_DIR"
|
||||
mkdir -p "$DATA_DIR"
|
||||
|
||||
local file_name="komari-linux-${arch}"
|
||||
local download_url="https://github.com/komari-monitor/komari/releases/latest/download/${file_name}"
|
||||
local download_url=$(get_download_url "$arch")
|
||||
if [ $? -ne 0 ]; then
|
||||
ui_msgbox "错误" "获取下载链接失败,请检查网络连接或稍后重试。"
|
||||
return 1
|
||||
fi
|
||||
|
||||
log_step "下载 Komari 二进制文件..."
|
||||
log_info "URL: $download_url"
|
||||
|
||||
if ! curl -L -o "$BINARY_PATH" "$download_url"; then
|
||||
log_error "下载失败"
|
||||
ui_msgbox "错误" "下载失败,请检查网络连接。"
|
||||
return 1
|
||||
fi
|
||||
|
||||
@@ -163,11 +355,7 @@ install_binary() {
|
||||
log_success "Komari 二进制文件安装完成: $BINARY_PATH"
|
||||
|
||||
if ! check_systemd; then
|
||||
log_step "警告:未检测到 systemd,跳过服务创建。"
|
||||
log_step "您可以从命令行手动运行 Komari:"
|
||||
log_step " $BINARY_PATH server -l 0.0.0.0:$LISTEN_PORT"
|
||||
echo
|
||||
log_success "安装完成!"
|
||||
ui_msgbox "安装完成" "警告:未检测到 systemd,已跳过服务创建。\n\n您可以手动运行 Komari:\n $BINARY_PATH server -l 0.0.0.0:$LISTEN_PORT"
|
||||
return
|
||||
fi
|
||||
|
||||
@@ -179,17 +367,16 @@ install_binary() {
|
||||
|
||||
if systemctl is-active --quiet ${SERVICE_NAME}.service; then
|
||||
log_success "Komari 服务启动成功"
|
||||
|
||||
|
||||
log_step "正在获取初始密码..."
|
||||
sleep 5
|
||||
sleep 5
|
||||
local password=$(journalctl -u ${SERVICE_NAME} --since "1 minute ago" | grep "admin account created." | tail -n 1 | sed -e 's/.*admin account created.//')
|
||||
if [ -z "$password" ]; then
|
||||
log_error "未能获取初始密码,请检查日志"
|
||||
fi
|
||||
show_access_info "$password" "$LISTEN_PORT"
|
||||
else
|
||||
log_error "Komari 服务启动失败"
|
||||
log_info "查看日志: journalctl -u ${SERVICE_NAME} -f"
|
||||
ui_msgbox "错误" "Komari 服务启动失败。\n\n查看日志: journalctl -u ${SERVICE_NAME} -f"
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
@@ -223,21 +410,22 @@ EOF
|
||||
show_access_info() {
|
||||
local password=$1
|
||||
local port=${2:-$DEFAULT_PORT}
|
||||
echo
|
||||
log_success "安装完成!"
|
||||
echo
|
||||
log_info "访问信息:"
|
||||
log_info " URL: http://$(hostname -I | awk '{print $1}'):${port}"
|
||||
local ip=$(hostname -I | awk '{print $1}')
|
||||
|
||||
local content="安装完成!\n\n"
|
||||
content+="访问信息:\n"
|
||||
content+=" URL: http://${ip}:${port}\n"
|
||||
if [ -n "$password" ]; then
|
||||
log_info "初始登录信息(仅显示一次): $password"
|
||||
content+=" 初始登录信息(仅显示一次): ${password}\n"
|
||||
fi
|
||||
echo
|
||||
log_info "服务管理命令:"
|
||||
log_info " 状态: systemctl status $SERVICE_NAME"
|
||||
log_info " 启动: systemctl start $SERVICE_NAME"
|
||||
log_info " 停止: systemctl stop $SERVICE_NAME"
|
||||
log_info " 重启: systemctl restart $SERVICE_NAME"
|
||||
log_info " 日志: journalctl -u $SERVICE_NAME -f"
|
||||
content+="\n服务管理命令:\n"
|
||||
content+=" 状态: systemctl status $SERVICE_NAME\n"
|
||||
content+=" 启动: systemctl start $SERVICE_NAME\n"
|
||||
content+=" 停止: systemctl stop $SERVICE_NAME\n"
|
||||
content+=" 重启: systemctl restart $SERVICE_NAME\n"
|
||||
content+=" 日志: journalctl -u $SERVICE_NAME -f"
|
||||
|
||||
ui_msgbox "安装完成" "$content"
|
||||
}
|
||||
|
||||
# Upgrade function
|
||||
@@ -245,15 +433,18 @@ upgrade_komari() {
|
||||
log_step "升级 Komari..."
|
||||
|
||||
if ! is_installed; then
|
||||
log_error "Komari 未安装。请先安装它。"
|
||||
ui_msgbox "错误" "Komari 未安装。请先安装它。"
|
||||
return 1
|
||||
fi
|
||||
|
||||
if ! check_systemd; then
|
||||
log_error "未检测到 systemd。无法管理服务。"
|
||||
ui_msgbox "错误" "未检测到 systemd。无法管理服务。"
|
||||
return 1
|
||||
fi
|
||||
|
||||
# 选择发布通道
|
||||
select_channel
|
||||
|
||||
log_step "停止 Komari 服务..."
|
||||
systemctl stop ${SERVICE_NAME}.service
|
||||
|
||||
@@ -261,14 +452,21 @@ upgrade_komari() {
|
||||
cp "$BINARY_PATH" "${BINARY_PATH}.backup.$(date +%Y%m%d_%H%M%S)"
|
||||
|
||||
local arch=$(detect_arch)
|
||||
local file_name="komari-linux-${arch}"
|
||||
local download_url="https://github.com/komari-monitor/komari/releases/latest/download/${file_name}"
|
||||
local download_url=$(get_download_url "$arch")
|
||||
if [ $? -ne 0 ]; then
|
||||
log_error "获取下载链接失败,正在从备份恢复"
|
||||
mv "${BINARY_PATH}.backup."* "$BINARY_PATH"
|
||||
systemctl start ${SERVICE_NAME}.service
|
||||
ui_msgbox "错误" "获取下载链接失败,已从备份恢复。"
|
||||
return 1
|
||||
fi
|
||||
|
||||
log_step "下载最新版本..."
|
||||
if ! curl -L -o "$BINARY_PATH" "$download_url"; then
|
||||
log_error "下载失败,正在从备份恢复"
|
||||
mv "${BINARY_PATH}.backup."* "$BINARY_PATH"
|
||||
systemctl start ${SERVICE_NAME}.service
|
||||
ui_msgbox "错误" "下载失败,已从备份恢复。"
|
||||
return 1
|
||||
fi
|
||||
|
||||
@@ -278,9 +476,9 @@ upgrade_komari() {
|
||||
systemctl start ${SERVICE_NAME}.service
|
||||
|
||||
if systemctl is-active --quiet ${SERVICE_NAME}.service; then
|
||||
log_success "Komari 升级成功"
|
||||
ui_msgbox "升级完成" "Komari 升级成功 (通道: $CHANNEL)。"
|
||||
else
|
||||
log_error "服务在升级后未能启动"
|
||||
ui_msgbox "错误" "服务在升级后未能启动,请检查日志。"
|
||||
fi
|
||||
}
|
||||
|
||||
@@ -289,12 +487,11 @@ uninstall_komari() {
|
||||
log_step "卸载 Komari..."
|
||||
|
||||
if ! is_installed; then
|
||||
log_info "Komari 未安装"
|
||||
ui_msgbox "提示" "Komari 未安装。"
|
||||
return 0
|
||||
fi
|
||||
|
||||
read -p "这将删除 Komari。您确定吗?(Y/n): " confirm
|
||||
if [[ $confirm =~ ^[Nn]$ ]]; then
|
||||
if ! ui_yesno "确认卸载" "这将删除 Komari 二进制文件和服务。\n\n您确定要继续吗?"; then
|
||||
log_info "卸载已取消"
|
||||
return 0
|
||||
fi
|
||||
@@ -314,102 +511,128 @@ uninstall_komari() {
|
||||
rmdir "$INSTALL_DIR" 2>/dev/null || log_info "数据目录 $INSTALL_DIR 不为空,未删除"
|
||||
log_success "Komari 二进制文件已删除"
|
||||
|
||||
log_success "Komari 卸载完成"
|
||||
log_info "数据文件保留在 $DATA_DIR"
|
||||
ui_msgbox "卸载完成" "Komari 卸载完成。\n\n数据文件保留在 $DATA_DIR"
|
||||
}
|
||||
|
||||
# Show service status
|
||||
show_status() {
|
||||
if ! is_installed; then
|
||||
log_error "Komari 未安装"
|
||||
ui_msgbox "错误" "Komari 未安装。"
|
||||
return
|
||||
fi
|
||||
if ! check_systemd; then
|
||||
log_error "未检测到 systemd。无法获取服务状态。"
|
||||
ui_msgbox "错误" "未检测到 systemd。无法获取服务状态。"
|
||||
return
|
||||
fi
|
||||
log_step "Komari 服务状态:"
|
||||
systemctl status ${SERVICE_NAME}.service --no-pager -l
|
||||
if tui_enabled; then
|
||||
local status_output
|
||||
status_output=$(systemctl status ${SERVICE_NAME}.service --no-pager -l 2>&1)
|
||||
ui_msgbox "服务状态" "$status_output"
|
||||
else
|
||||
log_step "Komari 服务状态:"
|
||||
systemctl status ${SERVICE_NAME}.service --no-pager -l
|
||||
read -r -p "按回车键继续..." _
|
||||
fi
|
||||
}
|
||||
|
||||
# Show service logs
|
||||
show_logs() {
|
||||
if ! is_installed; then
|
||||
log_error "Komari 未安装"
|
||||
ui_msgbox "错误" "Komari 未安装。"
|
||||
return
|
||||
fi
|
||||
if ! check_systemd; then
|
||||
log_error "未检测到 systemd。无法获取服务日志。"
|
||||
ui_msgbox "错误" "未检测到 systemd。无法获取服务日志。"
|
||||
return
|
||||
fi
|
||||
log_step "查看 Komari 服务日志..."
|
||||
# 日志为实时流,直接在终端显示
|
||||
if tui_enabled; then
|
||||
clear
|
||||
fi
|
||||
log_step "查看 Komari 服务日志 (按 Ctrl+C 退出)..."
|
||||
journalctl -u ${SERVICE_NAME} -f --no-pager
|
||||
}
|
||||
|
||||
# Restart service
|
||||
restart_service() {
|
||||
if ! is_installed; then
|
||||
log_error "Komari 未安装"
|
||||
ui_msgbox "错误" "Komari 未安装。"
|
||||
return
|
||||
fi
|
||||
if ! check_systemd; then
|
||||
log_error "未检测到 systemd。无法重启服务。"
|
||||
ui_msgbox "错误" "未检测到 systemd。无法重启服务。"
|
||||
return
|
||||
fi
|
||||
log_step "重启 Komari 服务..."
|
||||
systemctl restart ${SERVICE_NAME}.service
|
||||
if systemctl is-active --quiet ${SERVICE_NAME}.service; then
|
||||
log_success "服务重启成功"
|
||||
ui_msgbox "成功" "服务重启成功。"
|
||||
else
|
||||
log_error "服务重启失败"
|
||||
ui_msgbox "错误" "服务重启失败,请检查日志。"
|
||||
fi
|
||||
}
|
||||
|
||||
# Stop service
|
||||
stop_service() {
|
||||
if ! is_installed; then
|
||||
log_error "Komari 未安装"
|
||||
ui_msgbox "错误" "Komari 未安装。"
|
||||
return
|
||||
fi
|
||||
if ! check_systemd; then
|
||||
log_error "未检测到 systemd。无法停止服务。"
|
||||
ui_msgbox "错误" "未检测到 systemd。无法停止服务。"
|
||||
return
|
||||
fi
|
||||
log_step "停止 Komari 服务..."
|
||||
systemctl stop ${SERVICE_NAME}.service
|
||||
log_success "服务已停止"
|
||||
ui_msgbox "成功" "服务已停止。"
|
||||
}
|
||||
|
||||
|
||||
# Main menu
|
||||
main_menu() {
|
||||
show_banner
|
||||
echo "请选择操作:"
|
||||
echo " 1) 安装 Komari"
|
||||
echo " 2) 升级 Komari"
|
||||
echo " 3) 卸载 Komari"
|
||||
echo " 4) 查看状态"
|
||||
echo " 5) 查看日志"
|
||||
echo " 6) 重启服务"
|
||||
echo " 7) 停止服务"
|
||||
echo " 8) 退出"
|
||||
echo
|
||||
while true; do
|
||||
show_banner
|
||||
|
||||
read -p "输入选项 [1-8]: " choice
|
||||
local choice
|
||||
choice=$(ui_menu "Komari 监控系统安装器" "请选择操作:" \
|
||||
"1" "安装 Komari" \
|
||||
"2" "升级 Komari" \
|
||||
"3" "卸载 Komari" \
|
||||
"4" "查看状态" \
|
||||
"5" "查看日志" \
|
||||
"6" "重启服务" \
|
||||
"7" "停止服务" \
|
||||
"8" "退出")
|
||||
|
||||
case $choice in
|
||||
1) install_binary ;;
|
||||
2) upgrade_komari ;;
|
||||
3) uninstall_komari ;;
|
||||
4) show_status ;;
|
||||
5) show_logs ;;
|
||||
6) restart_service ;;
|
||||
7) stop_service ;;
|
||||
8) exit 0 ;;
|
||||
*) log_error "无效选项" ;;
|
||||
esac
|
||||
# 用户在 TUI 中取消(ESC/Cancel)则退出
|
||||
if [ $? -ne 0 ] && tui_enabled; then
|
||||
clear
|
||||
exit 0
|
||||
fi
|
||||
|
||||
case $choice in
|
||||
1) install_binary ;;
|
||||
2) upgrade_komari ;;
|
||||
3) uninstall_komari ;;
|
||||
4) show_status ;;
|
||||
5) show_logs ;;
|
||||
6) restart_service ;;
|
||||
7) stop_service ;;
|
||||
8)
|
||||
tui_enabled && clear
|
||||
exit 0
|
||||
;;
|
||||
*) ui_msgbox "错误" "无效选项" ;;
|
||||
esac
|
||||
|
||||
# 纯文本模式下单次执行后退出循环(保持原有行为,避免输出被覆盖)
|
||||
if ! tui_enabled; then
|
||||
break
|
||||
fi
|
||||
done
|
||||
}
|
||||
|
||||
# Main execution
|
||||
check_root
|
||||
detect_tui
|
||||
main_menu
|
||||
+1
-92
@@ -31,26 +31,6 @@ var (
|
||||
}
|
||||
)
|
||||
|
||||
// Get 获取原始值 (反序列化为 interface{})
|
||||
func Get(key string, defaul ...any) (any, error) {
|
||||
var item ConfigItem
|
||||
err := db.First(&item, "key = ?", key).Error
|
||||
if err != nil {
|
||||
if len(defaul) > 0 {
|
||||
v := defaul[0]
|
||||
err = Set(key, v)
|
||||
return v, err
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
|
||||
var val any
|
||||
if err := json.Unmarshal([]byte(item.Value), &val); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return val, nil
|
||||
}
|
||||
|
||||
// GetAs 获取并转换为指定类型 (泛型),支持数值类型自动转换
|
||||
func GetAs[T any](key string, defaul ...any) (T, error) {
|
||||
var t T
|
||||
@@ -180,9 +160,8 @@ func GetManyAs[T any]() (*T, error) {
|
||||
}
|
||||
|
||||
defaultTag := field.Tag.Get("default")
|
||||
hasDefault := defaultTag != "" || field.Tag.Get("default") == ""
|
||||
// 检查是否显式定义了 default tag (即使值为空)
|
||||
_, hasDefault = field.Tag.Lookup("default")
|
||||
_, hasDefault := field.Tag.Lookup("default")
|
||||
|
||||
fields = append(fields, fieldInfo{
|
||||
index: i,
|
||||
@@ -427,76 +406,6 @@ func Set(key string, value any) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// SetMany 将结构体保存为多个配置项
|
||||
func SetManyAs[T any](config T) error {
|
||||
val := reflect.ValueOf(config)
|
||||
if val.Kind() == reflect.Ptr {
|
||||
val = val.Elem()
|
||||
}
|
||||
|
||||
typ := val.Type()
|
||||
var items []ConfigItem
|
||||
|
||||
for i := 0; i < val.NumField(); i++ {
|
||||
fieldType := typ.Field(i)
|
||||
tag := fieldType.Tag.Get("json")
|
||||
|
||||
if tag == "" || tag == "-" {
|
||||
continue
|
||||
}
|
||||
|
||||
fieldValue := val.Field(i).Interface()
|
||||
|
||||
bytes, err := json.Marshal(fieldValue)
|
||||
if err != nil {
|
||||
return fmt.Errorf("marshal field %s failed: %w", fieldType.Name, err)
|
||||
}
|
||||
|
||||
items = append(items, ConfigItem{
|
||||
Key: tag,
|
||||
Value: string(bytes),
|
||||
})
|
||||
}
|
||||
|
||||
if len(items) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
keys := make([]string, 0, len(items))
|
||||
newVal := make(map[string]any, len(items))
|
||||
for _, it := range items {
|
||||
keys = append(keys, it.Key)
|
||||
var parsed any
|
||||
if err := json.Unmarshal([]byte(it.Value), &parsed); err == nil {
|
||||
newVal[it.Key] = parsed
|
||||
}
|
||||
}
|
||||
|
||||
oldVal := map[string]any{}
|
||||
if len(keys) > 0 {
|
||||
var oldItems []ConfigItem
|
||||
if err := db.Where("key IN ?", keys).Find(&oldItems).Error; err == nil {
|
||||
for _, oi := range oldItems {
|
||||
var parsed any
|
||||
if err := json.Unmarshal([]byte(oi.Value), &parsed); err == nil {
|
||||
oldVal[oi.Key] = parsed
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
err := db.Clauses(clause.OnConflict{
|
||||
Columns: []clause.Column{{Name: "key"}},
|
||||
DoUpdates: clause.AssignmentColumns([]string{"value"}),
|
||||
}).Create(&items).Error
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
publishEvent(oldVal, newVal)
|
||||
return nil
|
||||
}
|
||||
|
||||
func SetMany(cst map[string]any) error {
|
||||
var items []ConfigItem
|
||||
for k, v := range cst {
|
||||
|
||||
+29
-39
@@ -16,22 +16,20 @@ type Settings struct {
|
||||
AutoDiscoveryKey string `json:"auto_discovery_key" default:""` // 自动发现密钥
|
||||
ScriptDomain string `json:"script_domain" default:""` // 自定义脚本域名
|
||||
SendIpAddrToGuest bool `json:"send_ip_addr_to_guest" default:"false"` // 是否向访客页面发送 IP 地址,默认 false
|
||||
VisitorAuditEnabled bool `json:"visitor_audit_enabled" default:"false"` // 是否允许公开访客事件写入审计日志,默认 false
|
||||
EulaAccepted bool `json:"eula_accepted" default:"false"`
|
||||
BaseScriptsURLKey string `json:"base_scripts_url" default:""`
|
||||
// GeoIP 配置
|
||||
GeoIpEnabled bool `json:"geo_ip_enabled" default:"true"`
|
||||
GeoIpProvider string `json:"geo_ip_provider" default:"ipinfo"` // empty, mmdb, ip-api, geojs
|
||||
// Nezha 兼容(Agent gRPC)
|
||||
NezhaCompatEnabled bool `json:"nezha_compat_enabled" default:"false"`
|
||||
NezhaCompatListen string `json:"nezha_compat_listen" default:""` // 例如 0.0.0.0:5555
|
||||
// OAuth 配置
|
||||
OAuthEnabled bool `json:"o_auth_enabled" default:"false"`
|
||||
OAuthProvider string `json:"o_auth_provider" default:"github"`
|
||||
DisablePasswordLogin bool `json:"disable_password_login" default:"false"`
|
||||
CloudflareTunnelToken string `json:"cloudflare_tunnel_token" default:""`
|
||||
OAuthEnabled bool `json:"o_auth_enabled" default:"false"`
|
||||
OAuthProvider string `json:"o_auth_provider" default:"github"`
|
||||
DisablePasswordLogin bool `json:"disable_password_login" default:"false"`
|
||||
// 自定义美化
|
||||
CustomHead string `json:"custom_head" default:""`
|
||||
CustomBody string `json:"custom_body" default:""`
|
||||
|
||||
// 通知
|
||||
NotificationEnabled bool `json:"notification_enabled" default:"true"` // 通知总开关
|
||||
NotificationMethod string `json:"notification_method" default:"none"`
|
||||
@@ -40,38 +38,33 @@ type Settings struct {
|
||||
ExpireNotificationLeadDays int `json:"expire_notification_lead_days" default:"7"` // 过期前多少天通知,默认7天
|
||||
LoginNotification bool `json:"login_notification" default:"true"` // 登录通知
|
||||
TrafficLimitPercentage float64 `json:"traffic_limit_percentage" default:"80.00"` // 流量限制百分比,默认80.00%
|
||||
// Record
|
||||
RecordEnabled bool `json:"record_enabled" default:"true"` // 是否启用记录功能
|
||||
RecordPreserveTime int `json:"record_preserve_time" default:"720"` // 记录保留时间,单位小时,默认30天
|
||||
PingRecordPreserveTime int `json:"ping_record_preserve_time" default:"24"` // Ping 记录保留时间,单位小时,默认1天
|
||||
UpdatedAt time.Time
|
||||
UpdatedAt time.Time
|
||||
}
|
||||
|
||||
const (
|
||||
SitenameKey = "sitename"
|
||||
DescriptionKey = "description"
|
||||
CorsOriginCheckEnabledKey = "cors_origin_check_enabled"
|
||||
CorsAllowedOriginsKey = "cors_allowed_origins"
|
||||
WsOriginCheckEnabledKey = "ws_origin_check_enabled"
|
||||
WsAllowedOriginsKey = "ws_allowed_origins"
|
||||
ThemeKey = "theme"
|
||||
PrivateSiteKey = "private_site"
|
||||
ApiKeyKey = "api_key"
|
||||
AutoDiscoveryKeyKey = "auto_discovery_key"
|
||||
ScriptDomainKey = "script_domain"
|
||||
SendIpAddrToGuestKey = "send_ip_addr_to_guest"
|
||||
EulaAcceptedKey = "eula_accepted"
|
||||
BaseScriptsURLKey = "base_scripts_url"
|
||||
GeoIpEnabledKey = "geo_ip_enabled"
|
||||
GeoIpProviderKey = "geo_ip_provider"
|
||||
NezhaCompatEnabledKey = "nezha_compat_enabled"
|
||||
NezhaCompatListenKey = "nezha_compat_listen"
|
||||
OAuthEnabledKey = "o_auth_enabled"
|
||||
OAuthProviderKey = "o_auth_provider"
|
||||
DisablePasswordLoginKey = "disable_password_login"
|
||||
CloudflareTunnelTokenKey = "cloudflare_tunnel_token"
|
||||
CustomHeadKey = "custom_head"
|
||||
CustomBodyKey = "custom_body"
|
||||
SitenameKey = "sitename"
|
||||
DescriptionKey = "description"
|
||||
CorsOriginCheckEnabledKey = "cors_origin_check_enabled"
|
||||
CorsAllowedOriginsKey = "cors_allowed_origins"
|
||||
WsOriginCheckEnabledKey = "ws_origin_check_enabled"
|
||||
WsAllowedOriginsKey = "ws_allowed_origins"
|
||||
ThemeKey = "theme"
|
||||
PrivateSiteKey = "private_site"
|
||||
ApiKeyKey = "api_key"
|
||||
AutoDiscoveryKeyKey = "auto_discovery_key"
|
||||
ScriptDomainKey = "script_domain"
|
||||
SendIpAddrToGuestKey = "send_ip_addr_to_guest"
|
||||
VisitorAuditEnabledKey = "visitor_audit_enabled"
|
||||
EulaAcceptedKey = "eula_accepted"
|
||||
BaseScriptsURLKey = "base_scripts_url"
|
||||
GeoIpEnabledKey = "geo_ip_enabled"
|
||||
GeoIpProviderKey = "geo_ip_provider"
|
||||
OAuthEnabledKey = "o_auth_enabled"
|
||||
OAuthProviderKey = "o_auth_provider"
|
||||
DisablePasswordLoginKey = "disable_password_login"
|
||||
CustomHeadKey = "custom_head"
|
||||
CustomBodyKey = "custom_body"
|
||||
|
||||
NotificationEnabledKey = "notification_enabled"
|
||||
NotificationMethodKey = "notification_method"
|
||||
NotificationTemplateKey = "notification_template"
|
||||
@@ -79,9 +72,6 @@ const (
|
||||
ExpireNotificationLeadDaysKey = "expire_notification_lead_days"
|
||||
LoginNotificationKey = "login_notification"
|
||||
TrafficLimitPercentageKey = "traffic_limit_percentage"
|
||||
RecordEnabledKey = "record_enabled"
|
||||
RecordPreserveTimeKey = "record_preserve_time"
|
||||
PingRecordPreserveTimeKey = "ping_record_preserve_time"
|
||||
UpdatedAtKey = "updated_at"
|
||||
XtermjsSettingsKey = "xtermjs_settings"
|
||||
)
|
||||
+2
-1
@@ -40,7 +40,7 @@ type cronSchedule struct {
|
||||
}
|
||||
|
||||
func (s cronSchedule) Next(t time.Time) time.Time {
|
||||
next := t.Truncate(time.Second).Add(time.Second)
|
||||
next := t.UTC().Truncate(time.Second).Add(time.Second)
|
||||
limit := next.Add(366 * 24 * time.Hour)
|
||||
for next.Before(limit) {
|
||||
if s.match(next) {
|
||||
@@ -52,6 +52,7 @@ func (s cronSchedule) Next(t time.Time) time.Time {
|
||||
}
|
||||
|
||||
func (s cronSchedule) match(t time.Time) bool {
|
||||
t = t.In(time.Local)
|
||||
_, okSecond := s.seconds[t.Second()]
|
||||
_, okMinute := s.minutes[t.Minute()]
|
||||
_, okHour := s.hours[t.Hour()]
|
||||
|
||||
@@ -0,0 +1,35 @@
|
||||
package corn
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestCronScheduleUsesSystemLocalWallClock(t *testing.T) {
|
||||
originalLocal := time.Local
|
||||
time.Local = time.FixedZone("UTC+8", 8*60*60)
|
||||
t.Cleanup(func() { time.Local = originalLocal })
|
||||
|
||||
schedule, err := Parse("0 0 9 * * *")
|
||||
if err != nil {
|
||||
t.Fatalf("parse schedule: %v", err)
|
||||
}
|
||||
after := time.Date(2026, 7, 17, 0, 30, 0, 0, time.UTC)
|
||||
want := time.Date(2026, 7, 17, 1, 0, 0, 0, time.UTC)
|
||||
if got := schedule.Next(after); !got.Equal(want) {
|
||||
t.Fatalf("next run = %s, want %s", got, want)
|
||||
} else if got.Location() != time.UTC {
|
||||
t.Fatalf("next run location = %s, want UTC", got.Location())
|
||||
}
|
||||
}
|
||||
|
||||
func TestEverySchedulePreservesElapsedDuration(t *testing.T) {
|
||||
schedule, err := Parse("@every 90s")
|
||||
if err != nil {
|
||||
t.Fatalf("parse schedule: %v", err)
|
||||
}
|
||||
after := time.Now()
|
||||
if got := schedule.Next(after); got.Sub(after) != 90*time.Second {
|
||||
t.Fatalf("interval = %s, want 90s", got.Sub(after))
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,305 @@
|
||||
package metric
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"math"
|
||||
"sort"
|
||||
"time"
|
||||
)
|
||||
|
||||
type aggregateSeriesKey struct {
|
||||
metricName string
|
||||
entityID string
|
||||
tagsHash string
|
||||
}
|
||||
|
||||
type aggregateGroupKey struct {
|
||||
aggregateSeriesKey
|
||||
bucket int64
|
||||
}
|
||||
|
||||
// AggregatePoints groups raw points into time buckets and computes aggregate values.
|
||||
//
|
||||
// AggregatePoints 在内存中按查询配置将原始点分桶并聚合。
|
||||
func AggregatePoints(points []Point, query AggregateQuery) ([]AggregatePoint, error) {
|
||||
if err := query.Validate(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if len(points) == 0 {
|
||||
return []AggregatePoint{}, nil
|
||||
}
|
||||
|
||||
points = sortedPoints(points)
|
||||
groups := make(map[aggregateGroupKey][]Point)
|
||||
seriesTags := make(map[aggregateSeriesKey]map[string]string)
|
||||
for _, point := range points {
|
||||
point = point.normalized()
|
||||
if point.MetricName == "" {
|
||||
point.MetricName = query.MetricName
|
||||
}
|
||||
if point.EntityID == "" {
|
||||
point.EntityID = query.EntityID
|
||||
}
|
||||
tagsHash, _, err := tagsFingerprint(point.Tags)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
bucket := alignTime(point.Timestamp, query.Interval).UnixNano()
|
||||
series := aggregateSeriesKey{
|
||||
metricName: point.MetricName,
|
||||
entityID: point.EntityID,
|
||||
tagsHash: tagsHash,
|
||||
}
|
||||
key := aggregateGroupKey{aggregateSeriesKey: series, bucket: bucket}
|
||||
groups[key] = append(groups[key], point)
|
||||
if _, ok := seriesTags[series]; !ok {
|
||||
seriesTags[series] = cloneStringMap(point.Tags)
|
||||
}
|
||||
}
|
||||
|
||||
keys := make([]aggregateGroupKey, 0, len(groups))
|
||||
for key := range groups {
|
||||
keys = append(keys, key)
|
||||
}
|
||||
sortAggregateGroupKeys(keys)
|
||||
|
||||
out := make([]AggregatePoint, 0, len(keys))
|
||||
for _, key := range keys {
|
||||
group := groups[key]
|
||||
value, err := aggregateValue(group, query.Aggregation)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out = append(out, AggregatePoint{
|
||||
MetricName: key.metricName,
|
||||
EntityID: key.entityID,
|
||||
Bucket: time.Unix(0, key.bucket).UTC(),
|
||||
Value: value,
|
||||
Count: len(group),
|
||||
Tags: cloneStringMap(seriesTags[key.aggregateSeriesKey]),
|
||||
})
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func sortAggregateGroupKeys(keys []aggregateGroupKey) {
|
||||
sort.Slice(keys, func(i, j int) bool {
|
||||
if keys[i].bucket != keys[j].bucket {
|
||||
return keys[i].bucket < keys[j].bucket
|
||||
}
|
||||
if keys[i].metricName != keys[j].metricName {
|
||||
return keys[i].metricName < keys[j].metricName
|
||||
}
|
||||
if keys[i].entityID != keys[j].entityID {
|
||||
return keys[i].entityID < keys[j].entityID
|
||||
}
|
||||
return keys[i].tagsHash < keys[j].tagsHash
|
||||
})
|
||||
}
|
||||
|
||||
func cloneStringMap(in map[string]string) map[string]string {
|
||||
if len(in) == 0 {
|
||||
return map[string]string{}
|
||||
}
|
||||
out := make(map[string]string, len(in))
|
||||
for k, v := range in {
|
||||
out[k] = v
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// CalculateStats computes summary statistics for a point series.
|
||||
//
|
||||
// CalculateStats 基于一组点计算统计摘要,包括均值、百分位、首尾值和标准差。
|
||||
func CalculateStats(points []Point) (Stats, error) {
|
||||
if len(points) == 0 {
|
||||
// Distinguish "the metric/range yielded no samples" from "the metric
|
||||
// definition does not exist" (which surfaces as ErrNotFound elsewhere).
|
||||
return Stats{}, ErrNoData
|
||||
}
|
||||
points = sortedPoints(points)
|
||||
avg, _ := aggregateValue(points, AggAvg)
|
||||
min, _ := aggregateValue(points, AggMin)
|
||||
max, _ := aggregateValue(points, AggMax)
|
||||
sum, _ := aggregateValue(points, AggSum)
|
||||
p50, _ := aggregateValue(points, AggP50)
|
||||
p95, _ := aggregateValue(points, AggP95)
|
||||
p99, _ := aggregateValue(points, AggP99)
|
||||
first, _ := aggregateValue(points, AggFirst)
|
||||
last, _ := aggregateValue(points, AggLast)
|
||||
rate, _ := aggregateValue(points, AggRate)
|
||||
|
||||
var variance float64
|
||||
for _, p := range points {
|
||||
diff := p.Value - avg
|
||||
variance += diff * diff
|
||||
}
|
||||
|
||||
return Stats{
|
||||
Count: len(points),
|
||||
Min: min,
|
||||
Max: max,
|
||||
Avg: avg,
|
||||
Sum: sum,
|
||||
P50: p50,
|
||||
P95: p95,
|
||||
P99: p99,
|
||||
First: first,
|
||||
Last: last,
|
||||
Rate: rate,
|
||||
Start: points[0].Timestamp,
|
||||
End: points[len(points)-1].Timestamp,
|
||||
StdDev: math.Sqrt(variance / float64(len(points))),
|
||||
}, nil
|
||||
}
|
||||
|
||||
// aggregateValue computes one aggregation over a point group.
|
||||
//
|
||||
// aggregateValue 对一组点执行单个聚合类型的计算。
|
||||
func aggregateValue(points []Point, agg Aggregation) (float64, error) {
|
||||
if len(points) == 0 {
|
||||
return 0, nil
|
||||
}
|
||||
switch agg {
|
||||
case AggAvg:
|
||||
sum, _ := aggregateValue(points, AggSum)
|
||||
return sum / float64(len(points)), nil
|
||||
case AggMin:
|
||||
v := points[0].Value
|
||||
for _, p := range points[1:] {
|
||||
v = math.Min(v, p.Value)
|
||||
}
|
||||
return v, nil
|
||||
case AggMax:
|
||||
v := points[0].Value
|
||||
for _, p := range points[1:] {
|
||||
v = math.Max(v, p.Value)
|
||||
}
|
||||
return v, nil
|
||||
case AggSum:
|
||||
var sum float64
|
||||
for _, p := range points {
|
||||
sum += p.Value
|
||||
}
|
||||
return sum, nil
|
||||
case AggCount:
|
||||
return float64(len(points)), nil
|
||||
case AggFirst:
|
||||
// Callers (AggregatePoints, CalculateStats) pass time-ordered slices.
|
||||
return points[0].Value, nil
|
||||
case AggLast:
|
||||
return points[len(points)-1].Value, nil
|
||||
case AggRate:
|
||||
return counterRate(points), nil
|
||||
case AggStdDev:
|
||||
return stdDevPop(points), nil
|
||||
default:
|
||||
// Any percentile (p50, p95, p99, and arbitrary pXX / pXX.X) is computed
|
||||
// here via linear interpolation over the sorted values. The fixed
|
||||
// AggP50/AggP95/AggP99 constants are just common cases of this.
|
||||
if frac, ok := parsePercentile(agg); ok {
|
||||
return percentile(points, frac), nil
|
||||
}
|
||||
return 0, fmt.Errorf("%w: unsupported aggregation %q", ErrInvalidArgument, agg)
|
||||
}
|
||||
}
|
||||
|
||||
// counterRate computes a per-second rate of change that is resilient to counter
|
||||
// resets. It walks the time-ordered series and sums only positive deltas; a
|
||||
// decrease is treated as a reset (the counter restarted) and contributes zero
|
||||
// rather than a negative spike. For a strictly increasing counter this equals
|
||||
// (last-first)/seconds. For a gauge it yields the total upward movement per
|
||||
// second, which is a stable definition for an otherwise ill-defined quantity.
|
||||
//
|
||||
// counterRate 计算能抵抗计数器重置的每秒变化率。它会遍历按时间排序的序列,
|
||||
// 并且只累加正向增量;当值下降时会被视为重置(计数器重新开始),该段贡献为零,
|
||||
// 而不是产生负向尖峰。对于严格递增的计数器,这等于 (last-first)/seconds。
|
||||
// 对于 gauge,它表示每秒总上升量,为这种本来定义不明确的量提供稳定定义。
|
||||
func counterRate(points []Point) float64 {
|
||||
if len(points) < 2 {
|
||||
return 0
|
||||
}
|
||||
// Input is time-ordered by the callers; avoid an extra sort here.
|
||||
seconds := points[len(points)-1].Timestamp.Sub(points[0].Timestamp).Seconds()
|
||||
if seconds <= 0 {
|
||||
return 0
|
||||
}
|
||||
var increase float64
|
||||
for i := 1; i < len(points); i++ {
|
||||
delta := points[i].Value - points[i-1].Value
|
||||
if delta > 0 {
|
||||
increase += delta
|
||||
}
|
||||
}
|
||||
return increase / seconds
|
||||
}
|
||||
|
||||
// stdDevPop computes the population standard deviation (dividing by N), matching
|
||||
// SQL STDDEV_POP and the StdDev field produced by CalculateStats.
|
||||
//
|
||||
// stdDevPop 计算总体标准差(除以 N),与 SQL 的 STDDEV_POP 以及
|
||||
// CalculateStats 返回的 StdDev 语义保持一致。
|
||||
func stdDevPop(points []Point) float64 {
|
||||
if len(points) == 0 {
|
||||
return 0
|
||||
}
|
||||
var sum float64
|
||||
for _, p := range points {
|
||||
sum += p.Value
|
||||
}
|
||||
mean := sum / float64(len(points))
|
||||
var variance float64
|
||||
for _, p := range points {
|
||||
d := p.Value - mean
|
||||
variance += d * d
|
||||
}
|
||||
return math.Sqrt(variance / float64(len(points)))
|
||||
}
|
||||
|
||||
// percentile computes a quantile from point values.
|
||||
//
|
||||
// percentile 提取点值、排序,并计算指定小数形式的百分位。
|
||||
func percentile(points []Point, p float64) float64 {
|
||||
values := make([]float64, len(points))
|
||||
for i, point := range points {
|
||||
values[i] = point.Value
|
||||
}
|
||||
sort.Float64s(values)
|
||||
return percentileSorted(values, p)
|
||||
}
|
||||
|
||||
// percentileSorted returns the linear-interpolation percentile of an
|
||||
// already-sorted slice. Shared by percentile() and the raw-value paths so the
|
||||
// interpolation method stays identical everywhere.
|
||||
//
|
||||
// percentileSorted 基于已排序切片用线性插值计算百分位,供原始值路径和
|
||||
// percentile 共用,确保所有路径的插值方法一致。
|
||||
func percentileSorted(values []float64, p float64) float64 {
|
||||
if len(values) == 0 {
|
||||
return 0
|
||||
}
|
||||
if len(values) == 1 {
|
||||
return values[0]
|
||||
}
|
||||
pos := p * float64(len(values)-1)
|
||||
lower := int(math.Floor(pos))
|
||||
upper := int(math.Ceil(pos))
|
||||
if lower == upper {
|
||||
return values[lower]
|
||||
}
|
||||
weight := pos - float64(lower)
|
||||
return values[lower]*(1-weight) + values[upper]*weight
|
||||
}
|
||||
|
||||
// alignTime floors a timestamp to the start of its interval bucket.
|
||||
//
|
||||
// alignTime 将时间向下对齐到指定间隔的桶起点。
|
||||
func alignTime(t time.Time, interval time.Duration) time.Time {
|
||||
nano := t.UTC().UnixNano()
|
||||
size := interval.Nanoseconds()
|
||||
// Floor division toward negative infinity so timestamps before the Unix
|
||||
// epoch (negative nanos) align to the bucket start rather than rounding up.
|
||||
// Go's % returns a remainder with the dividend's sign, so normalize it.
|
||||
rem := ((nano % size) + size) % size
|
||||
return time.Unix(0, nano-rem).UTC()
|
||||
}
|
||||
@@ -0,0 +1,66 @@
|
||||
package metric
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"errors"
|
||||
"fmt"
|
||||
"time"
|
||||
)
|
||||
|
||||
// compactionWatermark returns the last boundary for which a metric's raw data
|
||||
// was successfully materialized into rollups and retained/deleted in the same
|
||||
// transaction. A missing row is expected for stores created before watermark
|
||||
// tracking was introduced or for metrics that have never been compacted.
|
||||
func (s *Store) compactionWatermark(ctx context.Context, metricName string) (time.Time, bool, error) {
|
||||
var nano int64
|
||||
err := s.reader().QueryRowContext(ctx, fmt.Sprintf(
|
||||
`SELECT watermark_nano FROM %s WHERE metric_name = %s`,
|
||||
s.tables.watermarks, s.dialect.placeholder(1),
|
||||
), metricName).Scan(&nano)
|
||||
if errors.Is(err, sql.ErrNoRows) {
|
||||
return time.Time{}, false, nil
|
||||
}
|
||||
if err != nil {
|
||||
return time.Time{}, false, err
|
||||
}
|
||||
if nano <= 0 {
|
||||
return time.Time{}, false, nil
|
||||
}
|
||||
return time.Unix(0, nano).UTC(), true, nil
|
||||
}
|
||||
|
||||
// persistCompactionWatermarkTx records a successful compaction boundary in the
|
||||
// same transaction as rollup writes and raw retention deletion. It is
|
||||
// monotonic so a delayed caller cannot move the read boundary backwards.
|
||||
func (s *Store) persistCompactionWatermarkTx(ctx context.Context, metricName string, watermark time.Time, tx *sql.Tx) error {
|
||||
watermark = watermark.UTC()
|
||||
if watermark.IsZero() {
|
||||
return s.clearCompactionWatermarkTx(ctx, metricName, tx)
|
||||
}
|
||||
|
||||
var previous int64
|
||||
err := tx.QueryRowContext(ctx, fmt.Sprintf(
|
||||
`SELECT watermark_nano FROM %s WHERE metric_name = %s`,
|
||||
s.tables.watermarks, s.dialect.placeholder(1),
|
||||
), metricName).Scan(&previous)
|
||||
if err != nil && !errors.Is(err, sql.ErrNoRows) {
|
||||
return err
|
||||
}
|
||||
if err == nil && previous > watermark.UnixNano() {
|
||||
watermark = time.Unix(0, previous).UTC()
|
||||
}
|
||||
|
||||
_, err = tx.ExecContext(ctx, s.dialect.upsertCompactionWatermarkSQL(s.tables),
|
||||
metricName, watermark.UnixNano(), time.Now().UTC().UnixNano(),
|
||||
)
|
||||
return err
|
||||
}
|
||||
|
||||
func (s *Store) clearCompactionWatermarkTx(ctx context.Context, metricName string, tx *sql.Tx) error {
|
||||
_, err := tx.ExecContext(ctx, fmt.Sprintf(
|
||||
`DELETE FROM %s WHERE metric_name = %s`,
|
||||
s.tables.watermarks, s.dialect.placeholder(1),
|
||||
), metricName)
|
||||
return err
|
||||
}
|
||||
@@ -0,0 +1,468 @@
|
||||
package metric
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"fmt"
|
||||
"net/url"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Config describes database, pooling, migration, retention, and rollup settings.
|
||||
//
|
||||
// Config 描述 Store 的数据库后端、连接池、迁移、保留和 rollup 配置。
|
||||
type Config struct {
|
||||
// Driver selects the database backend.
|
||||
//
|
||||
// Driver 选择数据库后端。
|
||||
Driver Driver
|
||||
// DSN is the database connection string when DB is not supplied.
|
||||
//
|
||||
// DSN 是未传入 DB 时使用的数据库连接字符串。
|
||||
DSN string
|
||||
|
||||
// DB can be supplied by the host app when it owns the connection pool.
|
||||
// When DB is set, DSN is ignored and Close leaves the supplied DB open.
|
||||
//
|
||||
// 当宿主程序自己管理连接池时可以传入 DB;设置 DB 后会忽略 DSN,
|
||||
// Close 也不会关闭这个外部传入的连接池。
|
||||
DB *sql.DB
|
||||
|
||||
// TablePrefix prefixes every table managed by the store.
|
||||
//
|
||||
// TablePrefix 是 Store 管理的所有表名前缀。
|
||||
TablePrefix string
|
||||
// AutoMigrate controls whether Open creates or updates the schema.
|
||||
//
|
||||
// AutoMigrate 控制 Open 是否创建或更新表结构。
|
||||
AutoMigrate bool
|
||||
// MaxOpenConns sets the primary pool's maximum open connections.
|
||||
//
|
||||
// MaxOpenConns 设置主连接池的最大打开连接数。
|
||||
MaxOpenConns int
|
||||
// MaxIdleConns sets the primary pool's maximum idle connections.
|
||||
//
|
||||
// MaxIdleConns 设置主连接池的最大空闲连接数。
|
||||
MaxIdleConns int
|
||||
// ConnMaxLifetime sets the maximum lifetime for pooled connections.
|
||||
//
|
||||
// ConnMaxLifetime 设置池化连接的最大生命周期。
|
||||
ConnMaxLifetime time.Duration
|
||||
// ConnectTimeout bounds the initial database ping in Open.
|
||||
//
|
||||
// ConnectTimeout 限制 Open 中首次 ping 数据库的时间。
|
||||
ConnectTimeout time.Duration
|
||||
// SQLite holds SQLite-specific options.
|
||||
//
|
||||
// SQLite 保存 SQLite 专用选项。
|
||||
SQLite SQLiteOptions
|
||||
|
||||
// RollupPolicy configures downsampling tiers and tiered retention. When it
|
||||
// defines tiers, Store.Compact materializes them and enforces every
|
||||
// retention window. The zero value disables rollups (raw points only).
|
||||
//
|
||||
// RollupPolicy 配置降采样层级和分层保留时间;定义层级后,
|
||||
// Store.Compact 会生成 rollup 并执行保留策略。零值表示禁用 rollup。
|
||||
RollupPolicy RollupPolicy
|
||||
}
|
||||
|
||||
// SQLiteOptions contains SQLite-specific performance and read-concurrency settings.
|
||||
//
|
||||
// SQLiteOptions 保存 SQLite 专用的性能和并发读取选项。
|
||||
type SQLiteOptions struct {
|
||||
// PerformanceProfile selects the SQLite synchronous durability preset.
|
||||
//
|
||||
// PerformanceProfile 选择 SQLite synchronous 持久化预设。
|
||||
PerformanceProfile SQLitePerformanceProfile
|
||||
// BusyTimeout is applied to SQLite busy_timeout.
|
||||
//
|
||||
// BusyTimeout 会应用到 SQLite busy_timeout。
|
||||
BusyTimeout time.Duration
|
||||
// CacheSizeKB sets the SQLite page cache size in KB.
|
||||
//
|
||||
// CacheSizeKB 设置 SQLite 页缓存大小,单位为 KB。
|
||||
CacheSizeKB int
|
||||
// PageSize sets SQLite page_size when positive.
|
||||
//
|
||||
// PageSize 为正数时设置 SQLite page_size。
|
||||
PageSize int
|
||||
// TempStoreMemory enables memory-backed temporary storage.
|
||||
//
|
||||
// TempStoreMemory 启用基于内存的临时存储。
|
||||
TempStoreMemory bool
|
||||
// MMapSizeBytes sets SQLite mmap_size in bytes.
|
||||
//
|
||||
// MMapSizeBytes 设置 SQLite mmap_size,单位为字节。
|
||||
MMapSizeBytes int64
|
||||
// WALAutoCheckpoint sets the SQLite WAL auto-checkpoint page count.
|
||||
//
|
||||
// WALAutoCheckpoint 设置 SQLite WAL 自动 checkpoint 页数。
|
||||
WALAutoCheckpoint int
|
||||
// ReadPoolSize, when > 0, opens a second read-only connection pool with
|
||||
// this many connections for SELECT-style calls. SQLite serializes writes,
|
||||
// but WAL lets readers run concurrently, so a read pool lifts read
|
||||
// throughput while keeping writes on the single primary connection.
|
||||
// 0 (default) means all calls share the single primary pool, preserving
|
||||
// the previous single-connection behavior.
|
||||
//
|
||||
// ReadPoolSize 大于 0 时会为 SELECT 类调用打开独立只读连接池。
|
||||
// SQLite 写入会被串行化,但 WAL 允许读取并发执行,因此读池可以在写入仍走
|
||||
// 单主连接的同时提升读取吞吐。0(默认值)表示所有调用共享单个主连接池,
|
||||
// 保留之前的单连接行为。
|
||||
ReadPoolSize int
|
||||
}
|
||||
|
||||
// SQLitePerformanceProfile names SQLite durability and performance presets.
|
||||
//
|
||||
// SQLitePerformanceProfile 表示 SQLite synchronous 等持久化策略预设。
|
||||
type SQLitePerformanceProfile string
|
||||
|
||||
const (
|
||||
// SQLiteProfileDefault uses the package default SQLite profile.
|
||||
//
|
||||
// SQLiteProfileDefault 使用包默认的 SQLite 配置档。
|
||||
SQLiteProfileDefault SQLitePerformanceProfile = ""
|
||||
// SQLiteProfileBalanced uses NORMAL synchronous mode.
|
||||
//
|
||||
// SQLiteProfileBalanced 使用 NORMAL synchronous 模式。
|
||||
SQLiteProfileBalanced SQLitePerformanceProfile = "balanced"
|
||||
// SQLiteProfilePerformance favors throughput over durability.
|
||||
//
|
||||
// SQLiteProfilePerformance 优先吞吐而非持久性。
|
||||
SQLiteProfilePerformance SQLitePerformanceProfile = "performance"
|
||||
// SQLiteProfileDurable favors durability over write throughput.
|
||||
//
|
||||
// SQLiteProfileDurable 优先持久性而非写入吞吐。
|
||||
SQLiteProfileDurable SQLitePerformanceProfile = "durable"
|
||||
)
|
||||
|
||||
// Option mutates a Config value.
|
||||
//
|
||||
// Option 是用于修改 Config 的函数式选项。
|
||||
type Option func(*Config)
|
||||
|
||||
// DefaultConfig returns the default configuration for a backend.
|
||||
//
|
||||
// DefaultConfig 返回指定后端的默认配置。
|
||||
func DefaultConfig(driver Driver, dsn string) Config {
|
||||
return Config{
|
||||
Driver: driver,
|
||||
DSN: dsn,
|
||||
TablePrefix: "metric_",
|
||||
AutoMigrate: true,
|
||||
MaxOpenConns: 25,
|
||||
MaxIdleConns: 5,
|
||||
ConnMaxLifetime: time.Hour,
|
||||
ConnectTimeout: 10 * time.Second,
|
||||
SQLite: SQLiteOptions{
|
||||
PerformanceProfile: SQLiteProfileBalanced,
|
||||
BusyTimeout: 5 * time.Second,
|
||||
CacheSizeKB: 64 * 1024,
|
||||
TempStoreMemory: true,
|
||||
MMapSizeBytes: 256 * 1024 * 1024,
|
||||
WALAutoCheckpoint: 1000,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
// Backend builds a generic backend configuration and applies options.
|
||||
//
|
||||
// Backend 构造通用后端配置,并应用额外选项。
|
||||
func Backend(driver Driver, dsn string, opts ...Option) Config {
|
||||
cfg := DefaultConfig(driver, dsn)
|
||||
applyOptions(&cfg, opts...)
|
||||
return cfg
|
||||
}
|
||||
|
||||
// SQLite builds a SQLite backend configuration.
|
||||
//
|
||||
// SQLite 构造 SQLite 后端配置。
|
||||
func SQLite(dsn string, opts ...Option) Config {
|
||||
cfg := SQLiteConfig(dsn)
|
||||
applyOptions(&cfg, opts...)
|
||||
return cfg
|
||||
}
|
||||
|
||||
// SQLiteInDir builds a managed SQLite file configuration rooted at a directory.
|
||||
//
|
||||
// SQLiteInDir 构造由 package 管理目录的 SQLite 文件数据库配置。
|
||||
func SQLiteInDir(dir string, opts ...Option) Config {
|
||||
cfg := SQLiteConfig(sqliteFileDSN(filepath.Join(dir, "metrics.db")))
|
||||
cfg.MaxOpenConns = 1
|
||||
cfg.MaxIdleConns = 1
|
||||
applyOptions(&cfg, opts...)
|
||||
return cfg
|
||||
}
|
||||
|
||||
// MySQL builds a MySQL backend configuration.
|
||||
//
|
||||
// MySQL 构造 MySQL 后端配置。
|
||||
func MySQL(dsn string, opts ...Option) Config {
|
||||
cfg := MySQLConfig(dsn)
|
||||
applyOptions(&cfg, opts...)
|
||||
return cfg
|
||||
}
|
||||
|
||||
// PostgreSQL builds a PostgreSQL backend configuration.
|
||||
//
|
||||
// PostgreSQL 构造 PostgreSQL 后端配置。
|
||||
func PostgreSQL(dsn string, opts ...Option) Config {
|
||||
cfg := PostgreSQLConfig(dsn)
|
||||
applyOptions(&cfg, opts...)
|
||||
return cfg
|
||||
}
|
||||
|
||||
// SQLiteConfig returns the default SQLite configuration.
|
||||
//
|
||||
// SQLiteConfig 返回 SQLite 后端的默认配置。
|
||||
func SQLiteConfig(dsn string) Config {
|
||||
cfg := DefaultConfig(DriverSQLite, dsn)
|
||||
if cfg.DSN == "" {
|
||||
cfg.DSN = "file:metric?mode=memory&cache=shared"
|
||||
}
|
||||
// SQLite serializes writes, so a single connection is the safe default and
|
||||
// avoids "database is locked" contention. Callers can still override with
|
||||
// WithMaxOpenConns/WithMaxIdleConns. Setting it here (rather than inferring
|
||||
// it later from the 25/5 defaults) means an explicit WithMaxOpenConns(25)
|
||||
// is honored instead of being mistaken for "unset".
|
||||
cfg.MaxOpenConns = 1
|
||||
cfg.MaxIdleConns = 1
|
||||
return cfg
|
||||
}
|
||||
|
||||
// MySQLConfig returns the default MySQL configuration.
|
||||
//
|
||||
// MySQLConfig 返回 MySQL 后端的默认配置。
|
||||
func MySQLConfig(dsn string) Config {
|
||||
return DefaultConfig(DriverMySQL, dsn)
|
||||
}
|
||||
|
||||
// PostgreSQLConfig returns the default PostgreSQL configuration.
|
||||
//
|
||||
// PostgreSQLConfig 返回 PostgreSQL 后端的默认配置。
|
||||
func PostgreSQLConfig(dsn string) Config {
|
||||
return DefaultConfig(DriverPostgreSQL, dsn)
|
||||
}
|
||||
|
||||
// WithDB sets a caller-owned database connection pool.
|
||||
//
|
||||
// WithDB 设置由调用方提供的数据库连接池。
|
||||
func WithDB(db *sql.DB) Option {
|
||||
return func(c *Config) {
|
||||
c.DB = db
|
||||
}
|
||||
}
|
||||
|
||||
// WithTablePrefix sets the table-name prefix used by metric tables.
|
||||
//
|
||||
// WithTablePrefix 设置 metric 私有表的表名前缀。
|
||||
func WithTablePrefix(prefix string) Option {
|
||||
return func(c *Config) {
|
||||
c.TablePrefix = prefix
|
||||
}
|
||||
}
|
||||
|
||||
// WithAutoMigrate controls whether Open creates the schema automatically.
|
||||
//
|
||||
// WithAutoMigrate 控制 Open 时是否自动创建表结构。
|
||||
func WithAutoMigrate(enabled bool) Option {
|
||||
return func(c *Config) {
|
||||
c.AutoMigrate = enabled
|
||||
}
|
||||
}
|
||||
|
||||
// WithMaxOpenConns sets the maximum number of open database connections.
|
||||
//
|
||||
// WithMaxOpenConns 设置底层数据库连接池的最大打开连接数。
|
||||
func WithMaxOpenConns(n int) Option {
|
||||
return func(c *Config) {
|
||||
c.MaxOpenConns = n
|
||||
}
|
||||
}
|
||||
|
||||
// WithMaxIdleConns sets the maximum number of idle database connections.
|
||||
//
|
||||
// WithMaxIdleConns 设置底层数据库连接池的最大空闲连接数。
|
||||
func WithMaxIdleConns(n int) Option {
|
||||
return func(c *Config) {
|
||||
c.MaxIdleConns = n
|
||||
}
|
||||
}
|
||||
|
||||
// WithConnMaxLifetime sets the maximum lifetime for pooled connections.
|
||||
//
|
||||
// WithConnMaxLifetime 设置连接最大复用时间。
|
||||
func WithConnMaxLifetime(d time.Duration) Option {
|
||||
return func(c *Config) {
|
||||
c.ConnMaxLifetime = d
|
||||
}
|
||||
}
|
||||
|
||||
// WithConnectTimeout sets the timeout used while opening the store.
|
||||
//
|
||||
// WithConnectTimeout 设置打开 Store 时 ping 数据库的超时时间。
|
||||
func WithConnectTimeout(d time.Duration) Option {
|
||||
return func(c *Config) {
|
||||
c.ConnectTimeout = d
|
||||
}
|
||||
}
|
||||
|
||||
// WithSQLiteProfile sets the SQLite durability and performance profile.
|
||||
//
|
||||
// WithSQLiteProfile 设置 SQLite 持久化性能预设。
|
||||
func WithSQLiteProfile(profile SQLitePerformanceProfile) Option {
|
||||
return func(c *Config) {
|
||||
c.SQLite.PerformanceProfile = profile
|
||||
}
|
||||
}
|
||||
|
||||
// WithSQLiteBusyTimeout sets SQLite busy_timeout.
|
||||
//
|
||||
// WithSQLiteBusyTimeout 设置 SQLite busy_timeout。
|
||||
func WithSQLiteBusyTimeout(d time.Duration) Option {
|
||||
return func(c *Config) {
|
||||
c.SQLite.BusyTimeout = d
|
||||
}
|
||||
}
|
||||
|
||||
// WithSQLiteCacheSizeKB sets the SQLite page cache size in KB.
|
||||
//
|
||||
// WithSQLiteCacheSizeKB 设置 SQLite 页缓存大小,单位为 KB。
|
||||
func WithSQLiteCacheSizeKB(kb int) Option {
|
||||
return func(c *Config) {
|
||||
c.SQLite.CacheSizeKB = kb
|
||||
}
|
||||
}
|
||||
|
||||
// WithSQLiteMMapSize sets SQLite mmap_size.
|
||||
//
|
||||
// WithSQLiteMMapSize 设置 SQLite mmap_size。
|
||||
func WithSQLiteMMapSize(bytes int64) Option {
|
||||
return func(c *Config) {
|
||||
c.SQLite.MMapSizeBytes = bytes
|
||||
}
|
||||
}
|
||||
|
||||
// WithSQLitePageSize sets SQLite page_size.
|
||||
//
|
||||
// WithSQLitePageSize 设置 SQLite page_size。
|
||||
func WithSQLitePageSize(bytes int) Option {
|
||||
return func(c *Config) {
|
||||
c.SQLite.PageSize = bytes
|
||||
}
|
||||
}
|
||||
|
||||
// WithSQLiteTempStoreMemory controls whether SQLite uses memory for temporary storage.
|
||||
//
|
||||
// WithSQLiteTempStoreMemory 设置 SQLite 是否使用内存临时存储。
|
||||
func WithSQLiteTempStoreMemory(enabled bool) Option {
|
||||
return func(c *Config) {
|
||||
c.SQLite.TempStoreMemory = enabled
|
||||
}
|
||||
}
|
||||
|
||||
// WithSQLiteWALAutoCheckpoint sets the SQLite WAL auto-checkpoint page count.
|
||||
//
|
||||
// WithSQLiteWALAutoCheckpoint 设置 SQLite WAL 自动 checkpoint 页数。
|
||||
func WithSQLiteWALAutoCheckpoint(pages int) Option {
|
||||
return func(c *Config) {
|
||||
c.SQLite.WALAutoCheckpoint = pages
|
||||
}
|
||||
}
|
||||
|
||||
// WithSQLiteReadPool enables a dedicated read-only connection pool of n
|
||||
// connections for SQLite. Writes stay on the single primary connection
|
||||
// (SQLite serializes them); reads fan out across the pool, which WAL mode
|
||||
// allows to run concurrently. Pass n <= 1 to disable (the default).
|
||||
//
|
||||
// WithSQLiteReadPool 为 SQLite 启用包含 n 个连接的独立只读连接池。写入仍走
|
||||
// 单主连接(SQLite 会串行化写入);读取会分散到该连接池中,WAL 模式允许它们
|
||||
// 并发执行。传入 n <= 1 会禁用该功能(默认行为)。
|
||||
func WithSQLiteReadPool(n int) Option {
|
||||
return func(c *Config) {
|
||||
c.SQLite.ReadPoolSize = n
|
||||
}
|
||||
}
|
||||
|
||||
// WithRollupPolicy sets the downsampling/retention ladder. Compact uses it to
|
||||
// build rollup tiers (each progressively coarser and longer-lived) and to age
|
||||
// out raw points and expired tiers.
|
||||
//
|
||||
// WithRollupPolicy 设置降采样与保留时间阶梯;Compact 会据此构建逐级
|
||||
// 更粗、保留更久的 rollup,并清理过期原始点和过期层级。
|
||||
func WithRollupPolicy(p RollupPolicy) Option {
|
||||
return func(c *Config) {
|
||||
c.RollupPolicy = p
|
||||
}
|
||||
}
|
||||
|
||||
// applyOptions applies configuration options in order.
|
||||
//
|
||||
// applyOptions 将一组选项依次应用到 Config。
|
||||
func applyOptions(cfg *Config, opts ...Option) {
|
||||
for _, opt := range opts {
|
||||
if opt != nil {
|
||||
opt(cfg)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Validate checks whether the value is well formed.
|
||||
//
|
||||
// Validate 检查 Config 的后端、表名前缀、保留时间和 rollup 策略是否合法。
|
||||
func (c Config) Validate() error {
|
||||
switch c.Driver {
|
||||
case DriverSQLite, DriverMySQL, DriverPostgreSQL:
|
||||
default:
|
||||
return fmt.Errorf("%w: unsupported driver %q", ErrInvalidArgument, c.Driver)
|
||||
}
|
||||
if c.DB == nil && strings.TrimSpace(c.DSN) == "" {
|
||||
return fmt.Errorf("%w: dsn is required when db is not supplied", ErrInvalidArgument)
|
||||
}
|
||||
if c.TablePrefix == "" {
|
||||
return fmt.Errorf("%w: table prefix cannot be empty", ErrInvalidArgument)
|
||||
}
|
||||
for _, r := range c.TablePrefix {
|
||||
if !(r == '_' || r >= '0' && r <= '9' || r >= 'A' && r <= 'Z' || r >= 'a' && r <= 'z') {
|
||||
return fmt.Errorf("%w: table prefix must contain only letters, digits, and underscores", ErrInvalidArgument)
|
||||
}
|
||||
}
|
||||
if err := c.RollupPolicy.Validate(); err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// driverName returns the database/sql driver name for the configured backend.
|
||||
//
|
||||
// driverName 返回 database/sql 注册使用的驱动名称。
|
||||
func (c Config) driverName() string {
|
||||
switch c.Driver {
|
||||
case DriverSQLite:
|
||||
return "sqlite3"
|
||||
case DriverMySQL:
|
||||
return "mysql"
|
||||
case DriverPostgreSQL:
|
||||
return "pgx"
|
||||
default:
|
||||
return string(c.Driver)
|
||||
}
|
||||
}
|
||||
|
||||
// sqliteFileDSN converts a filesystem path into a SQLite file DSN.
|
||||
//
|
||||
// sqliteFileDSN 将文件路径转换为 SQLite file: DSN。
|
||||
func sqliteFileDSN(path string) string {
|
||||
return "file:" + filepath.ToSlash(path) + "?cache=shared&mode=rwc"
|
||||
}
|
||||
|
||||
// appendSQLiteDSNParam appends a query parameter to a SQLite DSN.
|
||||
//
|
||||
// appendSQLiteDSNParam 向 SQLite DSN 追加查询参数。
|
||||
func appendSQLiteDSNParam(dsn, key, value string) string {
|
||||
if strings.Contains(dsn, "?") {
|
||||
return dsn + "&" + url.QueryEscape(key) + "=" + url.QueryEscape(value)
|
||||
}
|
||||
return dsn + "?" + url.QueryEscape(key) + "=" + url.QueryEscape(value)
|
||||
}
|
||||
@@ -0,0 +1,64 @@
|
||||
package metric
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// TestBackendBuildersApplyOptions verifies backend builders apply options.
|
||||
//
|
||||
// TestBackendBuildersApplyOptions 验证各后端构造器会正确应用选项。
|
||||
func TestBackendBuildersApplyOptions(t *testing.T) {
|
||||
cfg := SQLite(
|
||||
"file:metrics.db?cache=shared",
|
||||
WithTablePrefix("x_metric_"),
|
||||
WithAutoMigrate(false),
|
||||
WithMaxOpenConns(10),
|
||||
WithMaxIdleConns(2),
|
||||
WithConnMaxLifetime(30*time.Minute),
|
||||
WithConnectTimeout(2*time.Second),
|
||||
)
|
||||
|
||||
if cfg.Driver != DriverSQLite {
|
||||
t.Fatalf("expected sqlite driver, got %q", cfg.Driver)
|
||||
}
|
||||
if cfg.TablePrefix != "x_metric_" {
|
||||
t.Fatalf("table prefix option was not applied: %q", cfg.TablePrefix)
|
||||
}
|
||||
if cfg.AutoMigrate {
|
||||
t.Fatalf("auto migrate option was not applied")
|
||||
}
|
||||
if cfg.MaxOpenConns != 10 || cfg.MaxIdleConns != 2 {
|
||||
t.Fatalf("pool options were not applied: %#v", cfg)
|
||||
}
|
||||
if cfg.ConnMaxLifetime != 30*time.Minute || cfg.ConnectTimeout != 2*time.Second {
|
||||
t.Fatalf("timeout options were not applied: %#v", cfg)
|
||||
}
|
||||
}
|
||||
|
||||
// TestSQLiteInDirBuildsFileConfig verifies SQLiteInDir file configuration.
|
||||
//
|
||||
// TestSQLiteInDirBuildsFileConfig 验证 SQLiteInDir 会生成预期的文件数据库配置。
|
||||
func TestSQLiteInDirBuildsFileConfig(t *testing.T) {
|
||||
cfg := SQLiteInDir(
|
||||
"data/metrics",
|
||||
WithSQLiteProfile(SQLiteProfilePerformance),
|
||||
WithSQLiteCacheSizeKB(128*1024),
|
||||
)
|
||||
|
||||
if cfg.Driver != DriverSQLite {
|
||||
t.Fatalf("expected sqlite driver, got %q", cfg.Driver)
|
||||
}
|
||||
if cfg.MaxOpenConns != 1 || cfg.MaxIdleConns != 1 {
|
||||
t.Fatalf("sqlite directory config should default to a single connection: %#v", cfg)
|
||||
}
|
||||
if cfg.SQLite.PerformanceProfile != SQLiteProfilePerformance {
|
||||
t.Fatalf("sqlite profile option was not applied: %q", cfg.SQLite.PerformanceProfile)
|
||||
}
|
||||
if cfg.SQLite.CacheSizeKB != 128*1024 {
|
||||
t.Fatalf("sqlite cache option was not applied: %d", cfg.SQLite.CacheSizeKB)
|
||||
}
|
||||
if cfg.DSN != "file:data/metrics/metrics.db?cache=shared&mode=rwc" {
|
||||
t.Fatalf("unexpected sqlite dir dsn: %q", cfg.DSN)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,481 @@
|
||||
package metric
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"fmt"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// dialect abstracts SQL differences between supported databases.
|
||||
//
|
||||
// dialect 抽象不同数据库后端的 SQL 差异。
|
||||
type dialect interface {
|
||||
placeholder(n int) string
|
||||
// jsonPlaceholder returns the bind placeholder for a JSON column value.
|
||||
// PostgreSQL needs an explicit ::jsonb cast because the driver sends Go
|
||||
// strings with a text OID, which the server refuses to assign to a jsonb
|
||||
// column. SQLite and MySQL store JSON as text/native and need no cast.
|
||||
//
|
||||
// jsonPlaceholder 返回 JSON 列值的绑定占位符。PostgreSQL 需要显式的
|
||||
// ::jsonb 转换,因为驱动会以 text OID 发送 Go 字符串,服务器拒绝把它赋给
|
||||
// jsonb 列。SQLite 和 MySQL 将 JSON 存为文本或原生类型,不需要转换。
|
||||
jsonPlaceholder(n int) string
|
||||
jsonType() string
|
||||
autoIncrementPrimaryKey() string
|
||||
nowExpr() string
|
||||
insertDefinitionSQL(t tables) string
|
||||
upsertPointSQL(t tables, rowCount int) string
|
||||
// jsonExtractEquals renders a boolean predicate comparing the text value at
|
||||
// key inside a JSON column to a bind placeholder. The key is interpolated
|
||||
// into the SQL (callers must restrict it to a safe charset); the compared
|
||||
// value is always bound.
|
||||
//
|
||||
// jsonExtractEquals 渲染一个布尔谓词,用于比较 JSON 列内 key 对应的文本值和
|
||||
// 绑定占位符。key 会插入 SQL 中(调用方必须限制到安全字符集);被比较的值
|
||||
// 始终通过绑定参数传入。
|
||||
jsonExtractEquals(column, key, placeholder string) string
|
||||
// blobType is the column type for the rollup t-digest sketch.
|
||||
//
|
||||
// blobType 是 rollup t-digest sketch 的列类型。
|
||||
blobType() string
|
||||
// upsertRollupSQL builds a single-row upsert for one rollup bucket cell.
|
||||
//
|
||||
// upsertRollupSQL 为单个 rollup 桶单元构造单行 upsert。
|
||||
upsertRollupSQL(t tables) string
|
||||
// upsertCompactionWatermarkSQL builds a single-row upsert for the metric's
|
||||
// persisted compaction watermark.
|
||||
upsertCompactionWatermarkSQL(t tables) string
|
||||
// compactTxOptions returns the transaction options used for the compaction
|
||||
// transaction. PostgreSQL/MySQL escalate to SERIALIZABLE so the raw scan and
|
||||
// the raw deletion share one snapshot and a point inserted between them
|
||||
// cannot be deleted without first being rolled up. SQLite returns nil
|
||||
// (its default) because a single connection already serializes writes.
|
||||
//
|
||||
// compactTxOptions 返回 compaction 事务使用的事务选项。PostgreSQL/MySQL 会
|
||||
// 提升到 SERIALIZABLE,使 raw 扫描和 raw 删除共享同一个快照,让两者之间写入
|
||||
// 的点不会在尚未进入 rollup 前被删除。SQLite 返回 nil(默认),因为单连接
|
||||
// 已经串行化写入。
|
||||
compactTxOptions() *sql.TxOptions
|
||||
}
|
||||
|
||||
// tables stores the physical table names used by a Store.
|
||||
//
|
||||
// tables 保存 Store 使用的实际表名。
|
||||
type tables struct {
|
||||
// definitions is the metric definitions table.
|
||||
//
|
||||
// definitions 是指标定义表。
|
||||
definitions string
|
||||
// points is the raw points table.
|
||||
//
|
||||
// points 是原始采样点表。
|
||||
points string
|
||||
// rollups is the downsampled rollups table.
|
||||
//
|
||||
// rollups 是降采样 rollup 表。
|
||||
rollups string
|
||||
// watermarks stores the last successfully compacted raw boundary per metric.
|
||||
//
|
||||
// watermarks 保存每个指标最近一次成功压缩到的 raw 边界。
|
||||
watermarks string
|
||||
}
|
||||
|
||||
// newDialect returns the SQL dialect implementation for a backend.
|
||||
//
|
||||
// newDialect 根据数据库后端创建对应 SQL 方言实现。
|
||||
func newDialect(driver Driver) dialect {
|
||||
switch driver {
|
||||
case DriverPostgreSQL:
|
||||
return postgresDialect{}
|
||||
case DriverMySQL:
|
||||
return mysqlDialect{}
|
||||
default:
|
||||
return sqliteDialect{}
|
||||
}
|
||||
}
|
||||
|
||||
// sqliteDialect implements SQL rendering for SQLite.
|
||||
//
|
||||
// sqliteDialect 实现 SQLite 方言。
|
||||
type sqliteDialect struct{}
|
||||
|
||||
// placeholder returns a bind placeholder for this SQL dialect.
|
||||
//
|
||||
// placeholder 返回 SQLite 的绑定参数占位符。
|
||||
func (sqliteDialect) placeholder(int) string { return "?" }
|
||||
|
||||
// jsonPlaceholder returns a bind placeholder for JSON values.
|
||||
//
|
||||
// jsonPlaceholder 返回 SQLite JSON 值的绑定参数占位符。
|
||||
func (sqliteDialect) jsonPlaceholder(int) string { return "?" }
|
||||
|
||||
// jsonType returns the SQL column type used for JSON values.
|
||||
//
|
||||
// jsonType 返回 SQLite 使用的 JSON 存储类型。
|
||||
func (sqliteDialect) jsonType() string { return "TEXT" }
|
||||
|
||||
// autoIncrementPrimaryKey returns the SQL definition for an auto-incrementing primary key.
|
||||
//
|
||||
// autoIncrementPrimaryKey 返回 SQLite 自增主键定义。
|
||||
func (sqliteDialect) autoIncrementPrimaryKey() string { return "INTEGER PRIMARY KEY AUTOINCREMENT" }
|
||||
|
||||
// nowExpr returns the SQL expression for the current time.
|
||||
//
|
||||
// nowExpr 返回 SQLite 当前时间表达式。
|
||||
func (sqliteDialect) nowExpr() string { return "CURRENT_TIMESTAMP" }
|
||||
|
||||
// insertDefinitionSQL builds SQL for inserting or updating a metric definition.
|
||||
//
|
||||
// insertDefinitionSQL 构造 SQLite 指标定义 upsert SQL。
|
||||
func (sqliteDialect) insertDefinitionSQL(t tables) string {
|
||||
return fmt.Sprintf(`INSERT INTO %s
|
||||
(name, type, unit, description, retention_days, metadata, created_at, updated_at)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?)
|
||||
ON CONFLICT(name) DO UPDATE SET
|
||||
type = excluded.type,
|
||||
unit = excluded.unit,
|
||||
description = excluded.description,
|
||||
retention_days = excluded.retention_days,
|
||||
metadata = excluded.metadata,
|
||||
updated_at = excluded.updated_at`, t.definitions)
|
||||
}
|
||||
|
||||
// upsertPointSQL builds SQL for inserting or updating metric points.
|
||||
//
|
||||
// upsertPointSQL 构造 SQLite 采样点批量 upsert SQL。
|
||||
func (sqliteDialect) upsertPointSQL(t tables, rowCount int) string {
|
||||
return buildInsertPointsSQL(t.points, sqliteDialect{}, rowCount, `ON CONFLICT(metric_name, entity_id, tags_hash, ts_nano) DO UPDATE SET
|
||||
value = excluded.value,
|
||||
tags = excluded.tags,
|
||||
labels = excluded.labels,
|
||||
created_at = excluded.created_at`)
|
||||
}
|
||||
|
||||
// upsertCompactionWatermarkSQL builds the SQLite watermark upsert.
|
||||
func (sqliteDialect) upsertCompactionWatermarkSQL(t tables) string {
|
||||
return fmt.Sprintf(`INSERT INTO %s
|
||||
(metric_name, watermark_nano, updated_at)
|
||||
VALUES (?, ?, ?)
|
||||
ON CONFLICT(metric_name) DO UPDATE SET
|
||||
watermark_nano = excluded.watermark_nano,
|
||||
updated_at = excluded.updated_at`, t.watermarks)
|
||||
}
|
||||
|
||||
// mysqlDialect implements SQL rendering for MySQL.
|
||||
//
|
||||
// mysqlDialect 实现 MySQL 方言。
|
||||
type mysqlDialect struct{}
|
||||
|
||||
// placeholder returns a bind placeholder for this SQL dialect.
|
||||
//
|
||||
// placeholder 返回 MySQL 的绑定参数占位符。
|
||||
func (mysqlDialect) placeholder(int) string { return "?" }
|
||||
|
||||
// jsonPlaceholder returns a bind placeholder for JSON values.
|
||||
//
|
||||
// jsonPlaceholder 返回 MySQL JSON 值的绑定参数占位符。
|
||||
func (mysqlDialect) jsonPlaceholder(int) string { return "?" }
|
||||
|
||||
// jsonType returns the SQL column type used for JSON values.
|
||||
//
|
||||
// jsonType 返回 MySQL 使用的 JSON 存储类型。
|
||||
func (mysqlDialect) jsonType() string { return "JSON" }
|
||||
|
||||
// autoIncrementPrimaryKey returns the SQL definition for an auto-incrementing primary key.
|
||||
//
|
||||
// autoIncrementPrimaryKey 返回 MySQL 自增主键定义。
|
||||
func (mysqlDialect) autoIncrementPrimaryKey() string { return "BIGINT AUTO_INCREMENT PRIMARY KEY" }
|
||||
|
||||
// nowExpr returns the SQL expression for the current time.
|
||||
//
|
||||
// nowExpr 返回 MySQL 当前时间表达式。
|
||||
func (mysqlDialect) nowExpr() string { return "CURRENT_TIMESTAMP" }
|
||||
|
||||
// insertDefinitionSQL builds SQL for inserting or updating a metric definition.
|
||||
//
|
||||
// insertDefinitionSQL 构造 MySQL 指标定义 upsert SQL。
|
||||
func (mysqlDialect) insertDefinitionSQL(t tables) string {
|
||||
return fmt.Sprintf(`INSERT INTO %s
|
||||
(name, type, unit, description, retention_days, metadata, created_at, updated_at)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?)
|
||||
ON DUPLICATE KEY UPDATE
|
||||
type = VALUES(type),
|
||||
unit = VALUES(unit),
|
||||
description = VALUES(description),
|
||||
retention_days = VALUES(retention_days),
|
||||
metadata = VALUES(metadata),
|
||||
updated_at = VALUES(updated_at)`, t.definitions)
|
||||
}
|
||||
|
||||
// upsertPointSQL builds SQL for inserting or updating metric points.
|
||||
//
|
||||
// upsertPointSQL 构造 MySQL 采样点批量 upsert SQL。
|
||||
func (mysqlDialect) upsertPointSQL(t tables, rowCount int) string {
|
||||
return buildInsertPointsSQL(t.points, mysqlDialect{}, rowCount, `ON DUPLICATE KEY UPDATE
|
||||
value = VALUES(value),
|
||||
tags = VALUES(tags),
|
||||
labels = VALUES(labels),
|
||||
created_at = VALUES(created_at)`)
|
||||
}
|
||||
|
||||
// upsertCompactionWatermarkSQL builds the MySQL watermark upsert.
|
||||
func (mysqlDialect) upsertCompactionWatermarkSQL(t tables) string {
|
||||
return fmt.Sprintf(`INSERT INTO %s
|
||||
(metric_name, watermark_nano, updated_at)
|
||||
VALUES (?, ?, ?)
|
||||
ON DUPLICATE KEY UPDATE
|
||||
watermark_nano = VALUES(watermark_nano),
|
||||
updated_at = VALUES(updated_at)`, t.watermarks)
|
||||
}
|
||||
|
||||
// postgresDialect implements SQL rendering for PostgreSQL.
|
||||
//
|
||||
// postgresDialect 实现 PostgreSQL 方言。
|
||||
type postgresDialect struct{}
|
||||
|
||||
// placeholder returns a bind placeholder for this SQL dialect.
|
||||
//
|
||||
// placeholder 返回 PostgreSQL 的编号绑定参数占位符。
|
||||
func (postgresDialect) placeholder(n int) string { return fmt.Sprintf("$%d", n) }
|
||||
|
||||
// jsonPlaceholder returns a bind placeholder for JSON values.
|
||||
//
|
||||
// jsonPlaceholder 返回 PostgreSQL JSONB 值的绑定参数占位符。
|
||||
func (postgresDialect) jsonPlaceholder(n int) string { return fmt.Sprintf("$%d::jsonb", n) }
|
||||
|
||||
// jsonType returns the SQL column type used for JSON values.
|
||||
//
|
||||
// jsonType 返回 PostgreSQL 使用的 JSONB 存储类型。
|
||||
func (postgresDialect) jsonType() string { return "JSONB" }
|
||||
|
||||
// autoIncrementPrimaryKey returns the SQL definition for an auto-incrementing primary key.
|
||||
//
|
||||
// autoIncrementPrimaryKey 返回 PostgreSQL 自增主键定义。
|
||||
func (postgresDialect) autoIncrementPrimaryKey() string { return "BIGSERIAL PRIMARY KEY" }
|
||||
|
||||
// nowExpr returns the SQL expression for the current time.
|
||||
//
|
||||
// nowExpr 返回 PostgreSQL 当前时间表达式。
|
||||
func (postgresDialect) nowExpr() string { return "CURRENT_TIMESTAMP" }
|
||||
|
||||
// insertDefinitionSQL builds SQL for inserting or updating a metric definition.
|
||||
//
|
||||
// insertDefinitionSQL 构造 PostgreSQL 指标定义 upsert SQL。
|
||||
func (postgresDialect) insertDefinitionSQL(t tables) string {
|
||||
return fmt.Sprintf(`INSERT INTO %s
|
||||
(name, type, unit, description, retention_days, metadata, created_at, updated_at)
|
||||
VALUES ($1, $2, $3, $4, $5, $6::jsonb, $7, $8)
|
||||
ON CONFLICT(name) DO UPDATE SET
|
||||
type = EXCLUDED.type,
|
||||
unit = EXCLUDED.unit,
|
||||
description = EXCLUDED.description,
|
||||
retention_days = EXCLUDED.retention_days,
|
||||
metadata = EXCLUDED.metadata,
|
||||
updated_at = EXCLUDED.updated_at`, t.definitions)
|
||||
}
|
||||
|
||||
// upsertPointSQL builds SQL for inserting or updating metric points.
|
||||
//
|
||||
// upsertPointSQL 构造 PostgreSQL 采样点批量 upsert SQL。
|
||||
func (postgresDialect) upsertPointSQL(t tables, rowCount int) string {
|
||||
return buildInsertPointsSQL(t.points, postgresDialect{}, rowCount, `ON CONFLICT(metric_name, entity_id, tags_hash, ts_nano) DO UPDATE SET
|
||||
value = EXCLUDED.value,
|
||||
tags = EXCLUDED.tags,
|
||||
labels = EXCLUDED.labels,
|
||||
created_at = EXCLUDED.created_at`)
|
||||
}
|
||||
|
||||
// upsertCompactionWatermarkSQL builds the PostgreSQL watermark upsert.
|
||||
func (postgresDialect) upsertCompactionWatermarkSQL(t tables) string {
|
||||
return fmt.Sprintf(`INSERT INTO %s
|
||||
(metric_name, watermark_nano, updated_at)
|
||||
VALUES ($1, $2, $3)
|
||||
ON CONFLICT(metric_name) DO UPDATE SET
|
||||
watermark_nano = EXCLUDED.watermark_nano,
|
||||
updated_at = EXCLUDED.updated_at`, t.watermarks)
|
||||
}
|
||||
|
||||
// buildInsertPointsSQL builds a multi-row insert statement for metric points.
|
||||
//
|
||||
// buildInsertPointsSQL 构造多行采样点 INSERT/UPSERT SQL。
|
||||
func buildInsertPointsSQL(table string, d dialect, rowCount int, suffix string) string {
|
||||
var b strings.Builder
|
||||
b.WriteString("INSERT INTO ")
|
||||
b.WriteString(table)
|
||||
b.WriteString(" (metric_name, entity_id, tags_hash, ts_nano, value, tags, labels, created_at) VALUES ")
|
||||
arg := 1
|
||||
rows := make([]string, rowCount)
|
||||
for i := 0; i < rowCount; i++ {
|
||||
parts := make([]string, 8)
|
||||
for j := range parts {
|
||||
// Columns are (metric_name, entity_id, tags_hash, ts_nano, value, tags,
|
||||
// labels, created_at); positions 5 and 6 (0-indexed) are JSON values.
|
||||
// tags_hash makes the tag set part of a point's identity, so points
|
||||
// that share entity+timestamp but differ in tags no longer collide.
|
||||
if j == 5 || j == 6 {
|
||||
parts[j] = d.jsonPlaceholder(arg)
|
||||
} else {
|
||||
parts[j] = d.placeholder(arg)
|
||||
}
|
||||
arg++
|
||||
}
|
||||
rows[i] = "(" + strings.Join(parts, ", ") + ")"
|
||||
}
|
||||
b.WriteString(strings.Join(rows, ", "))
|
||||
if suffix != "" {
|
||||
b.WriteByte(' ')
|
||||
b.WriteString(suffix)
|
||||
}
|
||||
return b.String()
|
||||
}
|
||||
|
||||
// tableName combines a table prefix and logical table name.
|
||||
//
|
||||
// tableName 用表名前缀和逻辑表名生成实际表名。
|
||||
func tableName(prefix, name string) string {
|
||||
return prefix + name
|
||||
}
|
||||
|
||||
// insertDefinitionOnlySQL builds a plain INSERT (no upsert clause) for a metric
|
||||
// definition. CreateMetric uses it so a duplicate name surfaces as a unique
|
||||
// constraint violation instead of silently overwriting the existing row.
|
||||
//
|
||||
// insertDefinitionOnlySQL 构造普通 INSERT(不带 upsert),供
|
||||
// CreateMetric 保持“只创建”语义;重复名称会暴露为唯一约束错误。
|
||||
func insertDefinitionOnlySQL(d dialect, t tables) string {
|
||||
cols := "(name, type, unit, description, retention_days, metadata, created_at, updated_at)"
|
||||
ph := []string{
|
||||
d.placeholder(1),
|
||||
d.placeholder(2),
|
||||
d.placeholder(3),
|
||||
d.placeholder(4),
|
||||
d.placeholder(5),
|
||||
d.jsonPlaceholder(6),
|
||||
d.placeholder(7),
|
||||
d.placeholder(8),
|
||||
}
|
||||
return fmt.Sprintf("INSERT INTO %s %s VALUES (%s)", t.definitions, cols, strings.Join(ph, ", "))
|
||||
}
|
||||
|
||||
// sqlSingleQuote escapes a string for safe inclusion inside a single-quoted SQL
|
||||
// string literal by doubling embedded single quotes.
|
||||
//
|
||||
// sqlSingleQuote 通过把单引号加倍,安全地转义可放入 SQL 单引号字符串
|
||||
// 字面量的内容。
|
||||
func sqlSingleQuote(s string) string {
|
||||
return strings.ReplaceAll(s, "'", "''")
|
||||
}
|
||||
|
||||
// sqlJSONPathQuoted builds a quoted JSON path member access ($."key") for the
|
||||
// given object key and returns it ready to embed inside a single-quoted SQL
|
||||
// string literal. Using the quoted form ($."key") rather than the bare form
|
||||
// ($.key) is required for keys that contain dots, hyphens, spaces or other
|
||||
// characters the JSON path grammar would otherwise interpret (e.g. a dot is a
|
||||
// member separator, so the bare path "$.region.zone" would look up nested
|
||||
// member "zone" of object "region" instead of the flat key "region.zone").
|
||||
//
|
||||
// Escaping is applied in two layers, innermost first:
|
||||
// 1. JSON path string escaping inside the double quotes: backslash and double
|
||||
// quote are backslash-escaped.
|
||||
// 2. SQL string-literal escaping of the whole path: single quotes doubled.
|
||||
//
|
||||
// sqlJSONPathQuoted 为给定对象键构造带引号的 JSON path 成员访问($."key"),
|
||||
// 并返回可嵌入 SQL 单引号字符串字面量的结果。必须使用带引号形式($."key"),
|
||||
// 而不是裸形式($.key),因为包含点号、连字符、空格或其他 JSON path 语法字符的
|
||||
// key 会被误解;例如点号是成员分隔符,所以裸路径 "$.region.zone" 会查找对象
|
||||
// "region" 的嵌套成员 "zone",而不是平铺 key "region.zone"。
|
||||
//
|
||||
// 转义按从内到外两层应用:
|
||||
// 1. 双引号内的 JSON path 字符串转义:反斜杠和双引号使用反斜杠转义。
|
||||
// 2. 整个 path 的 SQL 字符串字面量转义:单引号加倍。
|
||||
func sqlJSONPathQuoted(key string) string {
|
||||
escaped := strings.ReplaceAll(key, "\\", "\\\\")
|
||||
escaped = strings.ReplaceAll(escaped, "\"", "\\\"")
|
||||
return sqlSingleQuote("$.\"" + escaped + "\"")
|
||||
}
|
||||
|
||||
// jsonExtractEquals renders a JSON equality predicate.
|
||||
//
|
||||
// jsonExtractEquals 构造 SQLite JSON 字段等值过滤表达式。
|
||||
func (sqliteDialect) jsonExtractEquals(column, key, placeholder string) string {
|
||||
return fmt.Sprintf("json_extract(%s, '%s') = %s", column, sqlJSONPathQuoted(key), placeholder)
|
||||
}
|
||||
|
||||
// jsonExtractEquals renders a JSON equality predicate.
|
||||
//
|
||||
// jsonExtractEquals 构造 MySQL JSON 字段等值过滤表达式。
|
||||
func (mysqlDialect) jsonExtractEquals(column, key, placeholder string) string {
|
||||
return fmt.Sprintf("JSON_UNQUOTE(JSON_EXTRACT(%s, '%s')) = %s", column, sqlJSONPathQuoted(key), placeholder)
|
||||
}
|
||||
|
||||
// jsonExtractEquals renders a JSON equality predicate.
|
||||
//
|
||||
// jsonExtractEquals 构造 PostgreSQL JSONB 字段等值过滤表达式。
|
||||
func (postgresDialect) jsonExtractEquals(column, key, placeholder string) string {
|
||||
// PostgreSQL's ->> takes the key as a plain text literal (not a JSON path),
|
||||
// so a dot or hyphen in the key is harmless; only single quotes need SQL
|
||||
// escaping.
|
||||
return fmt.Sprintf("%s->>'%s' = %s", column, sqlSingleQuote(key), placeholder)
|
||||
}
|
||||
|
||||
// sqlAggValueExpr returns the SQL value expression that computes agg over the
|
||||
// "value" column for the given driver, and whether that aggregation can be
|
||||
// pushed down to this backend. Aggregations that need the ordered raw series
|
||||
// (first/last/rate) are never pushed down here. Percentiles and population
|
||||
// standard deviation are pushed down only where the backend has a portable,
|
||||
// semantics-matching function:
|
||||
//
|
||||
// - STDDEV_POP: MySQL and PostgreSQL (matches the in-memory population stddev,
|
||||
// which divides by N). SQLite has no built-in, so it falls back to memory.
|
||||
// - percentile_cont(p) WITHIN GROUP (ORDER BY value): PostgreSQL only. This
|
||||
// matches the in-memory linear-interpolation percentile for ANY p (p50,
|
||||
// p95, p99, p99.9, ...). MySQL lacks a portable continuous-percentile
|
||||
// function and SQLite has none, so both fall back to memory.
|
||||
//
|
||||
// The simple reductions (avg/min/max/sum/count) are portable everywhere.
|
||||
//
|
||||
// sqlAggValueExpr 返回给定驱动上对 "value" 列计算 agg 的 SQL 值表达式,
|
||||
// 以及该聚合是否能下推到该后端。需要有序原始序列的聚合(first/last/rate)
|
||||
// 永远不会在这里下推。百分位和总体标准差只会在后端提供可移植且语义匹配的函数时
|
||||
// 下推:
|
||||
//
|
||||
// - STDDEV_POP:MySQL 和 PostgreSQL 支持(匹配内存中的总体标准差,
|
||||
// 即除以 N)。SQLite 没有内置函数,因此回退到内存计算。
|
||||
// - percentile_cont(p) WITHIN GROUP (ORDER BY value):仅 PostgreSQL 支持。
|
||||
// 它匹配内存中任意 p(p50、p95、p99、p99.9 等)的线性插值百分位。
|
||||
// MySQL 没有可移植的连续百分位函数,SQLite 也没有,因此两者都回退到内存计算。
|
||||
//
|
||||
// 简单归约(avg/min/max/sum/count)在所有后端都可移植。
|
||||
func sqlAggValueExpr(driver Driver, agg Aggregation) (string, bool) {
|
||||
switch agg {
|
||||
case AggAvg:
|
||||
return "AVG(value)", true
|
||||
case AggMin:
|
||||
return "MIN(value)", true
|
||||
case AggMax:
|
||||
return "MAX(value)", true
|
||||
case AggSum:
|
||||
return "SUM(value)", true
|
||||
case AggCount:
|
||||
return "COUNT(*)", true
|
||||
case AggStdDev:
|
||||
switch driver {
|
||||
case DriverMySQL, DriverPostgreSQL:
|
||||
return "STDDEV_POP(value)", true
|
||||
default:
|
||||
return "", false
|
||||
}
|
||||
default:
|
||||
// Any percentile (fixed p50/p95/p99 and arbitrary pXX / pXX.X all land
|
||||
// here). PostgreSQL has a portable continuous percentile that matches the
|
||||
// in-memory linear-interpolation percentile; other backends fall back to
|
||||
// the in-memory path.
|
||||
if frac, ok := percentileFractionString(agg); ok {
|
||||
if driver == DriverPostgreSQL {
|
||||
return fmt.Sprintf("percentile_cont(%s) WITHIN GROUP (ORDER BY value)", frac), true
|
||||
}
|
||||
return "", false
|
||||
}
|
||||
return "", false
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,160 @@
|
||||
package metric
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"fmt"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// rollupColumns is the fixed column list for the rollups table, shared by the
|
||||
// upsert builder and the row scanner so they cannot drift apart. tags_hash is a
|
||||
// stable fingerprint of the (canonical) tag set; tags carries the tag map itself
|
||||
// so reads can return it and tag filters can be pushed down. Including the tag
|
||||
// dimension in the key is what keeps each distinct tag combination (e.g. a GPU
|
||||
// device_index) as its own rollup series instead of being merged together.
|
||||
//
|
||||
// rollupColumns 是 rollups 表固定列清单,由 upsert 构造器和行扫描器共用,
|
||||
// 避免二者漂移。tags_hash 是(规范化)标签集合的稳定指纹;tags 携带标签 map
|
||||
// 本身,因此读取时可以返回它,也能下推标签过滤。把标签维度纳入 key,正是让
|
||||
// 每个不同标签组合(例如 GPU 的 device_index)保留为自己的 rollup 序列、
|
||||
// 而不是被合并在一起的原因。
|
||||
const rollupColumns = "metric_name, entity_id, tags_hash, tags, resolution_nano, bucket_nano, " +
|
||||
"count, sum, sum_sq, min_val, max_val, first_val, first_ts, last_val, last_ts, digest, created_at"
|
||||
|
||||
// rollupColumnCount is the number of columns in rollupColumns.
|
||||
//
|
||||
// rollupColumnCount 是 rollupColumns 中的列数量。
|
||||
const rollupColumnCount = 17
|
||||
|
||||
// rollupTagsArgIndex is the 1-based position of the `tags` JSON column in
|
||||
// rollupColumns. PostgreSQL needs a ::jsonb cast on that bind placeholder.
|
||||
//
|
||||
// rollupTagsArgIndex 是 tags JSON 列在 rollupColumns 中的 1 基位置;
|
||||
// PostgreSQL 需要在对应占位符上追加 ::jsonb 转换。
|
||||
const rollupTagsArgIndex = 4
|
||||
|
||||
// blobType returns the column type used to store the t-digest sketch.
|
||||
//
|
||||
// blobType 返回用于保存 t-digest 摘要的二进制列类型。
|
||||
func (sqliteDialect) blobType() string { return "BLOB" }
|
||||
|
||||
// blobType returns the binary column type used for t-digest blobs.
|
||||
//
|
||||
// blobType 返回 MySQL 用于保存 t-digest 摘要的二进制列类型。
|
||||
func (mysqlDialect) blobType() string { return "LONGBLOB" }
|
||||
|
||||
// blobType returns the binary column type used for t-digest blobs.
|
||||
//
|
||||
// blobType 返回 PostgreSQL 用于保存 t-digest 摘要的二进制列类型。
|
||||
func (postgresDialect) blobType() string { return "BYTEA" }
|
||||
|
||||
// compactTxOptions returns nil so SQLite uses its default isolation. A single
|
||||
// write connection already serializes the compaction transaction against
|
||||
// concurrent writers, so the raw scan and the raw deletion observe a stable
|
||||
// view without escalating the isolation level.
|
||||
//
|
||||
// compactTxOptions 返回 nil,让 SQLite 使用默认隔离级别。单个写连接已经把
|
||||
// compaction 事务与并发写入串行化,因此 raw 扫描和 raw 删除无需提升隔离级别
|
||||
// 即可观察到稳定视图。
|
||||
func (sqliteDialect) compactTxOptions() *sql.TxOptions { return nil }
|
||||
|
||||
// compactTxOptions escalates to SERIALIZABLE so a point inserted between the raw
|
||||
// scan and the raw deletion cannot be deleted without first being rolled up.
|
||||
// InnoDB's SERIALIZABLE turns the scan into a locking read that blocks such a
|
||||
// concurrent insert in the scanned range until the compaction commits.
|
||||
//
|
||||
// compactTxOptions 提升到 SERIALIZABLE,使在 raw 扫描和 raw 删除之间写入的点
|
||||
// 不会在尚未进入 rollup 前被删除。InnoDB 的 SERIALIZABLE 会把扫描变为加锁读,
|
||||
// 在 compaction 提交前阻塞扫描范围内的此类并发插入。
|
||||
func (mysqlDialect) compactTxOptions() *sql.TxOptions {
|
||||
return &sql.TxOptions{Isolation: sql.LevelSerializable}
|
||||
}
|
||||
|
||||
// compactTxOptions escalates to SERIALIZABLE so the raw scan and the raw
|
||||
// deletion share one snapshot. A concurrent insert into the scanned range that
|
||||
// would otherwise be deleted without a rollup instead triggers a serialization
|
||||
// failure that CompactMetric retries on a fresh snapshot.
|
||||
//
|
||||
// compactTxOptions 提升到 SERIALIZABLE,使 raw 扫描和 raw 删除共享同一个快照。
|
||||
// 对扫描范围内的并发插入(否则会被删除却没有进入 rollup)会触发序列化失败,
|
||||
// 由 CompactMetric 在新快照上重试。
|
||||
func (postgresDialect) compactTxOptions() *sql.TxOptions {
|
||||
return &sql.TxOptions{Isolation: sql.LevelSerializable}
|
||||
}
|
||||
|
||||
// upsertRollupSQL builds a single-row upsert for a rollup cell, keyed by
|
||||
// (metric_name, entity_id, tags_hash, resolution_nano, bucket_nano). Compact
|
||||
// recomputes a bucket wholesale from its inputs each run, so the conflict action
|
||||
// replaces every summary column rather than trying to merge in SQL.
|
||||
//
|
||||
// upsertRollupSQL 构造单个 rollup 单元的 upsert SQL,其 key 为
|
||||
// (metric_name, entity_id, tags_hash, resolution_nano, bucket_nano)。Compact
|
||||
// 每次都会从输入整体重新计算一个桶,因此冲突动作会替换每个摘要列,
|
||||
// 而不是尝试在 SQL 中合并。
|
||||
func (d sqliteDialect) upsertRollupSQL(t tables) string {
|
||||
return buildUpsertRollupSQL(t.rollups, d, sqliteRollupConflict)
|
||||
}
|
||||
|
||||
// upsertRollupSQL builds backend-specific SQL for upserting a rollup cell.
|
||||
//
|
||||
// upsertRollupSQL 构造 MySQL rollup 单元 upsert SQL。
|
||||
func (d mysqlDialect) upsertRollupSQL(t tables) string {
|
||||
return buildUpsertRollupSQL(t.rollups, d, mysqlRollupConflict)
|
||||
}
|
||||
|
||||
// upsertRollupSQL builds backend-specific SQL for upserting a rollup cell.
|
||||
//
|
||||
// upsertRollupSQL 构造 PostgreSQL rollup 单元 upsert SQL。
|
||||
func (d postgresDialect) upsertRollupSQL(t tables) string {
|
||||
return buildUpsertRollupSQL(t.rollups, d, postgresRollupConflict)
|
||||
}
|
||||
|
||||
// sqliteRollupConflict is the SQLite conflict clause for replacing rollup cells.
|
||||
//
|
||||
// sqliteRollupConflict 是 SQLite 用于替换 rollup 单元的冲突处理子句。
|
||||
const sqliteRollupConflict = `ON CONFLICT(metric_name, entity_id, tags_hash, resolution_nano, bucket_nano) DO UPDATE SET
|
||||
tags = excluded.tags,
|
||||
count = excluded.count, sum = excluded.sum, sum_sq = excluded.sum_sq,
|
||||
min_val = excluded.min_val, max_val = excluded.max_val,
|
||||
first_val = excluded.first_val, first_ts = excluded.first_ts,
|
||||
last_val = excluded.last_val, last_ts = excluded.last_ts,
|
||||
digest = excluded.digest, created_at = excluded.created_at`
|
||||
|
||||
// postgresRollupConflict is the PostgreSQL conflict clause for replacing rollup cells.
|
||||
//
|
||||
// postgresRollupConflict 是 PostgreSQL 用于替换 rollup 单元的冲突处理子句。
|
||||
const postgresRollupConflict = `ON CONFLICT(metric_name, entity_id, tags_hash, resolution_nano, bucket_nano) DO UPDATE SET
|
||||
tags = EXCLUDED.tags,
|
||||
count = EXCLUDED.count, sum = EXCLUDED.sum, sum_sq = EXCLUDED.sum_sq,
|
||||
min_val = EXCLUDED.min_val, max_val = EXCLUDED.max_val,
|
||||
first_val = EXCLUDED.first_val, first_ts = EXCLUDED.first_ts,
|
||||
last_val = EXCLUDED.last_val, last_ts = EXCLUDED.last_ts,
|
||||
digest = EXCLUDED.digest, created_at = EXCLUDED.created_at`
|
||||
|
||||
// mysqlRollupConflict is the MySQL conflict clause for replacing rollup cells.
|
||||
//
|
||||
// mysqlRollupConflict 是 MySQL 用于替换 rollup 单元的冲突处理子句。
|
||||
const mysqlRollupConflict = `ON DUPLICATE KEY UPDATE
|
||||
tags = VALUES(tags),
|
||||
count = VALUES(count), sum = VALUES(sum), sum_sq = VALUES(sum_sq),
|
||||
min_val = VALUES(min_val), max_val = VALUES(max_val),
|
||||
first_val = VALUES(first_val), first_ts = VALUES(first_ts),
|
||||
last_val = VALUES(last_val), last_ts = VALUES(last_ts),
|
||||
digest = VALUES(digest), created_at = VALUES(created_at)`
|
||||
|
||||
// buildUpsertRollupSQL builds a single-row rollup upsert statement.
|
||||
//
|
||||
// buildUpsertRollupSQL 构造单行 rollup upsert SQL。
|
||||
func buildUpsertRollupSQL(table string, d dialect, conflict string) string {
|
||||
ph := make([]string, rollupColumnCount)
|
||||
for i := 0; i < rollupColumnCount; i++ {
|
||||
if i+1 == rollupTagsArgIndex {
|
||||
// The tags column is JSON; PostgreSQL requires an explicit ::jsonb cast.
|
||||
ph[i] = d.jsonPlaceholder(i + 1)
|
||||
} else {
|
||||
ph[i] = d.placeholder(i + 1)
|
||||
}
|
||||
}
|
||||
return fmt.Sprintf("INSERT INTO %s (%s) VALUES (%s) %s",
|
||||
table, rollupColumns, strings.Join(ph, ", "), conflict)
|
||||
}
|
||||
@@ -0,0 +1,60 @@
|
||||
package metric
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// TestDialectsGenerateBackendSpecificSQL verifies backend-specific SQL rendering.
|
||||
//
|
||||
// TestDialectsGenerateBackendSpecificSQL 验证不同数据库后端生成各自的 SQL。
|
||||
func TestDialectsGenerateBackendSpecificSQL(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
driver Driver
|
||||
placeholder string
|
||||
upsert string
|
||||
jsonType string
|
||||
}{
|
||||
{
|
||||
name: "sqlite",
|
||||
driver: DriverSQLite,
|
||||
placeholder: "?",
|
||||
upsert: "ON CONFLICT(metric_name, entity_id, tags_hash, ts_nano)",
|
||||
jsonType: "TEXT",
|
||||
},
|
||||
{
|
||||
name: "mysql",
|
||||
driver: DriverMySQL,
|
||||
placeholder: "?",
|
||||
upsert: "ON DUPLICATE KEY UPDATE",
|
||||
jsonType: "JSON",
|
||||
},
|
||||
{
|
||||
name: "postgresql",
|
||||
driver: DriverPostgreSQL,
|
||||
placeholder: "$1",
|
||||
upsert: "ON CONFLICT(metric_name, entity_id, tags_hash, ts_nano)",
|
||||
jsonType: "JSONB",
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
d := newDialect(tt.driver)
|
||||
if got := d.placeholder(1); got != tt.placeholder {
|
||||
t.Fatalf("placeholder: expected %q, got %q", tt.placeholder, got)
|
||||
}
|
||||
if got := d.jsonType(); got != tt.jsonType {
|
||||
t.Fatalf("json type: expected %q, got %q", tt.jsonType, got)
|
||||
}
|
||||
sql := d.upsertPointSQL(tables{points: "metric_points"}, 2)
|
||||
if !strings.Contains(sql, "metric_points") {
|
||||
t.Fatalf("upsert sql should reference metric_points: %s", sql)
|
||||
}
|
||||
if !strings.Contains(sql, tt.upsert) {
|
||||
t.Fatalf("upsert sql should contain %q: %s", tt.upsert, sql)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,26 @@
|
||||
package metric
|
||||
|
||||
import "errors"
|
||||
|
||||
var (
|
||||
// ErrInvalidArgument reports an invalid caller-supplied argument.
|
||||
//
|
||||
// ErrInvalidArgument 表示调用方传入了无效参数。
|
||||
ErrInvalidArgument = errors.New("metric: invalid argument")
|
||||
// ErrNotFound reports a missing metric or record.
|
||||
//
|
||||
// ErrNotFound 表示指标或记录不存在。
|
||||
ErrNotFound = errors.New("metric: not found")
|
||||
// ErrNoData reports that a valid query range contained no samples.
|
||||
//
|
||||
// ErrNoData 表示有效查询范围内没有采样。
|
||||
ErrNoData = errors.New("metric: no data in range")
|
||||
// ErrAlreadyExists reports that a create-only operation found an existing row.
|
||||
//
|
||||
// ErrAlreadyExists 表示只创建操作遇到了已存在的行。
|
||||
ErrAlreadyExists = errors.New("metric: already exists")
|
||||
// ErrClosed reports that the store has already been closed.
|
||||
//
|
||||
// ErrClosed 表示 Store 已经关闭。
|
||||
ErrClosed = errors.New("metric: store is closed")
|
||||
)
|
||||
@@ -0,0 +1,347 @@
|
||||
package metric_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"log"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/komari-monitor/komari/pkg/metric"
|
||||
)
|
||||
|
||||
// Example_rollupTags demonstrates tagged rollups with automatic series routing.
|
||||
//
|
||||
// Example_rollupTags 演示带标签的 rollup 以及 Series 自动路由读取。
|
||||
func Example_rollupTags() {
|
||||
ctx := context.Background()
|
||||
base := time.Date(2026, 6, 18, 10, 0, 0, 0, time.UTC)
|
||||
|
||||
store, err := metric.Open(ctx, metric.SQLite(
|
||||
"file:metric-example?mode=memory&cache=shared",
|
||||
metric.WithRollupPolicy(metric.RollupPolicy{
|
||||
RawRetention: 2 * time.Minute,
|
||||
Tiers: []metric.RollupTier{
|
||||
{Interval: time.Minute, Retention: 24 * time.Hour},
|
||||
},
|
||||
}),
|
||||
))
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
defer store.Close()
|
||||
|
||||
if err := store.CreateMetric(ctx, metric.Definition{
|
||||
Name: "gpu.usage",
|
||||
Type: metric.TypeGauge,
|
||||
Unit: "%",
|
||||
RetentionDays: 30}); err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
|
||||
var points []metric.Point
|
||||
for i := 0; i < 10; i++ {
|
||||
ts := base.Add(time.Duration(i) * time.Second)
|
||||
points = append(points,
|
||||
metric.Point{
|
||||
MetricName: "gpu.usage",
|
||||
EntityID: "host-1",
|
||||
Timestamp: ts,
|
||||
Value: float64(10 + i),
|
||||
Tags: map[string]string{"device_index": "0"},
|
||||
},
|
||||
metric.Point{
|
||||
MetricName: "gpu.usage",
|
||||
EntityID: "host-1",
|
||||
Timestamp: ts,
|
||||
Value: float64(80 + i),
|
||||
Tags: map[string]string{"device_index": "1"},
|
||||
},
|
||||
)
|
||||
}
|
||||
if err := store.WriteBatch(ctx, points); err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
|
||||
// Compact builds one rollup series per tag set. Raw points older than the
|
||||
// policy's RawRetention are deleted after their rollups are materialized.
|
||||
now := base.Add(time.Hour)
|
||||
if _, err := store.Compact(ctx, now); err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
|
||||
series, err := store.Series(ctx, metric.AggregateQuery{
|
||||
Query: metric.Query{
|
||||
MetricName: "gpu.usage",
|
||||
EntityID: "host-1",
|
||||
Start: base,
|
||||
End: base.Add(time.Minute),
|
||||
Tags: map[string]string{"device_index": "0"},
|
||||
},
|
||||
Aggregation: metric.AggAvg,
|
||||
Interval: time.Minute,
|
||||
}, now)
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
|
||||
fmt.Printf("device 0: count=%d avg=%.1f\n", series[0].Count, series[0].Value)
|
||||
|
||||
// Output:
|
||||
// device 0: count=10 avg=14.5
|
||||
}
|
||||
|
||||
// Example_agentFleetLifecycle demonstrates a realistic Komari-style fleet:
|
||||
// 50 agents report system metrics every second, ping tasks are represented as a
|
||||
// tagged ping metric, and lifecycle events clean up task and agent series.
|
||||
//
|
||||
// Example_agentFleetLifecycle 演示贴近 Komari 的实际场景:50 个 agent 每秒上报系统
|
||||
// 指标,ping task 用带标签的 ping 指标表示,并通过生命周期事件清理 task 和 agent 序列。
|
||||
func Example_agentFleetLifecycle() {
|
||||
ctx := context.Background()
|
||||
base := time.Date(2026, 6, 18, 10, 0, 0, 0, time.UTC)
|
||||
|
||||
store, err := metric.Open(ctx, metric.SQLite(
|
||||
"file:metric-agent-fleet?mode=memory&cache=shared",
|
||||
metric.WithRollupPolicy(metric.RollupPolicy{
|
||||
RawRetention: 2 * time.Minute,
|
||||
Tiers: []metric.RollupTier{
|
||||
{Interval: time.Minute, Retention: 24 * time.Hour},
|
||||
{Interval: 5 * time.Minute, Retention: 30 * 24 * time.Hour},
|
||||
},
|
||||
}),
|
||||
))
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
defer store.Close()
|
||||
|
||||
for _, def := range []metric.Definition{
|
||||
{Name: "cpu.usage", Type: metric.TypeGauge, Unit: "%", RetentionDays: 30},
|
||||
{Name: "mem.usage", Type: metric.TypeGauge, Unit: "%", RetentionDays: 30},
|
||||
{Name: "disk.usage", Type: metric.TypeGauge, Unit: "%", RetentionDays: 30},
|
||||
{Name: "swap.usage", Type: metric.TypeGauge, Unit: "%", RetentionDays: 30},
|
||||
{Name: "net.in.bytes", Type: metric.TypeCounter, Unit: "bytes", RetentionDays: 30},
|
||||
{Name: "net.out.bytes", Type: metric.TypeCounter, Unit: "bytes", RetentionDays: 30},
|
||||
{Name: "process.count", Type: metric.TypeGauge, Unit: "count", RetentionDays: 30},
|
||||
{Name: "gpu.usage", Type: metric.TypeGauge, Unit: "%", RetentionDays: 30},
|
||||
{Name: "ping.latency_ms", Type: metric.TypeGauge, Unit: "ms", RetentionDays: 30},
|
||||
} {
|
||||
if err := store.UpsertMetric(ctx, def); err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
agents := makeAgents(50)
|
||||
pingTasks := []string{"pingtask1", "pingtask2"}
|
||||
for sec := 0; sec < 120; sec++ {
|
||||
ts := base.Add(time.Duration(sec) * time.Second)
|
||||
|
||||
switch sec {
|
||||
case 45:
|
||||
// A ping task is deleted by the admin. Because task_id is a tag, one
|
||||
// DeleteSeries removes that task's records for every agent, including
|
||||
// already materialized rollups.
|
||||
if _, err := store.DeleteSeries(ctx, metric.Query{
|
||||
MetricName: "ping.latency_ms",
|
||||
Tags: map[string]string{"task_id": "pingtask2"},
|
||||
}); err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
pingTasks = []string{"pingtask1"}
|
||||
case 70:
|
||||
// The admin clears all ping task records while keeping the task
|
||||
// definitions themselves. Deleting the metric data is the simplest
|
||||
// reset; UpsertMetric recreates the definition before new samples land.
|
||||
if err := store.DeleteMetric(ctx, "ping.latency_ms"); err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
if err := store.UpsertMetric(ctx, metric.Definition{
|
||||
Name: "ping.latency_ms", Type: metric.TypeGauge, Unit: "ms",
|
||||
RetentionDays: 30}); err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
case 85:
|
||||
// An agent is removed. DeleteEntity clears every metric series for
|
||||
// that agent, raw and rollup, across the whole store.
|
||||
if _, err := store.DeleteEntity(ctx, "agent-007"); err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
agents = removeAgent(agents, "agent-007")
|
||||
case 95:
|
||||
// A new agent is added. No schema change is needed; the first write
|
||||
// creates its series naturally.
|
||||
agents = append(agents, "agent-051")
|
||||
}
|
||||
|
||||
points := make([]metric.Point, 0, len(agents)*(7+2+len(pingTasks)))
|
||||
for _, agentID := range agents {
|
||||
points = append(points, systemPoints(agentID, ts, sec)...)
|
||||
for _, gpu := range []string{"0", "1"} {
|
||||
points = append(points, metric.Point{
|
||||
MetricName: "gpu.usage",
|
||||
EntityID: agentID,
|
||||
Timestamp: ts,
|
||||
Value: gpuUsage(agentID, gpu, sec),
|
||||
Tags: map[string]string{"device_index": gpu},
|
||||
})
|
||||
}
|
||||
for _, taskID := range pingTasks {
|
||||
points = append(points, metric.Point{
|
||||
MetricName: "ping.latency_ms",
|
||||
EntityID: agentID,
|
||||
Timestamp: ts,
|
||||
Value: pingLatency(agentID, taskID, sec),
|
||||
Tags: map[string]string{"task_id": taskID},
|
||||
})
|
||||
}
|
||||
}
|
||||
if err := store.WriteBatch(ctx, points); err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
if sec%30 == 29 {
|
||||
if _, err := store.Compact(ctx, ts); err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
now := base.Add(2 * time.Hour)
|
||||
if _, err := store.Compact(ctx, now); err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
|
||||
activeCPU, err := store.Series(ctx, metric.AggregateQuery{
|
||||
Query: metric.Query{
|
||||
MetricName: "cpu.usage",
|
||||
Start: base,
|
||||
End: base.Add(2 * time.Minute),
|
||||
},
|
||||
Aggregation: metric.AggAvg,
|
||||
Interval: time.Minute,
|
||||
}, now)
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
deletedAgentCPU, err := store.Series(ctx, metric.AggregateQuery{
|
||||
Query: metric.Query{
|
||||
MetricName: "cpu.usage",
|
||||
EntityID: "agent-007",
|
||||
Start: base,
|
||||
End: base.Add(2 * time.Minute),
|
||||
},
|
||||
Aggregation: metric.AggAvg,
|
||||
Interval: time.Minute,
|
||||
}, now)
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
task1, err := store.Series(ctx, metric.AggregateQuery{
|
||||
Query: metric.Query{
|
||||
MetricName: "ping.latency_ms",
|
||||
Start: base,
|
||||
End: base.Add(2 * time.Minute),
|
||||
Tags: map[string]string{"task_id": "pingtask1"},
|
||||
},
|
||||
Aggregation: metric.AggAvg,
|
||||
Interval: time.Minute,
|
||||
}, now)
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
task2, err := store.Series(ctx, metric.AggregateQuery{
|
||||
Query: metric.Query{
|
||||
MetricName: "ping.latency_ms",
|
||||
Start: base,
|
||||
End: base.Add(2 * time.Minute),
|
||||
Tags: map[string]string{"task_id": "pingtask2"},
|
||||
},
|
||||
Aggregation: metric.AggAvg,
|
||||
Interval: time.Minute,
|
||||
}, now)
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
|
||||
fmt.Printf("agents now=%d\n", len(agents))
|
||||
fmt.Printf("fleet cpu buckets=%d\n", len(nonEmpty(activeCPU)))
|
||||
fmt.Printf("deleted agent buckets=%d\n", len(nonEmpty(deletedAgentCPU)))
|
||||
fmt.Printf("pingtask1 buckets=%d\n", len(nonEmpty(task1)))
|
||||
fmt.Printf("pingtask2 buckets=%d\n", len(nonEmpty(task2)))
|
||||
|
||||
// Output:
|
||||
// agents now=50
|
||||
// fleet cpu buckets=2
|
||||
// deleted agent buckets=0
|
||||
// pingtask1 buckets=1
|
||||
// pingtask2 buckets=0
|
||||
}
|
||||
|
||||
func makeAgents(n int) []string {
|
||||
out := make([]string, 0, n)
|
||||
for i := 1; i <= n; i++ {
|
||||
out = append(out, fmt.Sprintf("agent-%03d", i))
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func removeAgent(agents []string, deleted string) []string {
|
||||
out := agents[:0]
|
||||
for _, agentID := range agents {
|
||||
if agentID != deleted {
|
||||
out = append(out, agentID)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func systemPoints(agentID string, ts time.Time, sec int) []metric.Point {
|
||||
id := numericSuffix(agentID)
|
||||
return []metric.Point{
|
||||
{MetricName: "cpu.usage", EntityID: agentID, Timestamp: ts, Value: float64((id*3 + sec) % 100)},
|
||||
{MetricName: "mem.usage", EntityID: agentID, Timestamp: ts, Value: 35 + float64((id+sec)%40)},
|
||||
{MetricName: "disk.usage", EntityID: agentID, Timestamp: ts, Value: 50 + float64(id%20)/2},
|
||||
{MetricName: "swap.usage", EntityID: agentID, Timestamp: ts, Value: float64((id + sec/10) % 12)},
|
||||
{MetricName: "net.in.bytes", EntityID: agentID, Timestamp: ts, Value: float64(sec * (1000 + id))},
|
||||
{MetricName: "net.out.bytes", EntityID: agentID, Timestamp: ts, Value: float64(sec * (700 + id))},
|
||||
{MetricName: "process.count", EntityID: agentID, Timestamp: ts, Value: float64(90 + id%25)},
|
||||
}
|
||||
}
|
||||
|
||||
func gpuUsage(agentID, deviceIndex string, sec int) float64 {
|
||||
id := numericSuffix(agentID)
|
||||
device := 0
|
||||
if deviceIndex == "1" {
|
||||
device = 37
|
||||
}
|
||||
return float64((id*5 + device + sec*2) % 100)
|
||||
}
|
||||
|
||||
func pingLatency(agentID, taskID string, sec int) float64 {
|
||||
id := numericSuffix(agentID)
|
||||
offset := 0
|
||||
if strings.HasSuffix(taskID, "2") {
|
||||
offset = 35
|
||||
}
|
||||
return float64(20 + (id+sec+offset)%80)
|
||||
}
|
||||
|
||||
func numericSuffix(agentID string) int {
|
||||
var n int
|
||||
for _, r := range agentID {
|
||||
if r >= '0' && r <= '9' {
|
||||
n = n*10 + int(r-'0')
|
||||
}
|
||||
}
|
||||
return n
|
||||
}
|
||||
|
||||
func nonEmpty(points []metric.AggregatePoint) []metric.AggregatePoint {
|
||||
out := points[:0]
|
||||
for _, p := range points {
|
||||
if p.Count > 0 {
|
||||
out = append(out, p)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
@@ -0,0 +1,620 @@
|
||||
package metric
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"math"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// newMemStore opens an isolated in-memory store for tests.
|
||||
//
|
||||
// newMemStore 打开一个用于测试的隔离内存 Store。
|
||||
func newMemStore(t *testing.T) *Store {
|
||||
t.Helper()
|
||||
store, err := Open(context.Background(), SQLite("file:imp-test?mode=memory&cache=shared"))
|
||||
if err != nil {
|
||||
t.Fatalf("open store: %v", err)
|
||||
}
|
||||
t.Cleanup(func() { store.Close() })
|
||||
return store
|
||||
}
|
||||
|
||||
// TestCreateMetricRejectsDuplicate verifies create-only metric semantics.
|
||||
//
|
||||
// TestCreateMetricRejectsDuplicate 验证 CreateMetric 遇到重复指标时会拒绝。
|
||||
func TestCreateMetricRejectsDuplicate(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
s := newMemStore(t)
|
||||
def := Definition{Name: "dup.metric", Type: TypeGauge, RetentionDays: 30}
|
||||
if err := s.CreateMetric(ctx, def); err != nil {
|
||||
t.Fatalf("first create: %v", err)
|
||||
}
|
||||
err := s.CreateMetric(ctx, def)
|
||||
if !errors.Is(err, ErrAlreadyExists) {
|
||||
t.Fatalf("expected ErrAlreadyExists on duplicate, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// TestUpsertMetricOverwrites verifies upsert updates mutable definition fields.
|
||||
//
|
||||
// TestUpsertMetricOverwrites 验证 UpsertMetric 会更新指标定义的可变字段。
|
||||
func TestUpsertMetricOverwrites(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
s := newMemStore(t)
|
||||
if err := s.CreateMetric(ctx, Definition{Name: "m", Type: TypeGauge, Unit: "ms", RetentionDays: 30}); err != nil {
|
||||
t.Fatalf("create: %v", err)
|
||||
}
|
||||
if err := s.UpsertMetric(ctx, Definition{Name: "m", Type: TypeCounter, Unit: "count", RetentionDays: 30}); err != nil {
|
||||
t.Fatalf("upsert: %v", err)
|
||||
}
|
||||
got, err := s.GetMetric(ctx, "m")
|
||||
if err != nil {
|
||||
t.Fatalf("get: %v", err)
|
||||
}
|
||||
if got.Type != TypeCounter || got.Unit != "count" {
|
||||
t.Fatalf("upsert did not overwrite: %#v", got)
|
||||
}
|
||||
}
|
||||
|
||||
// TestTagFilterPushdownWithPaging verifies tag filtering happens before paging.
|
||||
//
|
||||
// TestTagFilterPushdownWithPaging 验证标签过滤会先于分页在 SQL 中执行。
|
||||
func TestTagFilterPushdownWithPaging(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
s := newMemStore(t)
|
||||
if err := s.CreateMetric(ctx, Definition{Name: "req", Type: TypeCounter, RetentionDays: 30}); err != nil {
|
||||
t.Fatalf("create: %v", err)
|
||||
}
|
||||
base := time.Date(2026, 6, 18, 0, 0, 0, 0, time.UTC)
|
||||
var batch []Point
|
||||
for i := 0; i < 6; i++ {
|
||||
env := "prod"
|
||||
if i%2 == 1 {
|
||||
env = "stage"
|
||||
}
|
||||
batch = append(batch, Point{
|
||||
MetricName: "req", EntityID: "n1",
|
||||
Timestamp: base.Add(time.Duration(i) * time.Minute),
|
||||
Value: float64(i),
|
||||
Tags: map[string]string{"env": env},
|
||||
})
|
||||
}
|
||||
if err := s.WriteBatch(ctx, batch); err != nil {
|
||||
t.Fatalf("write: %v", err)
|
||||
}
|
||||
// 3 prod points (i=0,2,4); with Limit=2 the SQL LIMIT must apply to the
|
||||
// tag-filtered set, returning the first two prod points in time order.
|
||||
got, err := s.Query(ctx, Query{
|
||||
MetricName: "req", EntityID: "n1",
|
||||
Start: base, End: base.Add(time.Hour),
|
||||
Tags: map[string]string{"env": "prod"},
|
||||
Limit: 2,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("query: %v", err)
|
||||
}
|
||||
if len(got) != 2 {
|
||||
t.Fatalf("expected 2 paged prod points, got %d", len(got))
|
||||
}
|
||||
if got[0].Value != 0 || got[1].Value != 2 {
|
||||
t.Fatalf("unexpected paged values: %#v", got)
|
||||
}
|
||||
}
|
||||
|
||||
// TestSQLAggregateMatchesInMemory compares SQL and in-memory aggregation.
|
||||
//
|
||||
// TestSQLAggregateMatchesInMemory 对比 SQL 下推聚合和内存聚合结果。
|
||||
func TestSQLAggregateMatchesInMemory(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
s := newMemStore(t)
|
||||
if err := s.CreateMetric(ctx, Definition{Name: "g", Type: TypeGauge, RetentionDays: 30}); err != nil {
|
||||
t.Fatalf("create: %v", err)
|
||||
}
|
||||
base := time.Date(2026, 6, 18, 0, 0, 0, 0, time.UTC)
|
||||
var batch []Point
|
||||
vals := []float64{5, 15, 25, 35, 100, 200}
|
||||
for i, v := range vals {
|
||||
batch = append(batch, Point{
|
||||
MetricName: "g", EntityID: "n1",
|
||||
Timestamp: base.Add(time.Duration(i) * time.Minute),
|
||||
Value: v,
|
||||
})
|
||||
}
|
||||
if err := s.WriteBatch(ctx, batch); err != nil {
|
||||
t.Fatalf("write: %v", err)
|
||||
}
|
||||
q := AggregateQuery{
|
||||
Query: Query{MetricName: "g", EntityID: "n1", Start: base, End: base.Add(time.Hour)},
|
||||
Aggregation: AggAvg,
|
||||
Interval: 3 * time.Minute,
|
||||
}
|
||||
// SQL pushdown path.
|
||||
sqlRes, err := s.Aggregate(ctx, q)
|
||||
if err != nil {
|
||||
t.Fatalf("sql aggregate: %v", err)
|
||||
}
|
||||
// In-memory reference over the same raw points.
|
||||
pts, err := s.Query(ctx, q.Query)
|
||||
if err != nil {
|
||||
t.Fatalf("query: %v", err)
|
||||
}
|
||||
memRes, err := AggregatePoints(pts, q)
|
||||
if err != nil {
|
||||
t.Fatalf("mem aggregate: %v", err)
|
||||
}
|
||||
if len(sqlRes) != len(memRes) {
|
||||
t.Fatalf("bucket count mismatch: sql=%d mem=%d", len(sqlRes), len(memRes))
|
||||
}
|
||||
for i := range sqlRes {
|
||||
if !sqlRes[i].Bucket.Equal(memRes[i].Bucket) || sqlRes[i].Value != memRes[i].Value || sqlRes[i].Count != memRes[i].Count {
|
||||
t.Fatalf("bucket %d mismatch: sql=%#v mem=%#v", i, sqlRes[i], memRes[i])
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestCounterRateHandlesReset verifies reset-aware counter rate calculation.
|
||||
//
|
||||
// TestCounterRateHandlesReset 验证计数器重置时速率计算仍然稳定。
|
||||
func TestCounterRateHandlesReset(t *testing.T) {
|
||||
base := time.Date(2026, 6, 18, 0, 0, 0, 0, time.UTC)
|
||||
// Counter goes 0 -> 10 -> reset -> 5; naive (last-first)/sec would give a
|
||||
// positive-but-wrong 5/30s. Correct counter rate sums positive deltas:
|
||||
// (10-0) + (5-0 after reset) = 15 over 30s = 0.5/s.
|
||||
pts := []Point{
|
||||
{Timestamp: base, Value: 0},
|
||||
{Timestamp: base.Add(10 * time.Second), Value: 10},
|
||||
{Timestamp: base.Add(20 * time.Second), Value: 0},
|
||||
{Timestamp: base.Add(30 * time.Second), Value: 5},
|
||||
}
|
||||
rate := counterRate(pts)
|
||||
if rate != 0.5 {
|
||||
t.Fatalf("expected reset-aware rate 0.5/s, got %v", rate)
|
||||
}
|
||||
}
|
||||
|
||||
// TestAlignTimeNegativeTimestamp verifies pre-epoch bucket alignment.
|
||||
//
|
||||
// TestAlignTimeNegativeTimestamp 验证 Unix epoch 之前的时间也能正确对齐桶。
|
||||
func TestAlignTimeNegativeTimestamp(t *testing.T) {
|
||||
interval := time.Minute
|
||||
// 30s before the epoch should align down to -60s, not up to 0.
|
||||
tm := time.Unix(-30, 0).UTC()
|
||||
got := alignTime(tm, interval)
|
||||
want := time.Unix(-60, 0).UTC()
|
||||
if !got.Equal(want) {
|
||||
t.Fatalf("alignTime negative: got %v want %v", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
// TestLatestReturnsMostRecent verifies latest-point ordering.
|
||||
//
|
||||
// TestLatestReturnsMostRecent 验证 Latest 返回最新采样点。
|
||||
func TestLatestReturnsMostRecent(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
s := newMemStore(t)
|
||||
if err := s.CreateMetric(ctx, Definition{Name: "l", Type: TypeGauge, RetentionDays: 30}); err != nil {
|
||||
t.Fatalf("create: %v", err)
|
||||
}
|
||||
base := time.Date(2026, 6, 18, 0, 0, 0, 0, time.UTC)
|
||||
for i := 0; i < 5; i++ {
|
||||
if err := s.Write(ctx, Point{MetricName: "l", EntityID: "n1", Timestamp: base.Add(time.Duration(i) * time.Minute), Value: float64(i)}); err != nil {
|
||||
t.Fatalf("write: %v", err)
|
||||
}
|
||||
}
|
||||
got, err := s.Latest(ctx, "l", "n1", 2)
|
||||
if err != nil {
|
||||
t.Fatalf("latest: %v", err)
|
||||
}
|
||||
if len(got) != 2 || got[0].Value != 4 || got[1].Value != 3 {
|
||||
t.Fatalf("unexpected latest: %#v", got)
|
||||
}
|
||||
}
|
||||
|
||||
// TestStatsDistinguishesNoDataFromUnknownMetric verifies stats error semantics.
|
||||
//
|
||||
// TestStatsDistinguishesNoDataFromUnknownMetric 验证 Stats 能区分无数据和未知指标。
|
||||
func TestStatsDistinguishesNoDataFromUnknownMetric(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
s := newMemStore(t)
|
||||
base := time.Date(2026, 6, 18, 0, 0, 0, 0, time.UTC)
|
||||
|
||||
// Unknown metric -> ErrNotFound.
|
||||
_, err := s.Stats(ctx, Query{MetricName: "nope", Start: base, End: base.Add(time.Hour)})
|
||||
if !errors.Is(err, ErrNotFound) {
|
||||
t.Fatalf("unknown metric: expected ErrNotFound, got %v", err)
|
||||
}
|
||||
|
||||
// Known metric but empty window -> ErrNoData.
|
||||
if err := s.CreateMetric(ctx, Definition{Name: "known", Type: TypeGauge, RetentionDays: 30}); err != nil {
|
||||
t.Fatalf("create: %v", err)
|
||||
}
|
||||
_, err = s.Stats(ctx, Query{MetricName: "known", Start: base, End: base.Add(time.Hour)})
|
||||
if !errors.Is(err, ErrNoData) {
|
||||
t.Fatalf("empty window: expected ErrNoData, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// TestStdDevPopMatchesCalculateStats verifies population standard deviation.
|
||||
//
|
||||
// TestStdDevPopMatchesCalculateStats 验证总体标准差与统计摘要一致。
|
||||
func TestStdDevPopMatchesCalculateStats(t *testing.T) {
|
||||
base := time.Date(2026, 6, 18, 0, 0, 0, 0, time.UTC)
|
||||
var pts []Point
|
||||
for i, v := range []float64{10, 20, 30, 40, 50} {
|
||||
pts = append(pts, Point{Timestamp: base.Add(time.Duration(i) * time.Minute), Value: v})
|
||||
}
|
||||
st, err := CalculateStats(pts)
|
||||
if err != nil {
|
||||
t.Fatalf("stats: %v", err)
|
||||
}
|
||||
got, _ := aggregateValue(pts, AggStdDev)
|
||||
if math.Abs(got-st.StdDev) > 1e-9 {
|
||||
t.Fatalf("AggStdDev %v != Stats.StdDev %v", got, st.StdDev)
|
||||
}
|
||||
// Known value: population stddev of 10..50 step 10 is sqrt(200) ~= 14.142135.
|
||||
if math.Abs(got-14.142135623730951) > 1e-9 {
|
||||
t.Fatalf("unexpected population stddev: %v", got)
|
||||
}
|
||||
}
|
||||
|
||||
// TestAggregateStdDevSQLiteUsesMemoryPath verifies SQLite stddev fallback.
|
||||
//
|
||||
// TestAggregateStdDevSQLiteUsesMemoryPath 验证 SQLite 标准差聚合会回退到内存路径。
|
||||
func TestAggregateStdDevSQLiteUsesMemoryPath(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
s := newMemStore(t)
|
||||
if err := s.CreateMetric(ctx, Definition{Name: "sd", Type: TypeGauge, RetentionDays: 30}); err != nil {
|
||||
t.Fatalf("create: %v", err)
|
||||
}
|
||||
base := time.Date(2026, 6, 18, 0, 0, 0, 0, time.UTC)
|
||||
var batch []Point
|
||||
for i, v := range []float64{10, 20, 30} {
|
||||
batch = append(batch, Point{MetricName: "sd", EntityID: "n1", Timestamp: base.Add(time.Duration(i) * time.Minute), Value: v})
|
||||
}
|
||||
if err := s.WriteBatch(ctx, batch); err != nil {
|
||||
t.Fatalf("write: %v", err)
|
||||
}
|
||||
res, err := s.Aggregate(ctx, AggregateQuery{
|
||||
Query: Query{MetricName: "sd", EntityID: "n1", Start: base, End: base.Add(time.Hour)},
|
||||
Aggregation: AggStdDev,
|
||||
Interval: time.Hour,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("aggregate stddev: %v", err)
|
||||
}
|
||||
if len(res) != 1 {
|
||||
t.Fatalf("expected 1 bucket, got %d", len(res))
|
||||
}
|
||||
// Population stddev of {10,20,30} = sqrt(200/3) ~= 8.16496.
|
||||
if math.Abs(res[0].Value-8.16496580927726) > 1e-9 {
|
||||
t.Fatalf("unexpected stddev bucket value: %v", res[0].Value)
|
||||
}
|
||||
}
|
||||
|
||||
// TestSQLAggValueExprPushdownMatrix verifies aggregation pushdown support.
|
||||
//
|
||||
// TestSQLAggValueExprPushdownMatrix 验证各后端支持的聚合下推矩阵。
|
||||
func TestSQLAggValueExprPushdownMatrix(t *testing.T) {
|
||||
cases := []struct {
|
||||
driver Driver
|
||||
agg Aggregation
|
||||
ok bool
|
||||
}{
|
||||
{DriverSQLite, AggAvg, true},
|
||||
{DriverSQLite, AggStdDev, false}, // no STDDEV_POP in sqlite -> memory
|
||||
{DriverSQLite, AggP95, false}, // no percentile in sqlite -> memory
|
||||
{DriverMySQL, AggStdDev, true}, // STDDEV_POP
|
||||
{DriverMySQL, AggP95, false}, // no portable continuous percentile
|
||||
{DriverPostgreSQL, AggStdDev, true},
|
||||
{DriverPostgreSQL, AggP50, true}, // percentile_cont
|
||||
{DriverPostgreSQL, AggP99, true},
|
||||
{DriverPostgreSQL, AggFirst, false}, // needs ordered raw series
|
||||
}
|
||||
for _, c := range cases {
|
||||
expr, ok := sqlAggValueExpr(c.driver, c.agg)
|
||||
if ok != c.ok {
|
||||
t.Fatalf("%s/%s: pushdown=%v want %v (expr=%q)", c.driver, c.agg, ok, c.ok, expr)
|
||||
}
|
||||
if ok && expr == "" {
|
||||
t.Fatalf("%s/%s: pushdown ok but empty expr", c.driver, c.agg)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestSQLiteReadPoolOpens verifies SQLite read-pool creation.
|
||||
//
|
||||
// TestSQLiteReadPoolOpens 验证 SQLite 只读连接池会按配置打开。
|
||||
func TestSQLiteReadPoolOpens(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
dir := filepath.Join(t.TempDir(), "rp")
|
||||
store, err := Open(ctx, SQLiteInDir(dir, WithSQLiteReadPool(4)))
|
||||
if err != nil {
|
||||
t.Fatalf("open with read pool: %v", err)
|
||||
}
|
||||
t.Cleanup(func() { store.Close() })
|
||||
if store.readDB == nil {
|
||||
t.Fatalf("expected a dedicated read pool to be opened")
|
||||
}
|
||||
if store.reader() != store.readDB {
|
||||
t.Fatalf("reader() should return the dedicated read pool")
|
||||
}
|
||||
// Round-trip a write (primary) and a read (read pool).
|
||||
if err := store.CreateMetric(ctx, Definition{Name: "rp", Type: TypeGauge, RetentionDays: 30}); err != nil {
|
||||
t.Fatalf("create: %v", err)
|
||||
}
|
||||
base := time.Date(2026, 6, 18, 0, 0, 0, 0, time.UTC)
|
||||
if err := store.Write(ctx, Point{MetricName: "rp", EntityID: "n1", Timestamp: base, Value: 7}); err != nil {
|
||||
t.Fatalf("write: %v", err)
|
||||
}
|
||||
got, err := store.Latest(ctx, "rp", "n1", 1)
|
||||
if err != nil {
|
||||
t.Fatalf("latest: %v", err)
|
||||
}
|
||||
if len(got) != 1 || got[0].Value != 7 {
|
||||
t.Fatalf("read pool round-trip failed: %#v", got)
|
||||
}
|
||||
}
|
||||
|
||||
// TestMemoryDSNSkipsReadPool verifies memory SQLite skips read pools.
|
||||
//
|
||||
// TestMemoryDSNSkipsReadPool 验证内存 SQLite 不会打开独立读池。
|
||||
func TestMemoryDSNSkipsReadPool(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
store, err := Open(ctx, SQLite("file:rp-mem?mode=memory&cache=shared", WithSQLiteReadPool(4)))
|
||||
if err != nil {
|
||||
t.Fatalf("open: %v", err)
|
||||
}
|
||||
t.Cleanup(func() { store.Close() })
|
||||
if store.readDB != nil {
|
||||
t.Fatalf("in-memory database must not open a second read pool")
|
||||
}
|
||||
}
|
||||
|
||||
// TestWriteBatchAtomicAcrossChunks verifies chunked writes are atomic.
|
||||
//
|
||||
// TestWriteBatchAtomicAcrossChunks 验证分块批量写入仍保持整体原子性。
|
||||
func TestWriteBatchAtomicAcrossChunks(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
s := newMemStore(t)
|
||||
if err := s.CreateMetric(ctx, Definition{Name: "atom", Type: TypeGauge, RetentionDays: 30}); err != nil {
|
||||
t.Fatalf("create: %v", err)
|
||||
}
|
||||
base := time.Date(2026, 6, 18, 0, 0, 0, 0, time.UTC)
|
||||
// 1500 valid points, then one invalid (missing entity) to force a mid-batch
|
||||
// failure on the second chunk. The whole batch must roll back.
|
||||
var batch []Point
|
||||
for i := 0; i < 1500; i++ {
|
||||
batch = append(batch, Point{MetricName: "atom", EntityID: "n1", Timestamp: base.Add(time.Duration(i) * time.Millisecond), Value: float64(i)})
|
||||
}
|
||||
batch = append(batch, Point{MetricName: "atom", EntityID: "", Timestamp: base.Add(2 * time.Second), Value: 1}) // invalid
|
||||
if err := s.WriteBatch(ctx, batch); err == nil {
|
||||
t.Fatalf("expected WriteBatch to fail on invalid point")
|
||||
}
|
||||
// Nothing should have been committed.
|
||||
pts, err := s.Query(ctx, Query{MetricName: "atom", EntityID: "n1", Start: base, End: base.Add(time.Hour)})
|
||||
if err != nil {
|
||||
t.Fatalf("query: %v", err)
|
||||
}
|
||||
if len(pts) != 0 {
|
||||
t.Fatalf("expected 0 committed points after rolled-back batch, got %d", len(pts))
|
||||
}
|
||||
}
|
||||
|
||||
// TestAggregateBucketPagingSQLPath verifies bucket paging in SQL aggregation.
|
||||
//
|
||||
// TestAggregateBucketPagingSQLPath 验证 SQL 聚合路径按桶分页。
|
||||
func TestAggregateBucketPagingSQLPath(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
s := newMemStore(t)
|
||||
if err := s.CreateMetric(ctx, Definition{Name: "bp", Type: TypeGauge, RetentionDays: 30}); err != nil {
|
||||
t.Fatalf("create: %v", err)
|
||||
}
|
||||
base := time.Date(2026, 6, 18, 0, 0, 0, 0, time.UTC)
|
||||
// 6 points across 6 one-minute buckets, values 0..5.
|
||||
var batch []Point
|
||||
for i := 0; i < 6; i++ {
|
||||
batch = append(batch, Point{MetricName: "bp", EntityID: "n1", Timestamp: base.Add(time.Duration(i) * time.Minute), Value: float64(i)})
|
||||
}
|
||||
if err := s.WriteBatch(ctx, batch); err != nil {
|
||||
t.Fatalf("write: %v", err)
|
||||
}
|
||||
// AggAvg -> SQL pushdown path. One bucket per minute. Page: offset 1, limit 2.
|
||||
res, err := s.Aggregate(ctx, AggregateQuery{
|
||||
Query: Query{MetricName: "bp", EntityID: "n1", Start: base, End: base.Add(time.Hour)},
|
||||
Aggregation: AggAvg,
|
||||
Interval: time.Minute,
|
||||
BucketLimit: 2,
|
||||
BucketOffset: 1,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("aggregate: %v", err)
|
||||
}
|
||||
if len(res) != 2 {
|
||||
t.Fatalf("expected 2 paged buckets, got %d", len(res))
|
||||
}
|
||||
// Buckets are values 0..5; offset 1 skips the value-0 bucket, so we expect 1,2.
|
||||
if res[0].Value != 1 || res[1].Value != 2 {
|
||||
t.Fatalf("unexpected paged bucket values: %#v", res)
|
||||
}
|
||||
}
|
||||
|
||||
// TestAggregateBucketPagingMemoryPathMatchesSQL compares bucket paging paths.
|
||||
//
|
||||
// TestAggregateBucketPagingMemoryPathMatchesSQL 对比内存聚合和 SQL 聚合的桶分页语义。
|
||||
func TestAggregateBucketPagingMemoryPathMatchesSQL(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
s := newMemStore(t)
|
||||
if err := s.CreateMetric(ctx, Definition{Name: "bp2", Type: TypeGauge, RetentionDays: 30}); err != nil {
|
||||
t.Fatalf("create: %v", err)
|
||||
}
|
||||
base := time.Date(2026, 6, 18, 0, 0, 0, 0, time.UTC)
|
||||
var batch []Point
|
||||
for i := 0; i < 6; i++ {
|
||||
batch = append(batch, Point{MetricName: "bp2", EntityID: "n1", Timestamp: base.Add(time.Duration(i) * time.Minute), Value: float64(i * 10)})
|
||||
}
|
||||
if err := s.WriteBatch(ctx, batch); err != nil {
|
||||
t.Fatalf("write: %v", err)
|
||||
}
|
||||
q := Query{MetricName: "bp2", EntityID: "n1", Start: base, End: base.Add(time.Hour)}
|
||||
// AggMin (SQL pushdown) and AggP50 (memory fallback on sqlite) with identical
|
||||
// bucket paging must select the same bucket window (the same timestamps).
|
||||
sqlPaged, err := s.Aggregate(ctx, AggregateQuery{Query: q, Aggregation: AggMin, Interval: time.Minute, BucketLimit: 3, BucketOffset: 2})
|
||||
if err != nil {
|
||||
t.Fatalf("sql aggregate: %v", err)
|
||||
}
|
||||
memPaged, err := s.Aggregate(ctx, AggregateQuery{Query: q, Aggregation: AggP50, Interval: time.Minute, BucketLimit: 3, BucketOffset: 2})
|
||||
if err != nil {
|
||||
t.Fatalf("mem aggregate: %v", err)
|
||||
}
|
||||
if len(sqlPaged) != 3 || len(memPaged) != 3 {
|
||||
t.Fatalf("expected 3 buckets each, got sql=%d mem=%d", len(sqlPaged), len(memPaged))
|
||||
}
|
||||
for i := range sqlPaged {
|
||||
if !sqlPaged[i].Bucket.Equal(memPaged[i].Bucket) {
|
||||
t.Fatalf("bucket window mismatch at %d: sql=%v mem=%v", i, sqlPaged[i].Bucket, memPaged[i].Bucket)
|
||||
}
|
||||
}
|
||||
// Per-bucket single point, so min == p50 == the value.
|
||||
for i := range sqlPaged {
|
||||
if sqlPaged[i].Value != memPaged[i].Value {
|
||||
t.Fatalf("value mismatch at %d: sql=%v mem=%v", i, sqlPaged[i].Value, memPaged[i].Value)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestAggregateSeparatesTagSeries(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
s := newMemStore(t)
|
||||
if err := s.CreateMetric(ctx, Definition{Name: "tagged.util", Type: TypeGauge, RetentionDays: 30}); err != nil {
|
||||
t.Fatalf("create: %v", err)
|
||||
}
|
||||
base := time.Date(2026, 6, 18, 0, 0, 0, 0, time.UTC)
|
||||
points := []Point{
|
||||
{MetricName: "tagged.util", EntityID: "n1", Timestamp: base.Add(10 * time.Second), Value: 10, Tags: map[string]string{"device": "0"}},
|
||||
{MetricName: "tagged.util", EntityID: "n1", Timestamp: base.Add(20 * time.Second), Value: 30, Tags: map[string]string{"device": "0"}},
|
||||
{MetricName: "tagged.util", EntityID: "n1", Timestamp: base.Add(10 * time.Second), Value: 100, Tags: map[string]string{"device": "1"}},
|
||||
{MetricName: "tagged.util", EntityID: "n1", Timestamp: base.Add(20 * time.Second), Value: 200, Tags: map[string]string{"device": "1"}},
|
||||
}
|
||||
if err := s.WriteBatch(ctx, points); err != nil {
|
||||
t.Fatalf("write: %v", err)
|
||||
}
|
||||
query := Query{MetricName: "tagged.util", EntityID: "n1", Start: base, End: base.Add(time.Minute)}
|
||||
|
||||
sqlRes, err := s.Aggregate(ctx, AggregateQuery{
|
||||
Query: query,
|
||||
Aggregation: AggAvg,
|
||||
Interval: time.Minute,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("sql aggregate: %v", err)
|
||||
}
|
||||
assertTaggedAggregate(t, sqlRes, "0", 20, 2)
|
||||
assertTaggedAggregate(t, sqlRes, "1", 150, 2)
|
||||
|
||||
memRes, err := s.Aggregate(ctx, AggregateQuery{
|
||||
Query: query,
|
||||
Aggregation: AggP50,
|
||||
Interval: time.Minute,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("memory aggregate: %v", err)
|
||||
}
|
||||
assertTaggedAggregate(t, memRes, "0", 20, 2)
|
||||
assertTaggedAggregate(t, memRes, "1", 150, 2)
|
||||
}
|
||||
|
||||
func assertTaggedAggregate(t *testing.T, points []AggregatePoint, device string, wantValue float64, wantCount int) {
|
||||
t.Helper()
|
||||
for _, point := range points {
|
||||
if point.Tags["device"] != device {
|
||||
continue
|
||||
}
|
||||
if point.Value != wantValue || point.Count != wantCount {
|
||||
t.Fatalf("device %s aggregate = value %v count %d, want value %v count %d; all=%#v", device, point.Value, point.Count, wantValue, wantCount, points)
|
||||
}
|
||||
return
|
||||
}
|
||||
t.Fatalf("missing aggregate for device %s; all=%#v", device, points)
|
||||
}
|
||||
|
||||
// TestAggregateIgnoresRawPointLimit verifies raw limits do not affect aggregation.
|
||||
//
|
||||
// TestAggregateIgnoresRawPointLimit 验证原始点分页参数不会影响聚合输入。
|
||||
func TestAggregateIgnoresRawPointLimit(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
s := newMemStore(t)
|
||||
if err := s.CreateMetric(ctx, Definition{Name: "ig", Type: TypeGauge, RetentionDays: 30}); err != nil {
|
||||
t.Fatalf("create: %v", err)
|
||||
}
|
||||
base := time.Date(2026, 6, 18, 0, 0, 0, 0, time.UTC)
|
||||
var batch []Point
|
||||
for i := 0; i < 6; i++ {
|
||||
batch = append(batch, Point{MetricName: "ig", EntityID: "n1", Timestamp: base.Add(time.Duration(i) * time.Minute), Value: float64(i)})
|
||||
}
|
||||
if err := s.WriteBatch(ctx, batch); err != nil {
|
||||
t.Fatalf("write: %v", err)
|
||||
}
|
||||
// The embedded Query.Limit must be ignored for aggregation: a single 1-hour
|
||||
// bucket should aggregate ALL 6 points (count 6), not just 2.
|
||||
res, err := s.Aggregate(ctx, AggregateQuery{
|
||||
Query: Query{MetricName: "ig", EntityID: "n1", Start: base, End: base.Add(time.Hour), Limit: 2},
|
||||
Aggregation: AggP95, // memory path, where the bug would have surfaced
|
||||
Interval: time.Hour,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("aggregate: %v", err)
|
||||
}
|
||||
if len(res) != 1 || res[0].Count != 6 {
|
||||
t.Fatalf("embedded Limit leaked into aggregation: %#v", res)
|
||||
}
|
||||
}
|
||||
|
||||
// TestJSONTagKeyWithSpecialChars verifies JSON tag keys with special characters.
|
||||
//
|
||||
// TestJSONTagKeyWithSpecialChars 验证包含特殊字符的标签键可正确查询。
|
||||
func TestJSONTagKeyWithSpecialChars(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
s := newMemStore(t)
|
||||
if err := s.CreateMetric(ctx, Definition{Name: "tk", Type: TypeGauge, RetentionDays: 30}); err != nil {
|
||||
t.Fatalf("create: %v", err)
|
||||
}
|
||||
base := time.Date(2026, 6, 18, 0, 0, 0, 0, time.UTC)
|
||||
// Keys containing a dot, a hyphen, a space, and a quote.
|
||||
tags := map[string]string{
|
||||
"region.zone": "ap-1",
|
||||
"a-b": "yes",
|
||||
"with space": "ok",
|
||||
}
|
||||
if err := s.Write(ctx, Point{MetricName: "tk", EntityID: "n1", Timestamp: base, Value: 1, Tags: tags}); err != nil {
|
||||
t.Fatalf("write: %v", err)
|
||||
}
|
||||
// A second point that should NOT match the filter below.
|
||||
if err := s.Write(ctx, Point{MetricName: "tk", EntityID: "n1", Timestamp: base.Add(time.Minute), Value: 2, Tags: map[string]string{"region.zone": "eu-1"}}); err != nil {
|
||||
t.Fatalf("write 2: %v", err)
|
||||
}
|
||||
|
||||
for _, tc := range []struct {
|
||||
key, val string
|
||||
want int
|
||||
}{
|
||||
{"region.zone", "ap-1", 1}, // dotted key must match the flat key, not nested
|
||||
{"a-b", "yes", 1}, // hyphen
|
||||
{"with space", "ok", 1}, // space
|
||||
{"region.zone", "eu-1", 1}, // the other point
|
||||
} {
|
||||
got, err := s.Query(ctx, Query{
|
||||
MetricName: "tk", EntityID: "n1",
|
||||
Start: base.Add(-time.Second), End: base.Add(time.Hour),
|
||||
Tags: map[string]string{tc.key: tc.val},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("query %s=%s: %v", tc.key, tc.val, err)
|
||||
}
|
||||
if len(got) != tc.want {
|
||||
t.Fatalf("tag %q=%q: expected %d points, got %d", tc.key, tc.val, tc.want, len(got))
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,48 @@
|
||||
package metric
|
||||
|
||||
import "time"
|
||||
|
||||
var standardQueryIntervals = [...]time.Duration{
|
||||
time.Second,
|
||||
5 * time.Second,
|
||||
10 * time.Second,
|
||||
15 * time.Second,
|
||||
30 * time.Second,
|
||||
time.Minute,
|
||||
2 * time.Minute,
|
||||
5 * time.Minute,
|
||||
10 * time.Minute,
|
||||
15 * time.Minute,
|
||||
30 * time.Minute,
|
||||
time.Hour,
|
||||
2 * time.Hour,
|
||||
3 * time.Hour,
|
||||
6 * time.Hour,
|
||||
12 * time.Hour,
|
||||
24 * time.Hour,
|
||||
}
|
||||
|
||||
// FloorStandardInterval returns the largest standard query interval that does
|
||||
// not exceed interval. Values below one second use the minimum interval.
|
||||
func FloorStandardInterval(interval time.Duration) time.Duration {
|
||||
selected := standardQueryIntervals[0]
|
||||
for _, candidate := range standardQueryIntervals {
|
||||
if candidate > interval {
|
||||
break
|
||||
}
|
||||
selected = candidate
|
||||
}
|
||||
return selected
|
||||
}
|
||||
|
||||
// CeilStandardInterval returns the smallest standard query interval that is
|
||||
// at least interval. Values above one day are rounded up to whole days.
|
||||
func CeilStandardInterval(interval time.Duration) time.Duration {
|
||||
for _, candidate := range standardQueryIntervals {
|
||||
if candidate >= interval {
|
||||
return candidate
|
||||
}
|
||||
}
|
||||
day := standardQueryIntervals[len(standardQueryIntervals)-1]
|
||||
return ((interval-1)/day + 1) * day
|
||||
}
|
||||
@@ -0,0 +1,31 @@
|
||||
package metric
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestStandardIntervals(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
interval time.Duration
|
||||
floor time.Duration
|
||||
ceil time.Duration
|
||||
}{
|
||||
{name: "below minimum", interval: 500 * time.Millisecond, floor: time.Second, ceil: time.Second},
|
||||
{name: "exact", interval: 5 * time.Minute, floor: 5 * time.Minute, ceil: 5 * time.Minute},
|
||||
{name: "between standards", interval: 7 * time.Minute, floor: 5 * time.Minute, ceil: 10 * time.Minute},
|
||||
{name: "above one day", interval: 25 * time.Hour, floor: 24 * time.Hour, ceil: 48 * time.Hour},
|
||||
}
|
||||
|
||||
for _, test := range tests {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
if got := FloorStandardInterval(test.interval); got != test.floor {
|
||||
t.Fatalf("FloorStandardInterval(%s) = %s, want %s", test.interval, got, test.floor)
|
||||
}
|
||||
if got := CeilStandardInterval(test.interval); got != test.ceil {
|
||||
t.Fatalf("CeilStandardInterval(%s) = %s, want %s", test.interval, got, test.ceil)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,83 @@
|
||||
package metric
|
||||
|
||||
import (
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
)
|
||||
|
||||
// encodeMap encodes a string map as JSON.
|
||||
//
|
||||
// encodeMap 将字符串 map 编码为 JSON 字符串,nil 会按空对象处理。
|
||||
func encodeMap(m map[string]string) (string, error) {
|
||||
if m == nil {
|
||||
m = map[string]string{}
|
||||
}
|
||||
b, err := json.Marshal(m)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return string(b), nil
|
||||
}
|
||||
|
||||
// decodeMap decodes a scanned JSON value into a string map.
|
||||
//
|
||||
// decodeMap 将数据库扫描出的 JSON 值还原为字符串 map。
|
||||
func decodeMap(v any) (map[string]string, error) {
|
||||
switch x := v.(type) {
|
||||
case nil:
|
||||
return map[string]string{}, nil
|
||||
case string:
|
||||
return decodeMapString(x)
|
||||
case []byte:
|
||||
return decodeMapString(string(x))
|
||||
default:
|
||||
return nil, fmt.Errorf("unsupported json value type %T", v)
|
||||
}
|
||||
}
|
||||
|
||||
// decodeMapString decodes a JSON object string into a string map.
|
||||
//
|
||||
// decodeMapString 将 JSON 字符串解码为字符串 map。
|
||||
func decodeMapString(s string) (map[string]string, error) {
|
||||
if s == "" {
|
||||
return map[string]string{}, nil
|
||||
}
|
||||
var out map[string]string
|
||||
if err := json.Unmarshal([]byte(s), &out); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if out == nil {
|
||||
out = map[string]string{}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// canonicalTags returns a deterministic JSON encoding of a tag map. encodeMap
|
||||
// already relies on encoding/json, which sorts map keys, so equal tag maps
|
||||
// always produce identical bytes — the property tagsFingerprint depends on.
|
||||
//
|
||||
// canonicalTags 返回标签 map 的确定性 JSON 编码;相同标签集合总会得到
|
||||
// 相同字节序列,tagsFingerprint 依赖这个性质。
|
||||
func canonicalTags(m map[string]string) (string, error) {
|
||||
return encodeMap(m)
|
||||
}
|
||||
|
||||
// tagsFingerprint returns a stable hex fingerprint of a tag set, used as the
|
||||
// rollups table's tags_hash key column. Equal tag maps (regardless of Go map
|
||||
// iteration order) hash identically, so each distinct tag combination becomes
|
||||
// its own rollup series; an empty/nil tag map hashes to the fingerprint of "{}".
|
||||
//
|
||||
// tagsFingerprint 为标签集合生成稳定的十六进制指纹,用作 rollups 表的
|
||||
// tags_hash key 列。相同标签 map(无论 Go map 迭代顺序如何)都会得到相同 hash,
|
||||
// 因此每种不同标签组合都会成为自己的 rollup 序列;空或 nil 标签 map 会得到 "{}"
|
||||
// 的指纹。
|
||||
func tagsFingerprint(m map[string]string) (hash string, canonical string, err error) {
|
||||
canonical, err = canonicalTags(m)
|
||||
if err != nil {
|
||||
return "", "", err
|
||||
}
|
||||
sum := sha256.Sum256([]byte(canonical))
|
||||
return hex.EncodeToString(sum[:]), canonical, nil
|
||||
}
|
||||
@@ -0,0 +1,305 @@
|
||||
package metric
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// MaintenanceAction identifies the backend-specific operation used to reclaim
|
||||
// physical database space.
|
||||
//
|
||||
// MaintenanceAction 表示回收数据库物理空间时使用的后端专用操作。
|
||||
type MaintenanceAction string
|
||||
|
||||
const (
|
||||
// MaintenanceVacuum is SQLite's VACUUM operation.
|
||||
MaintenanceVacuum MaintenanceAction = "vacuum"
|
||||
// MaintenanceOptimize is MySQL's OPTIMIZE TABLE operation.
|
||||
MaintenanceOptimize MaintenanceAction = "optimize"
|
||||
// MaintenanceVacuumFull is PostgreSQL's blocking VACUUM FULL operation.
|
||||
MaintenanceVacuumFull MaintenanceAction = "vacuum_full"
|
||||
)
|
||||
|
||||
const (
|
||||
sqliteCheckpointSQL = "PRAGMA wal_checkpoint(TRUNCATE)"
|
||||
sqliteVacuumSQL = "VACUUM"
|
||||
)
|
||||
|
||||
// Driver returns the Store's configured database backend.
|
||||
//
|
||||
// Driver 返回 Store 配置的数据库后端。
|
||||
func (s *Store) Driver() Driver {
|
||||
return s.cfg.Driver
|
||||
}
|
||||
|
||||
// MaintenanceAction returns the physical-space reclamation operation used by
|
||||
// the Store's backend.
|
||||
//
|
||||
// MaintenanceAction 返回当前后端用于回收物理空间的操作。
|
||||
func (s *Store) MaintenanceAction() MaintenanceAction {
|
||||
return maintenanceActionFor(s.cfg.Driver)
|
||||
}
|
||||
|
||||
// StorageSize returns the physical bytes occupied by this Store. SQLite
|
||||
// includes the main database, WAL, and shared-memory files. Server backends
|
||||
// include only the definitions, points, and rollups tables managed by Store.
|
||||
//
|
||||
// StorageSize 返回当前 Store 占用的物理字节数。SQLite 会统计主数据库、WAL
|
||||
// 和共享内存文件;服务端数据库只统计 Store 管理的 definitions、points 和
|
||||
// rollups 三张表。
|
||||
func (s *Store) StorageSize(ctx context.Context) (int64, error) {
|
||||
s.maintenanceMu.RLock()
|
||||
defer s.maintenanceMu.RUnlock()
|
||||
|
||||
s.mu.RLock()
|
||||
defer s.mu.RUnlock()
|
||||
if s.closed || s.db == nil {
|
||||
return 0, ErrClosed
|
||||
}
|
||||
|
||||
if s.cfg.Driver == DriverSQLite {
|
||||
return s.sqliteStorageSize(ctx)
|
||||
}
|
||||
|
||||
query, args, err := managedStorageSizeQuery(s.cfg.Driver, s.tables)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
var size int64
|
||||
if err := s.db.QueryRowContext(ctx, query, args...).Scan(&size); err != nil {
|
||||
return 0, fmt.Errorf("metric: query %s storage size: %w", s.cfg.Driver, err)
|
||||
}
|
||||
return size, nil
|
||||
}
|
||||
|
||||
// ReclaimSpace performs the backend-specific blocking operation that returns
|
||||
// unused database pages to the filesystem. It serializes against other
|
||||
// maintenance calls and keeps Close from closing the pool mid-operation.
|
||||
//
|
||||
// ReclaimSpace 执行后端专用的阻塞式空间回收操作。该方法会与其他维护调用
|
||||
// 串行执行,并阻止 Close 在操作过程中关闭连接池。
|
||||
func (s *Store) ReclaimSpace(ctx context.Context) error {
|
||||
s.maintenanceMu.Lock()
|
||||
defer s.maintenanceMu.Unlock()
|
||||
|
||||
s.mu.RLock()
|
||||
defer s.mu.RUnlock()
|
||||
if s.closed || s.db == nil {
|
||||
return ErrClosed
|
||||
}
|
||||
|
||||
switch s.cfg.Driver {
|
||||
case DriverSQLite:
|
||||
if err := sqliteCheckpoint(ctx, s.db); err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := s.db.ExecContext(ctx, sqliteVacuumSQL); err != nil {
|
||||
return fmt.Errorf("metric: vacuum sqlite database: %w", err)
|
||||
}
|
||||
// VACUUM itself can populate the WAL; truncate it again so the reported
|
||||
// physical size reflects the completed reclamation.
|
||||
return sqliteCheckpoint(ctx, s.db)
|
||||
case DriverMySQL:
|
||||
query, err := managedReclaimQuery(s.cfg.Driver, s.tables)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return s.optimizeMySQLTables(ctx, query)
|
||||
case DriverPostgreSQL:
|
||||
query, err := managedReclaimQuery(s.cfg.Driver, s.tables)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := s.db.ExecContext(ctx, query); err != nil {
|
||||
return fmt.Errorf("metric: vacuum PostgreSQL metric tables: %w", err)
|
||||
}
|
||||
return nil
|
||||
default:
|
||||
return fmt.Errorf("%w: unsupported driver %q", ErrInvalidArgument, s.cfg.Driver)
|
||||
}
|
||||
}
|
||||
|
||||
func (s *Store) sqliteStorageSize(ctx context.Context) (int64, error) {
|
||||
rows, err := s.db.QueryContext(ctx, "PRAGMA database_list")
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("metric: list sqlite databases: %w", err)
|
||||
}
|
||||
|
||||
var path string
|
||||
for rows.Next() {
|
||||
var sequence int
|
||||
var name, file string
|
||||
if err := rows.Scan(&sequence, &name, &file); err != nil {
|
||||
_ = rows.Close()
|
||||
return 0, fmt.Errorf("metric: scan sqlite database path: %w", err)
|
||||
}
|
||||
if name == "main" {
|
||||
path = file
|
||||
}
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
_ = rows.Close()
|
||||
return 0, fmt.Errorf("metric: list sqlite databases: %w", err)
|
||||
}
|
||||
if err := rows.Close(); err != nil {
|
||||
return 0, fmt.Errorf("metric: close sqlite database list: %w", err)
|
||||
}
|
||||
if path == "" {
|
||||
// SQLite reports an empty path for in-memory databases.
|
||||
return 0, nil
|
||||
}
|
||||
|
||||
var size int64
|
||||
for _, name := range []string{path, path + "-wal", path + "-shm"} {
|
||||
info, err := os.Stat(name)
|
||||
switch {
|
||||
case err == nil:
|
||||
size += info.Size()
|
||||
case errors.Is(err, os.ErrNotExist):
|
||||
default:
|
||||
return 0, fmt.Errorf("metric: stat sqlite storage file %q: %w", name, err)
|
||||
}
|
||||
}
|
||||
return size, nil
|
||||
}
|
||||
|
||||
func sqliteCheckpoint(ctx context.Context, db *sql.DB) error {
|
||||
var busy, logFrames, checkpointedFrames int
|
||||
if err := db.QueryRowContext(ctx, sqliteCheckpointSQL).Scan(&busy, &logFrames, &checkpointedFrames); err != nil {
|
||||
return fmt.Errorf("metric: checkpoint sqlite WAL: %w", err)
|
||||
}
|
||||
if busy != 0 {
|
||||
return fmt.Errorf("metric: checkpoint sqlite WAL: database is busy (%d log frames, %d checkpointed)", logFrames, checkpointedFrames)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *Store) optimizeMySQLTables(ctx context.Context, query string) error {
|
||||
rows, err := s.db.QueryContext(ctx, query)
|
||||
if err != nil {
|
||||
return fmt.Errorf("metric: optimize MySQL metric tables: %w", err)
|
||||
}
|
||||
|
||||
var resultErrors []error
|
||||
for rows.Next() {
|
||||
var table, operation, messageType, message string
|
||||
if err := rows.Scan(&table, &operation, &messageType, &message); err != nil {
|
||||
_ = rows.Close()
|
||||
return fmt.Errorf("metric: scan MySQL optimize result: %w", err)
|
||||
}
|
||||
if err := mysqlOptimizeResultError(table, messageType, message); err != nil {
|
||||
resultErrors = append(resultErrors, err)
|
||||
}
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
_ = rows.Close()
|
||||
return fmt.Errorf("metric: read MySQL optimize results: %w", err)
|
||||
}
|
||||
if err := rows.Close(); err != nil {
|
||||
return fmt.Errorf("metric: close MySQL optimize results: %w", err)
|
||||
}
|
||||
return errors.Join(resultErrors...)
|
||||
}
|
||||
|
||||
func mysqlOptimizeResultError(table, messageType, message string) error {
|
||||
if !strings.EqualFold(strings.TrimSpace(messageType), "error") {
|
||||
return nil
|
||||
}
|
||||
table = strings.TrimSpace(table)
|
||||
if table == "" {
|
||||
table = "unknown table"
|
||||
}
|
||||
message = strings.TrimSpace(message)
|
||||
if message == "" {
|
||||
message = "unknown error"
|
||||
}
|
||||
return fmt.Errorf("metric: optimize MySQL table %s: %s", table, message)
|
||||
}
|
||||
|
||||
func maintenanceActionFor(driver Driver) MaintenanceAction {
|
||||
switch driver {
|
||||
case DriverSQLite:
|
||||
return MaintenanceVacuum
|
||||
case DriverMySQL:
|
||||
return MaintenanceOptimize
|
||||
case DriverPostgreSQL:
|
||||
return MaintenanceVacuumFull
|
||||
default:
|
||||
return ""
|
||||
}
|
||||
}
|
||||
|
||||
func managedStorageSizeQuery(driver Driver, t tables) (string, []any, error) {
|
||||
names := managedTableNames(t)
|
||||
d := newDialect(driver)
|
||||
placeholders := make([]string, len(names))
|
||||
for i := range names {
|
||||
placeholders[i] = d.placeholder(i + 1)
|
||||
}
|
||||
inClause := strings.Join(placeholders, ", ")
|
||||
switch driver {
|
||||
case DriverMySQL:
|
||||
return fmt.Sprintf(`SELECT COALESCE(SUM(DATA_LENGTH + INDEX_LENGTH), 0)
|
||||
FROM information_schema.TABLES
|
||||
WHERE TABLE_SCHEMA = DATABASE()
|
||||
AND TABLE_NAME IN (%s)`, inClause), stringsToAny(names), nil
|
||||
case DriverPostgreSQL:
|
||||
for i := range names {
|
||||
names[i] = strings.ToLower(names[i])
|
||||
}
|
||||
return fmt.Sprintf(`SELECT COALESCE(SUM(pg_total_relation_size(c.oid)), 0)
|
||||
FROM pg_catalog.pg_class AS c
|
||||
JOIN pg_catalog.pg_namespace AS n ON n.oid = c.relnamespace
|
||||
WHERE n.nspname = current_schema()
|
||||
AND c.relname IN (%s)`, inClause), stringsToAny(names), nil
|
||||
default:
|
||||
return "", nil, fmt.Errorf("%w: storage-size query is unavailable for driver %q", ErrInvalidArgument, driver)
|
||||
}
|
||||
}
|
||||
|
||||
func managedReclaimQuery(driver Driver, t tables) (string, error) {
|
||||
names := managedTableNames(t)
|
||||
switch driver {
|
||||
case DriverSQLite:
|
||||
return sqliteVacuumSQL, nil
|
||||
case DriverMySQL:
|
||||
for i := range names {
|
||||
names[i] = quoteMaintenanceIdentifier(driver, names[i])
|
||||
}
|
||||
return "OPTIMIZE TABLE " + strings.Join(names, ", "), nil
|
||||
case DriverPostgreSQL:
|
||||
for i := range names {
|
||||
names[i] = quoteMaintenanceIdentifier(driver, strings.ToLower(names[i]))
|
||||
}
|
||||
return "VACUUM (FULL, ANALYZE) " + strings.Join(names, ", "), nil
|
||||
default:
|
||||
return "", fmt.Errorf("%w: reclaim query is unavailable for driver %q", ErrInvalidArgument, driver)
|
||||
}
|
||||
}
|
||||
|
||||
func managedTableNames(t tables) []string {
|
||||
names := []string{t.definitions, t.points, t.rollups}
|
||||
if t.watermarks != "" {
|
||||
names = append(names, t.watermarks)
|
||||
}
|
||||
return names
|
||||
}
|
||||
|
||||
func quoteMaintenanceIdentifier(driver Driver, identifier string) string {
|
||||
if driver == DriverMySQL {
|
||||
return "`" + strings.ReplaceAll(identifier, "`", "``") + "`"
|
||||
}
|
||||
return `"` + strings.ReplaceAll(identifier, `"`, `""`) + `"`
|
||||
}
|
||||
|
||||
func stringsToAny(values []string) []any {
|
||||
args := make([]any, len(values))
|
||||
for i := range values {
|
||||
args[i] = values[i]
|
||||
}
|
||||
return args
|
||||
}
|
||||
@@ -0,0 +1,182 @@
|
||||
package metric
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestSQLiteStorageSizeAndReclaimSpace(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
dir := t.TempDir()
|
||||
store, err := Open(ctx, SQLiteInDir(dir, WithSQLiteWALAutoCheckpoint(1_000_000)))
|
||||
if err != nil {
|
||||
t.Fatalf("open sqlite store: %v", err)
|
||||
}
|
||||
t.Cleanup(func() { _ = store.Close() })
|
||||
|
||||
if got := store.Driver(); got != DriverSQLite {
|
||||
t.Fatalf("Driver() = %q, want %q", got, DriverSQLite)
|
||||
}
|
||||
if got := store.MaintenanceAction(); got != MaintenanceVacuum {
|
||||
t.Fatalf("MaintenanceAction() = %q, want %q", got, MaintenanceVacuum)
|
||||
}
|
||||
|
||||
if _, err := store.db.ExecContext(ctx, `CREATE TABLE reclaim_fixture (payload BLOB NOT NULL)`); err != nil {
|
||||
t.Fatalf("create reclaim fixture: %v", err)
|
||||
}
|
||||
if _, err := store.db.ExecContext(ctx, `INSERT INTO reclaim_fixture (payload) VALUES (zeroblob(4194304))`); err != nil {
|
||||
t.Fatalf("populate reclaim fixture: %v", err)
|
||||
}
|
||||
if _, err := store.db.ExecContext(ctx, `DROP TABLE reclaim_fixture`); err != nil {
|
||||
t.Fatalf("drop reclaim fixture: %v", err)
|
||||
}
|
||||
|
||||
before, err := store.StorageSize(ctx)
|
||||
if err != nil {
|
||||
t.Fatalf("storage size before reclaim: %v", err)
|
||||
}
|
||||
path := filepath.Join(dir, "metrics.db")
|
||||
if want := sqliteFileSetSize(t, path); before != want {
|
||||
t.Fatalf("StorageSize() = %d, file sum = %d", before, want)
|
||||
}
|
||||
|
||||
if err := store.ReclaimSpace(ctx); err != nil {
|
||||
t.Fatalf("reclaim sqlite space: %v", err)
|
||||
}
|
||||
after, err := store.StorageSize(ctx)
|
||||
if err != nil {
|
||||
t.Fatalf("storage size after reclaim: %v", err)
|
||||
}
|
||||
if want := sqliteFileSetSize(t, path); after != want {
|
||||
t.Fatalf("StorageSize() after reclaim = %d, file sum = %d", after, want)
|
||||
}
|
||||
if after >= before {
|
||||
t.Fatalf("reclaim did not reduce physical storage: before=%d after=%d", before, after)
|
||||
}
|
||||
if err := store.Ping(ctx); err != nil {
|
||||
t.Fatalf("store unusable after reclaim: %v", err)
|
||||
}
|
||||
|
||||
if err := store.Close(); err != nil {
|
||||
t.Fatalf("close store: %v", err)
|
||||
}
|
||||
if _, err := store.StorageSize(ctx); !errors.Is(err, ErrClosed) {
|
||||
t.Fatalf("StorageSize() after Close error = %v, want ErrClosed", err)
|
||||
}
|
||||
if err := store.ReclaimSpace(ctx); !errors.Is(err, ErrClosed) {
|
||||
t.Fatalf("ReclaimSpace() after Close error = %v, want ErrClosed", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMaintenanceMappings(t *testing.T) {
|
||||
tables := tables{
|
||||
definitions: "Metric_definitions",
|
||||
points: "Metric_points",
|
||||
rollups: "Metric_rollups",
|
||||
watermarks: "Metric_compaction_watermarks",
|
||||
}
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
driver Driver
|
||||
action MaintenanceAction
|
||||
reclaim string
|
||||
sizeParts []string
|
||||
sizeArgs []any
|
||||
hasSizeSQL bool
|
||||
}{
|
||||
{
|
||||
name: "sqlite",
|
||||
driver: DriverSQLite,
|
||||
action: MaintenanceVacuum,
|
||||
reclaim: "VACUUM",
|
||||
hasSizeSQL: false,
|
||||
},
|
||||
{
|
||||
name: "mysql",
|
||||
driver: DriverMySQL,
|
||||
action: MaintenanceOptimize,
|
||||
reclaim: "OPTIMIZE TABLE `Metric_definitions`, `Metric_points`, `Metric_rollups`, `Metric_compaction_watermarks`",
|
||||
sizeParts: []string{"information_schema.TABLES", "TABLE_SCHEMA = DATABASE()", "TABLE_NAME IN (?, ?, ?, ?)"},
|
||||
sizeArgs: []any{"Metric_definitions", "Metric_points", "Metric_rollups", "Metric_compaction_watermarks"},
|
||||
hasSizeSQL: true,
|
||||
},
|
||||
{
|
||||
name: "postgresql",
|
||||
driver: DriverPostgreSQL,
|
||||
action: MaintenanceVacuumFull,
|
||||
reclaim: `VACUUM (FULL, ANALYZE) "metric_definitions", "metric_points", "metric_rollups", "metric_compaction_watermarks"`,
|
||||
sizeParts: []string{"pg_total_relation_size(c.oid)", "n.nspname = current_schema()", "c.relname IN ($1, $2, $3, $4)"},
|
||||
sizeArgs: []any{"metric_definitions", "metric_points", "metric_rollups", "metric_compaction_watermarks"},
|
||||
hasSizeSQL: true,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
if got := maintenanceActionFor(tt.driver); got != tt.action {
|
||||
t.Fatalf("maintenanceActionFor(%q) = %q, want %q", tt.driver, got, tt.action)
|
||||
}
|
||||
gotReclaim, err := managedReclaimQuery(tt.driver, tables)
|
||||
if err != nil {
|
||||
t.Fatalf("managedReclaimQuery(%q): %v", tt.driver, err)
|
||||
}
|
||||
if gotReclaim != tt.reclaim {
|
||||
t.Fatalf("managedReclaimQuery(%q) = %q, want %q", tt.driver, gotReclaim, tt.reclaim)
|
||||
}
|
||||
|
||||
gotSize, gotArgs, err := managedStorageSizeQuery(tt.driver, tables)
|
||||
if !tt.hasSizeSQL {
|
||||
if err == nil {
|
||||
t.Fatalf("managedStorageSizeQuery(%q) unexpectedly succeeded: %q", tt.driver, gotSize)
|
||||
}
|
||||
return
|
||||
}
|
||||
if err != nil {
|
||||
t.Fatalf("managedStorageSizeQuery(%q): %v", tt.driver, err)
|
||||
}
|
||||
for _, part := range tt.sizeParts {
|
||||
if !strings.Contains(gotSize, part) {
|
||||
t.Errorf("size query for %q does not contain %q: %s", tt.driver, part, gotSize)
|
||||
}
|
||||
}
|
||||
if !reflect.DeepEqual(gotArgs, tt.sizeArgs) {
|
||||
t.Fatalf("size args for %q = %#v, want %#v", tt.driver, gotArgs, tt.sizeArgs)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestMySQLOptimizeResultError(t *testing.T) {
|
||||
if err := mysqlOptimizeResultError("metric_points", "status", "OK"); err != nil {
|
||||
t.Fatalf("status result returned an error: %v", err)
|
||||
}
|
||||
if err := mysqlOptimizeResultError("metric_points", "note", "recreate and analyze instead"); err != nil {
|
||||
t.Fatalf("note result returned an error: %v", err)
|
||||
}
|
||||
err := mysqlOptimizeResultError("komari.metric_points", " Error ", "operation failed")
|
||||
if err == nil || !strings.Contains(err.Error(), "komari.metric_points") || !strings.Contains(err.Error(), "operation failed") {
|
||||
t.Fatalf("error result was not preserved: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func sqliteFileSetSize(t *testing.T, path string) int64 {
|
||||
t.Helper()
|
||||
var size int64
|
||||
for _, name := range []string{path, path + "-wal", path + "-shm"} {
|
||||
info, err := os.Stat(name)
|
||||
if errors.Is(err, os.ErrNotExist) {
|
||||
continue
|
||||
}
|
||||
if err != nil {
|
||||
t.Fatalf("stat %q: %v", name, err)
|
||||
}
|
||||
size += info.Size()
|
||||
}
|
||||
return size
|
||||
}
|
||||
@@ -0,0 +1,197 @@
|
||||
package metric
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
)
|
||||
|
||||
// Migrate creates the tables and indexes used by the metric store.
|
||||
//
|
||||
// Migrate 创建 metric 包需要的表和索引。
|
||||
func (s *Store) Migrate(ctx context.Context) error {
|
||||
if err := s.ensureOpen(); err != nil {
|
||||
return err
|
||||
}
|
||||
d := s.dialect
|
||||
jsonType := d.jsonType()
|
||||
pk := d.autoIncrementPrimaryKey()
|
||||
blob := d.blobType()
|
||||
|
||||
statements := []string{
|
||||
fmt.Sprintf(`CREATE TABLE IF NOT EXISTS %s (
|
||||
name VARCHAR(191) PRIMARY KEY,
|
||||
type VARCHAR(32) NOT NULL,
|
||||
unit VARCHAR(64) NOT NULL DEFAULT '',
|
||||
description TEXT NOT NULL DEFAULT '',
|
||||
retention_days INTEGER NOT NULL DEFAULT 0,
|
||||
metadata %s NOT NULL,
|
||||
created_at BIGINT NOT NULL,
|
||||
updated_at BIGINT NOT NULL
|
||||
)`, s.tables.definitions, jsonType),
|
||||
fmt.Sprintf(`CREATE TABLE IF NOT EXISTS %s (
|
||||
id %s,
|
||||
metric_name VARCHAR(191) NOT NULL,
|
||||
entity_id VARCHAR(191) NOT NULL,
|
||||
tags_hash VARCHAR(64) NOT NULL,
|
||||
ts_nano BIGINT NOT NULL,
|
||||
value DOUBLE PRECISION NOT NULL,
|
||||
tags %s NOT NULL,
|
||||
labels %s NOT NULL,
|
||||
created_at BIGINT NOT NULL,
|
||||
UNIQUE(metric_name, entity_id, tags_hash, ts_nano)
|
||||
)`, s.tables.points, pk, jsonType, jsonType),
|
||||
fmt.Sprintf(`CREATE INDEX IF NOT EXISTS %s_points_metric_entity_time_idx ON %s (metric_name, entity_id, ts_nano)`, s.cfg.TablePrefix, s.tables.points),
|
||||
fmt.Sprintf(`CREATE INDEX IF NOT EXISTS %s_points_metric_time_idx ON %s (metric_name, ts_nano)`, s.cfg.TablePrefix, s.tables.points),
|
||||
fmt.Sprintf(`CREATE INDEX IF NOT EXISTS %s_points_entity_time_idx ON %s (entity_id, ts_nano)`, s.cfg.TablePrefix, s.tables.points),
|
||||
fmt.Sprintf(`CREATE TABLE IF NOT EXISTS %s (
|
||||
id %s,
|
||||
metric_name VARCHAR(191) NOT NULL,
|
||||
entity_id VARCHAR(191) NOT NULL,
|
||||
tags_hash VARCHAR(64) NOT NULL,
|
||||
tags %s NOT NULL,
|
||||
resolution_nano BIGINT NOT NULL,
|
||||
bucket_nano BIGINT NOT NULL,
|
||||
count BIGINT NOT NULL,
|
||||
sum DOUBLE PRECISION NOT NULL,
|
||||
sum_sq DOUBLE PRECISION NOT NULL,
|
||||
min_val DOUBLE PRECISION NOT NULL,
|
||||
max_val DOUBLE PRECISION NOT NULL,
|
||||
first_val DOUBLE PRECISION NOT NULL,
|
||||
first_ts BIGINT NOT NULL,
|
||||
last_val DOUBLE PRECISION NOT NULL,
|
||||
last_ts BIGINT NOT NULL,
|
||||
digest %s,
|
||||
created_at BIGINT NOT NULL,
|
||||
UNIQUE(metric_name, entity_id, tags_hash, resolution_nano, bucket_nano)
|
||||
)`, s.tables.rollups, pk, jsonType, blob),
|
||||
fmt.Sprintf(`CREATE INDEX IF NOT EXISTS %s_rollups_lookup_idx ON %s (metric_name, entity_id, tags_hash, resolution_nano, bucket_nano)`, s.cfg.TablePrefix, s.tables.rollups),
|
||||
fmt.Sprintf(`CREATE INDEX IF NOT EXISTS %s_rollups_res_time_idx ON %s (metric_name, resolution_nano, bucket_nano)`, s.cfg.TablePrefix, s.tables.rollups),
|
||||
fmt.Sprintf(`CREATE TABLE IF NOT EXISTS %s (
|
||||
metric_name VARCHAR(191) PRIMARY KEY,
|
||||
watermark_nano BIGINT NOT NULL,
|
||||
updated_at BIGINT NOT NULL
|
||||
)`, s.tables.watermarks),
|
||||
}
|
||||
|
||||
if s.cfg.Driver == DriverMySQL {
|
||||
statements = []string{
|
||||
fmt.Sprintf(`CREATE TABLE IF NOT EXISTS %s (
|
||||
name VARCHAR(191) PRIMARY KEY,
|
||||
type VARCHAR(32) NOT NULL,
|
||||
unit VARCHAR(64) NOT NULL DEFAULT '',
|
||||
description TEXT NOT NULL,
|
||||
retention_days INT NOT NULL DEFAULT 0,
|
||||
metadata %s NOT NULL,
|
||||
created_at BIGINT NOT NULL,
|
||||
updated_at BIGINT NOT NULL
|
||||
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci`, s.tables.definitions, jsonType),
|
||||
fmt.Sprintf(`CREATE TABLE IF NOT EXISTS %s (
|
||||
id %s,
|
||||
metric_name VARCHAR(191) NOT NULL,
|
||||
entity_id VARCHAR(191) NOT NULL,
|
||||
tags_hash VARCHAR(64) NOT NULL,
|
||||
ts_nano BIGINT NOT NULL,
|
||||
value DOUBLE NOT NULL,
|
||||
tags %s NOT NULL,
|
||||
labels %s NOT NULL,
|
||||
created_at BIGINT NOT NULL,
|
||||
UNIQUE KEY uq_metric_entity_time (metric_name, entity_id, tags_hash, ts_nano),
|
||||
INDEX idx_metric_entity_time (metric_name, entity_id, ts_nano),
|
||||
INDEX idx_metric_time (metric_name, ts_nano),
|
||||
INDEX idx_entity_time (entity_id, ts_nano)
|
||||
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci`, s.tables.points, pk, jsonType, jsonType),
|
||||
fmt.Sprintf(`CREATE TABLE IF NOT EXISTS %s (
|
||||
id %s,
|
||||
metric_name VARCHAR(191) NOT NULL,
|
||||
entity_id VARCHAR(191) NOT NULL,
|
||||
tags_hash VARCHAR(64) NOT NULL,
|
||||
tags %s NOT NULL,
|
||||
resolution_nano BIGINT NOT NULL,
|
||||
bucket_nano BIGINT NOT NULL,
|
||||
count BIGINT NOT NULL,
|
||||
sum DOUBLE NOT NULL,
|
||||
sum_sq DOUBLE NOT NULL,
|
||||
min_val DOUBLE NOT NULL,
|
||||
max_val DOUBLE NOT NULL,
|
||||
first_val DOUBLE NOT NULL,
|
||||
first_ts BIGINT NOT NULL,
|
||||
last_val DOUBLE NOT NULL,
|
||||
last_ts BIGINT NOT NULL,
|
||||
digest %s,
|
||||
created_at BIGINT NOT NULL,
|
||||
UNIQUE KEY uq_rollup (metric_name, entity_id, tags_hash, resolution_nano, bucket_nano),
|
||||
INDEX idx_rollup_res_time (metric_name, resolution_nano, bucket_nano)
|
||||
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci`, s.tables.rollups, pk, jsonType, blob),
|
||||
fmt.Sprintf(`CREATE TABLE IF NOT EXISTS %s (
|
||||
metric_name VARCHAR(191) PRIMARY KEY,
|
||||
watermark_nano BIGINT NOT NULL,
|
||||
updated_at BIGINT NOT NULL
|
||||
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci`, s.tables.watermarks),
|
||||
}
|
||||
}
|
||||
|
||||
if s.cfg.Driver == DriverPostgreSQL {
|
||||
statements = []string{
|
||||
fmt.Sprintf(`CREATE TABLE IF NOT EXISTS %s (
|
||||
name VARCHAR(191) PRIMARY KEY,
|
||||
type VARCHAR(32) NOT NULL,
|
||||
unit VARCHAR(64) NOT NULL DEFAULT '',
|
||||
description TEXT NOT NULL DEFAULT '',
|
||||
retention_days INTEGER NOT NULL DEFAULT 0,
|
||||
metadata %s NOT NULL,
|
||||
created_at BIGINT NOT NULL,
|
||||
updated_at BIGINT NOT NULL
|
||||
)`, s.tables.definitions, jsonType),
|
||||
fmt.Sprintf(`CREATE TABLE IF NOT EXISTS %s (
|
||||
id %s,
|
||||
metric_name VARCHAR(191) NOT NULL,
|
||||
entity_id VARCHAR(191) NOT NULL,
|
||||
tags_hash VARCHAR(64) NOT NULL,
|
||||
ts_nano BIGINT NOT NULL,
|
||||
value DOUBLE PRECISION NOT NULL,
|
||||
tags %s NOT NULL,
|
||||
labels %s NOT NULL,
|
||||
created_at BIGINT NOT NULL,
|
||||
UNIQUE(metric_name, entity_id, tags_hash, ts_nano)
|
||||
)`, s.tables.points, pk, jsonType, jsonType),
|
||||
fmt.Sprintf(`CREATE INDEX IF NOT EXISTS %s_points_metric_entity_time_idx ON %s (metric_name, entity_id, ts_nano)`, s.cfg.TablePrefix, s.tables.points),
|
||||
fmt.Sprintf(`CREATE INDEX IF NOT EXISTS %s_points_metric_time_idx ON %s (metric_name, ts_nano)`, s.cfg.TablePrefix, s.tables.points),
|
||||
fmt.Sprintf(`CREATE INDEX IF NOT EXISTS %s_points_entity_time_idx ON %s (entity_id, ts_nano)`, s.cfg.TablePrefix, s.tables.points),
|
||||
fmt.Sprintf(`CREATE TABLE IF NOT EXISTS %s (
|
||||
id %s,
|
||||
metric_name VARCHAR(191) NOT NULL,
|
||||
entity_id VARCHAR(191) NOT NULL,
|
||||
tags_hash VARCHAR(64) NOT NULL,
|
||||
tags %s NOT NULL,
|
||||
resolution_nano BIGINT NOT NULL,
|
||||
bucket_nano BIGINT NOT NULL,
|
||||
count BIGINT NOT NULL,
|
||||
sum DOUBLE PRECISION NOT NULL,
|
||||
sum_sq DOUBLE PRECISION NOT NULL,
|
||||
min_val DOUBLE PRECISION NOT NULL,
|
||||
max_val DOUBLE PRECISION NOT NULL,
|
||||
first_val DOUBLE PRECISION NOT NULL,
|
||||
first_ts BIGINT NOT NULL,
|
||||
last_val DOUBLE PRECISION NOT NULL,
|
||||
last_ts BIGINT NOT NULL,
|
||||
digest %s,
|
||||
created_at BIGINT NOT NULL,
|
||||
UNIQUE(metric_name, entity_id, tags_hash, resolution_nano, bucket_nano)
|
||||
)`, s.tables.rollups, pk, jsonType, blob),
|
||||
fmt.Sprintf(`CREATE INDEX IF NOT EXISTS %s_rollups_lookup_idx ON %s (metric_name, entity_id, tags_hash, resolution_nano, bucket_nano)`, s.cfg.TablePrefix, s.tables.rollups),
|
||||
fmt.Sprintf(`CREATE INDEX IF NOT EXISTS %s_rollups_res_time_idx ON %s (metric_name, resolution_nano, bucket_nano)`, s.cfg.TablePrefix, s.tables.rollups),
|
||||
fmt.Sprintf(`CREATE TABLE IF NOT EXISTS %s (
|
||||
metric_name VARCHAR(191) PRIMARY KEY,
|
||||
watermark_nano BIGINT NOT NULL,
|
||||
updated_at BIGINT NOT NULL
|
||||
)`, s.tables.watermarks),
|
||||
}
|
||||
}
|
||||
|
||||
for _, stmt := range statements {
|
||||
if _, err := s.db.ExecContext(ctx, stmt); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,76 @@
|
||||
package metric
|
||||
|
||||
import (
|
||||
"strconv"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// Pxx builds the Aggregation for an arbitrary percentile. The argument is a
|
||||
// percentage in (0,100): Pxx(99.9) -> "p99.9", Pxx(50) -> "p50". The fixed
|
||||
// AggP50/AggP95/AggP99 constants are just the common cases of this same string
|
||||
// form, so they keep working unchanged.
|
||||
//
|
||||
// This is what turns the package from "p50/p95/p99 only" into "any percentile":
|
||||
// callers can ask for p75, p90, p99.99, etc., and every path (in-memory,
|
||||
// SQL pushdown, and rollup-via-t-digest) understands it.
|
||||
//
|
||||
// Pxx 根据任意百分位构造 Aggregation。参数是 (0,100) 内的百分比:
|
||||
// Pxx(99.9) -> "p99.9",Pxx(50) -> "p50"。固定的 AggP50/AggP95/AggP99
|
||||
// 常量只是同一字符串形式的常用情况,因此会保持原有行为。
|
||||
//
|
||||
// 这让 package 从“只支持 p50/p95/p99”变成“支持任意百分位”:调用方可以请求
|
||||
// p75、p90、p99.99 等,并且每条路径(内存、SQL 下推、基于 t-digest 的 rollup)
|
||||
// 都能理解它。
|
||||
func Pxx(p float64) Aggregation {
|
||||
// Trim trailing zeros so Pxx(95) == AggP95 ("p95"), not "p95.000000".
|
||||
s := strconv.FormatFloat(p, 'f', -1, 64)
|
||||
return Aggregation("p" + s)
|
||||
}
|
||||
|
||||
// parsePercentile reports whether agg names a percentile and, if so, returns
|
||||
// the corresponding fraction in [0,1]. "p99.9" -> 0.999. Out-of-range
|
||||
// percentages (<=0 or >=100) are rejected so validation can reject them.
|
||||
//
|
||||
// parsePercentile 判断 agg 是否命名了百分位;如果是,则返回对应的 [0,1] 小数。
|
||||
// 例如 "p99.9" -> 0.999。越界百分比(<=0 或 >=100)会被拒绝,以便校验逻辑
|
||||
// 能拒绝它们。
|
||||
func parsePercentile(agg Aggregation) (float64, bool) {
|
||||
s := string(agg)
|
||||
if len(s) < 2 || (s[0] != 'p' && s[0] != 'P') {
|
||||
return 0, false
|
||||
}
|
||||
pct, err := strconv.ParseFloat(s[1:], 64)
|
||||
if err != nil {
|
||||
return 0, false
|
||||
}
|
||||
if pct <= 0 || pct >= 100 {
|
||||
return 0, false
|
||||
}
|
||||
return pct / 100, true
|
||||
}
|
||||
|
||||
// isPercentile reports whether agg is any percentile aggregation.
|
||||
//
|
||||
// isPercentile 判断聚合类型是否为任意百分位聚合。
|
||||
func isPercentile(agg Aggregation) bool {
|
||||
_, ok := parsePercentile(agg)
|
||||
return ok
|
||||
}
|
||||
|
||||
// percentileFractionString renders the fraction for SQL percentile_cont, e.g.
|
||||
// "p99.9" -> "0.999". Trailing zeros are trimmed for stable SQL text.
|
||||
//
|
||||
// percentileFractionString 把百分位聚合转换为 SQL percentile_cont 需要的
|
||||
// 小数字符串,并去掉尾随零以保持 SQL 文本稳定。
|
||||
func percentileFractionString(agg Aggregation) (string, bool) {
|
||||
f, ok := parsePercentile(agg)
|
||||
if !ok {
|
||||
return "", false
|
||||
}
|
||||
s := strconv.FormatFloat(f, 'f', -1, 64)
|
||||
if !strings.Contains(s, ".") {
|
||||
// f is in (0,1) so this should not happen, but guard anyway.
|
||||
s = strconv.FormatFloat(f, 'f', 1, 64)
|
||||
}
|
||||
return s, true
|
||||
}
|
||||
@@ -0,0 +1,198 @@
|
||||
package metric
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// TestPostgreSQLIntegration runs the PostgreSQL integration test when configured.
|
||||
//
|
||||
// TestPostgreSQLIntegration 在配置 DSN 后运行 PostgreSQL 集成测试。
|
||||
func TestPostgreSQLIntegration(t *testing.T) {
|
||||
dsn := os.Getenv("METRIC_POSTGRES_DSN")
|
||||
if dsn == "" {
|
||||
t.Skip("METRIC_POSTGRES_DSN is not set")
|
||||
}
|
||||
|
||||
runSQLIntegration(t, "postgres", PostgreSQL(dsn), true)
|
||||
}
|
||||
|
||||
// TestMySQLIntegration runs the MySQL integration test when configured.
|
||||
//
|
||||
// TestMySQLIntegration 在配置 DSN 后运行 MySQL 集成测试。
|
||||
func TestMySQLIntegration(t *testing.T) {
|
||||
dsn := os.Getenv("METRIC_MYSQL_DSN")
|
||||
if dsn == "" {
|
||||
t.Skip("METRIC_MYSQL_DSN is not set")
|
||||
}
|
||||
|
||||
runSQLIntegration(t, "mysql", MySQL(dsn), false)
|
||||
}
|
||||
|
||||
// runSQLIntegration exercises the SQL store against an external database.
|
||||
//
|
||||
// runSQLIntegration 在外部数据库上执行通用 SQL 集成测试流程。
|
||||
func runSQLIntegration(t *testing.T, name string, cfg Config, expectSQLPercentile bool) {
|
||||
t.Helper()
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||||
defer cancel()
|
||||
|
||||
prefix := fmt.Sprintf("it_%d_", time.Now().UnixNano())
|
||||
cfg.TablePrefix = prefix
|
||||
store, err := Open(ctx, cfg)
|
||||
if err != nil {
|
||||
t.Fatalf("open %s store: %v", name, err)
|
||||
}
|
||||
defer store.Close()
|
||||
defer dropIntegrationTables(t, store, prefix)
|
||||
|
||||
if err := store.CreateMetric(ctx, Definition{
|
||||
Name: "http.latency",
|
||||
Type: TypeGauge,
|
||||
Unit: "ms",
|
||||
Description: "HTTP latency",
|
||||
Metadata: map[string]string{"source": "integration"},
|
||||
RetentionDays: 30}); err != nil {
|
||||
t.Fatalf("create metric: %v", err)
|
||||
}
|
||||
if err := store.CreateMetric(ctx, Definition{Name: "http.latency", Type: TypeGauge, RetentionDays: 30}); err == nil {
|
||||
t.Fatalf("duplicate CreateMetric should fail")
|
||||
}
|
||||
if err := store.UpsertMetric(ctx, Definition{Name: "http.latency", Type: TypeGauge, Unit: "milliseconds", RetentionDays: 30}); err != nil {
|
||||
t.Fatalf("upsert metric: %v", err)
|
||||
}
|
||||
def, err := store.GetMetric(ctx, "http.latency")
|
||||
if err != nil {
|
||||
t.Fatalf("get metric: %v", err)
|
||||
}
|
||||
if def.Unit != "milliseconds" {
|
||||
t.Fatalf("upsert did not update unit: %#v", def)
|
||||
}
|
||||
|
||||
base := time.Date(2026, 6, 20, 1, 0, 0, 0, time.UTC)
|
||||
points := []Point{
|
||||
{MetricName: "http.latency", EntityID: "api-1", Timestamp: base, Value: 10, Tags: map[string]string{"region.zone": "ap-1", "route-name": "/v1/nodes"}},
|
||||
{MetricName: "http.latency", EntityID: "api-1", Timestamp: base.Add(time.Minute), Value: 20, Tags: map[string]string{"region.zone": "ap-1", "route-name": "/v1/nodes"}},
|
||||
{MetricName: "http.latency", EntityID: "api-1", Timestamp: base.Add(2 * time.Minute), Value: 30, Tags: map[string]string{"region.zone": "ap-1", "route-name": "/v1/nodes"}},
|
||||
{MetricName: "http.latency", EntityID: "api-1", Timestamp: base.Add(3 * time.Minute), Value: 40, Tags: map[string]string{"region.zone": "eu-1", "route-name": "/v1/nodes"}},
|
||||
}
|
||||
if err := store.WriteBatch(ctx, points); err != nil {
|
||||
t.Fatalf("write batch: %v", err)
|
||||
}
|
||||
|
||||
got, err := store.Query(ctx, Query{
|
||||
MetricName: "http.latency",
|
||||
EntityID: "api-1",
|
||||
Start: base.Add(-time.Second),
|
||||
End: base.Add(5 * time.Minute),
|
||||
Tags: map[string]string{"region.zone": "ap-1", "route-name": "/v1/nodes"},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("query with json tags: %v", err)
|
||||
}
|
||||
if len(got) != 3 {
|
||||
t.Fatalf("expected 3 ap-1 points, got %d: %#v", len(got), got)
|
||||
}
|
||||
|
||||
stats, err := store.Stats(ctx, Query{
|
||||
MetricName: "http.latency",
|
||||
EntityID: "api-1",
|
||||
Start: base,
|
||||
End: base.Add(5 * time.Minute),
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("stats: %v", err)
|
||||
}
|
||||
if stats.Count != 4 || stats.Avg != 25 {
|
||||
t.Fatalf("unexpected stats: %#v", stats)
|
||||
}
|
||||
|
||||
avgBuckets, err := store.Aggregate(ctx, AggregateQuery{
|
||||
Query: Query{
|
||||
MetricName: "http.latency",
|
||||
EntityID: "api-1",
|
||||
Start: base,
|
||||
End: base.Add(5 * time.Minute),
|
||||
},
|
||||
Aggregation: AggAvg,
|
||||
Interval: 2 * time.Minute,
|
||||
BucketLimit: 1,
|
||||
BucketOffset: 1,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("avg aggregate: %v", err)
|
||||
}
|
||||
if len(avgBuckets) != 1 || avgBuckets[0].Value != 35 || avgBuckets[0].Count != 2 {
|
||||
t.Fatalf("unexpected bucket-paged avg aggregate: %#v", avgBuckets)
|
||||
}
|
||||
|
||||
p95Buckets, err := store.Aggregate(ctx, AggregateQuery{
|
||||
Query: Query{
|
||||
MetricName: "http.latency",
|
||||
EntityID: "api-1",
|
||||
Start: base,
|
||||
End: base.Add(5 * time.Minute),
|
||||
},
|
||||
Aggregation: AggP95,
|
||||
Interval: 10 * time.Minute,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("p95 aggregate: %v", err)
|
||||
}
|
||||
if len(p95Buckets) != 1 || p95Buckets[0].Count != 4 || p95Buckets[0].Value <= 38 {
|
||||
t.Fatalf("unexpected p95 aggregate: %#v", p95Buckets)
|
||||
}
|
||||
if _, ok := sqlAggValueExpr(cfg.Driver, AggP95); ok != expectSQLPercentile {
|
||||
t.Fatalf("%s p95 pushdown expectation mismatch: got %v want %v", name, ok, expectSQLPercentile)
|
||||
}
|
||||
|
||||
latest, err := store.Latest(ctx, "http.latency", "api-1", 1)
|
||||
if err != nil {
|
||||
t.Fatalf("latest: %v", err)
|
||||
}
|
||||
if len(latest) != 1 || latest[0].Value != 40 {
|
||||
t.Fatalf("unexpected latest point: %#v", latest)
|
||||
}
|
||||
|
||||
deleted, err := store.DeleteBefore(ctx, "http.latency", base.Add(2*time.Minute))
|
||||
if err != nil {
|
||||
t.Fatalf("delete before: %v", err)
|
||||
}
|
||||
if deleted != 2 {
|
||||
t.Fatalf("expected 2 deleted points, got %d", deleted)
|
||||
}
|
||||
|
||||
storageSize, err := store.StorageSize(ctx)
|
||||
if err != nil {
|
||||
t.Fatalf("storage size: %v", err)
|
||||
}
|
||||
if storageSize <= 0 {
|
||||
t.Fatalf("storage size = %d, want a positive value", storageSize)
|
||||
}
|
||||
if err := store.ReclaimSpace(ctx); err != nil {
|
||||
t.Fatalf("reclaim storage space: %v", err)
|
||||
}
|
||||
if err := store.Ping(ctx); err != nil {
|
||||
t.Fatalf("store unusable after space reclaim: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// dropIntegrationTables drops integration-test tables.
|
||||
//
|
||||
// dropIntegrationTables 删除集成测试创建的表。
|
||||
func dropIntegrationTables(t *testing.T, store *Store, prefix string) {
|
||||
t.Helper()
|
||||
if strings.TrimSpace(prefix) == "" {
|
||||
t.Fatal("refusing to drop tables with empty prefix")
|
||||
}
|
||||
for _, name := range []string{prefix + "rollups", prefix + "points", prefix + "definitions"} {
|
||||
if _, err := store.db.Exec("DROP TABLE IF EXISTS " + name); err != nil {
|
||||
t.Fatalf("drop integration table %s: %v", name, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,344 @@
|
||||
package metric
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"math"
|
||||
"time"
|
||||
)
|
||||
|
||||
// RollupTier describes one downsampled resolution: raw points (or the next
|
||||
// finer tier) are aggregated into buckets Interval wide, and those buckets are
|
||||
// kept for Retention. A policy lists tiers from finest to coarsest, e.g.
|
||||
//
|
||||
// 1m kept 7d -> 5m kept 30d -> 1h kept 1y
|
||||
//
|
||||
// As data ages it falls off the finer tiers and survives only in coarser ones,
|
||||
// so storage shrinks with age — the "data gets sparser as it gets older"
|
||||
// behavior of a downsampling TSDB.
|
||||
//
|
||||
// RollupTier 描述一个降采样分辨率:原始点(或下一层更细的层级)会被聚合进
|
||||
// Interval 宽的桶,而这些桶会保留 Retention。策略会按从细到粗排列层级,例如:
|
||||
//
|
||||
// 1m 保留 7d -> 5m 保留 30d -> 1h 保留 1y
|
||||
//
|
||||
// 随着数据变旧,它会从更细层级中过期,只在更粗层级中保留下来,因此存储量会随
|
||||
// 数据年龄下降,也就是降采样 TSDB 中“数据越旧越稀疏”的行为。
|
||||
type RollupTier struct {
|
||||
// Interval is the bucket width for this rollup tier.
|
||||
//
|
||||
// Interval 是该 rollup 层的桶宽。
|
||||
Interval time.Duration `json:"interval"`
|
||||
// Retention is how long buckets in this tier are kept.
|
||||
//
|
||||
// Retention 是该层桶的保留时间。
|
||||
Retention time.Duration `json:"retention"`
|
||||
}
|
||||
|
||||
// RollupPolicy is the full retention ladder for a store: how long raw points
|
||||
// live, then a chain of progressively coarser tiers. Compact materializes the
|
||||
// tiers and enforces every retention window.
|
||||
//
|
||||
// RollupPolicy 描述完整保留阶梯:原始点保留多久,以及后续逐级变粗的
|
||||
// rollup 层;Compact 会物化这些层并执行保留窗口。
|
||||
type RollupPolicy struct {
|
||||
// RawRetention is how long raw points are kept before Compact deletes them
|
||||
// (after they have been rolled into the finest tier). Zero means "never
|
||||
// delete raw" — rollups are still built, but raw is retained.
|
||||
//
|
||||
// RawRetention 是 Compact 删除原始点之前保留它们的时间(在它们已进入最细
|
||||
// rollup 层之后)。零值表示“永不删除原始点”;rollup 仍会构建,但原始点保留。
|
||||
RawRetention time.Duration `json:"raw_retention"`
|
||||
// Tiers are ordered finest-first. Each Interval must be a positive integer
|
||||
// multiple of the previous tier's Interval (so a coarse bucket is composed
|
||||
// of whole finer buckets), and each Retention must be >= the previous
|
||||
// tier's Retention (coarse data outlives fine data).
|
||||
//
|
||||
// Tiers 按从细到粗排序。每个 Interval 必须是前一层 Interval 的正整数倍
|
||||
// (这样粗桶由完整细桶组成),每个 Retention 必须 >= 前一层 Retention
|
||||
// (粗数据比细数据活得更久)。
|
||||
Tiers []RollupTier `json:"tiers"`
|
||||
// Compression tunes the per-bucket t-digest (size vs. percentile accuracy).
|
||||
// <=1 uses the default (100).
|
||||
//
|
||||
// Compression 调整每个桶的 t-digest(大小与百分位精度之间的取舍)。
|
||||
// <=1 时使用默认值(100)。
|
||||
Compression float64 `json:"compression"`
|
||||
}
|
||||
|
||||
// Enabled reports whether the policy actually defines any rollup tiers.
|
||||
//
|
||||
// Enabled 表示策略是否实际定义了 rollup 层级。
|
||||
func (p RollupPolicy) Enabled() bool { return len(p.Tiers) > 0 }
|
||||
|
||||
// compression returns the effective t-digest compression for a rollup policy.
|
||||
//
|
||||
// compression 返回 rollup 策略实际使用的 t-digest 压缩参数。
|
||||
func (p RollupPolicy) compression() float64 {
|
||||
if p.Compression <= 1 {
|
||||
return defaultTDigestCompression
|
||||
}
|
||||
return p.Compression
|
||||
}
|
||||
|
||||
func (p RollupPolicy) rawCutoff(now time.Time) time.Time {
|
||||
if p.RawRetention <= 0 || len(p.Tiers) == 0 {
|
||||
return time.Time{}
|
||||
}
|
||||
cutoff := now.UTC().Add(-p.RawRetention)
|
||||
interval := p.Tiers[0].Interval.Nanoseconds()
|
||||
return time.Unix(0, floorDivNano(cutoff.UnixNano(), interval)).UTC()
|
||||
}
|
||||
|
||||
// withMetricRetention applies the metric definition's retention to a rollup
|
||||
// policy. Fine tiers keep their configured short windows, while the final tier
|
||||
// follows the metric so definitions are never capped by a store-wide setting.
|
||||
// Coarser tiers that do not extend retention are omitted because their buckets
|
||||
// can be derived from the retained finer tier on demand.
|
||||
func (p RollupPolicy) withMetricRetention(retention time.Duration) RollupPolicy {
|
||||
if retention <= 0 || len(p.Tiers) == 0 {
|
||||
return p
|
||||
}
|
||||
|
||||
out := p
|
||||
out.Tiers = make([]RollupTier, 0, len(p.Tiers))
|
||||
for i, tier := range p.Tiers {
|
||||
if i == len(p.Tiers)-1 || tier.Retention > retention {
|
||||
tier.Retention = retention
|
||||
}
|
||||
if len(out.Tiers) > 0 && tier.Retention <= out.Tiers[len(out.Tiers)-1].Retention {
|
||||
continue
|
||||
}
|
||||
out.Tiers = append(out.Tiers, tier)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// Validate enforces the structural rules that make cascading composition and
|
||||
// retention well-defined.
|
||||
//
|
||||
// Validate 检查策略结构是否满足级联合成和保留语义所需的约束。
|
||||
func (p RollupPolicy) Validate() error {
|
||||
if len(p.Tiers) == 0 {
|
||||
return nil // a store with no tiers simply does no rollup work
|
||||
}
|
||||
if p.RawRetention < 0 {
|
||||
return fmt.Errorf("%w: raw retention cannot be negative", ErrInvalidArgument)
|
||||
}
|
||||
var prev RollupTier
|
||||
for i, tr := range p.Tiers {
|
||||
if tr.Interval <= 0 {
|
||||
return fmt.Errorf("%w: tier %d interval must be positive", ErrInvalidArgument, i)
|
||||
}
|
||||
if tr.Retention <= 0 {
|
||||
return fmt.Errorf("%w: tier %d retention must be positive", ErrInvalidArgument, i)
|
||||
}
|
||||
if i == 0 {
|
||||
// The finest tier rolls up raw points. Require raw retention (when
|
||||
// set) to cover at least two of its buckets so the in-progress and
|
||||
// most-recent sealed bucket still have their raw backing when Compact
|
||||
// runs and deletes old raw.
|
||||
if p.RawRetention > 0 && p.RawRetention < 2*tr.Interval {
|
||||
return fmt.Errorf("%w: raw retention must be >= 2x the finest tier interval", ErrInvalidArgument)
|
||||
}
|
||||
} else {
|
||||
if tr.Interval <= prev.Interval {
|
||||
return fmt.Errorf("%w: tier %d interval must be larger than tier %d", ErrInvalidArgument, i, i-1)
|
||||
}
|
||||
if tr.Interval%prev.Interval != 0 {
|
||||
return fmt.Errorf("%w: tier %d interval must be a multiple of tier %d interval", ErrInvalidArgument, i, i-1)
|
||||
}
|
||||
if tr.Retention < prev.Retention {
|
||||
return fmt.Errorf("%w: tier %d retention must be >= tier %d retention", ErrInvalidArgument, i, i-1)
|
||||
}
|
||||
}
|
||||
prev = tr
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// rollupBucket is the in-memory accumulator for one (metric, entity,
|
||||
// resolution, bucket) cell. It carries exactly the summaries that can be
|
||||
// re-aggregated losslessly when composing coarser tiers — count/sum/sumSq for
|
||||
// avg & population stddev, min/max, first/last by timestamp — plus a t-digest
|
||||
// so arbitrary percentiles survive downsampling with bounded error.
|
||||
//
|
||||
// rollupBucket 是单个(指标、实体、分辨率、桶)单元的内存累加器。它只携带在
|
||||
// 合成更粗层级时可无损重新聚合的摘要:用于平均值和总体标准差的 count/sum/sumSq,
|
||||
// min/max,按时间记录的 first/last,外加一个 t-digest,让任意百分位能以
|
||||
// 有界误差在降采样后保留下来。
|
||||
type rollupBucket struct {
|
||||
// count is the total number of raw points represented.
|
||||
//
|
||||
// count 是该桶代表的原始点总数。
|
||||
count int64
|
||||
// sum is the sum of represented values.
|
||||
//
|
||||
// sum 是该桶代表值的总和。
|
||||
sum float64
|
||||
// sumSq is the sum of squared values for population stddev.
|
||||
//
|
||||
// sumSq 是用于总体标准差的平方和。
|
||||
sumSq float64
|
||||
// min is the minimum represented value.
|
||||
//
|
||||
// min 是该桶代表的最小值。
|
||||
min float64
|
||||
// max is the maximum represented value.
|
||||
//
|
||||
// max 是该桶代表的最大值。
|
||||
max float64
|
||||
// firstVal is the value with the earliest timestamp.
|
||||
//
|
||||
// firstVal 是时间戳最早的值。
|
||||
firstVal float64
|
||||
// firstTS is the earliest timestamp in nanoseconds.
|
||||
//
|
||||
// firstTS 是最早时间戳的纳秒值。
|
||||
firstTS int64
|
||||
// lastVal is the value with the latest timestamp.
|
||||
//
|
||||
// lastVal 是时间戳最晚的值。
|
||||
lastVal float64
|
||||
// lastTS is the latest timestamp in nanoseconds.
|
||||
//
|
||||
// lastTS 是最晚时间戳的纳秒值。
|
||||
lastTS int64
|
||||
// digest estimates percentiles for represented values.
|
||||
//
|
||||
// digest 用于估算该桶代表值的百分位。
|
||||
digest *TDigest
|
||||
// tagsHash carries the stable tag-set fingerprint for this rollup cell.
|
||||
//
|
||||
// tagsHash 携带该 rollup 单元的稳定标签集合指纹。
|
||||
tagsHash string
|
||||
// tagsJSON carries the canonical tag map written back to the tags column.
|
||||
//
|
||||
// tagsJSON 携带写回 tags 列的规范标签 map。
|
||||
tagsJSON string
|
||||
}
|
||||
|
||||
// newRollupBucket creates an empty rollup accumulator.
|
||||
//
|
||||
// newRollupBucket 创建一个空的 rollup 累加器。
|
||||
func newRollupBucket(compression float64) *rollupBucket {
|
||||
return &rollupBucket{
|
||||
min: 0,
|
||||
max: 0,
|
||||
firstTS: 0,
|
||||
lastTS: 0,
|
||||
digest: NewTDigest(compression),
|
||||
}
|
||||
}
|
||||
|
||||
// addPoint folds a raw observation into the bucket.
|
||||
//
|
||||
// addPoint 将一个原始观测值合入当前桶。
|
||||
func (b *rollupBucket) addPoint(value float64, tsNano int64) {
|
||||
if b.count == 0 {
|
||||
b.min, b.max = value, value
|
||||
b.firstVal, b.firstTS = value, tsNano
|
||||
b.lastVal, b.lastTS = value, tsNano
|
||||
} else {
|
||||
if value < b.min {
|
||||
b.min = value
|
||||
}
|
||||
if value > b.max {
|
||||
b.max = value
|
||||
}
|
||||
if tsNano < b.firstTS {
|
||||
b.firstVal, b.firstTS = value, tsNano
|
||||
}
|
||||
if tsNano > b.lastTS {
|
||||
b.lastVal, b.lastTS = value, tsNano
|
||||
}
|
||||
}
|
||||
b.count++
|
||||
b.sum += value
|
||||
b.sumSq += value * value
|
||||
b.digest.Add(value, 1)
|
||||
}
|
||||
|
||||
// mergeStored folds a finer rollup row (already-summarized) into this coarser
|
||||
// bucket. This is the cascade step: tier i+1 buckets are built by merging the
|
||||
// tier i rows they span.
|
||||
//
|
||||
// mergeStored 将一个已汇总的细层 rollup 行合入当前更粗桶。这是级联步骤:
|
||||
// tier i+1 的桶通过合并其覆盖的 tier i 行构建。
|
||||
func (b *rollupBucket) mergeStored(o *rollupBucket) {
|
||||
if o.count == 0 {
|
||||
return
|
||||
}
|
||||
if b.count == 0 {
|
||||
b.min, b.max = o.min, o.max
|
||||
b.firstVal, b.firstTS = o.firstVal, o.firstTS
|
||||
b.lastVal, b.lastTS = o.lastVal, o.lastTS
|
||||
} else {
|
||||
if o.min < b.min {
|
||||
b.min = o.min
|
||||
}
|
||||
if o.max > b.max {
|
||||
b.max = o.max
|
||||
}
|
||||
if o.firstTS < b.firstTS {
|
||||
b.firstVal, b.firstTS = o.firstVal, o.firstTS
|
||||
}
|
||||
if o.lastTS > b.lastTS {
|
||||
b.lastVal, b.lastTS = o.lastVal, o.lastTS
|
||||
}
|
||||
}
|
||||
b.count += o.count
|
||||
b.sum += o.sum
|
||||
b.sumSq += o.sumSq
|
||||
if b.digest == nil {
|
||||
b.digest = NewTDigest(defaultTDigestCompression)
|
||||
}
|
||||
b.digest.Merge(o.digest)
|
||||
}
|
||||
|
||||
// value computes the requested aggregation from the bucket summaries. ok=false
|
||||
// means the aggregation is not derivable from a rollup (only AggRate, which
|
||||
// needs the ordered raw series).
|
||||
//
|
||||
// value 从桶摘要中计算请求的聚合值;ok=false 表示该聚合无法由 rollup 推导
|
||||
// (目前主要是需要有序原始序列的 AggRate)。
|
||||
func (b *rollupBucket) value(agg Aggregation) (float64, bool) {
|
||||
switch agg {
|
||||
case AggAvg:
|
||||
if b.count == 0 {
|
||||
return 0, true
|
||||
}
|
||||
return b.sum / float64(b.count), true
|
||||
case AggMin:
|
||||
return b.min, true
|
||||
case AggMax:
|
||||
return b.max, true
|
||||
case AggSum:
|
||||
return b.sum, true
|
||||
case AggCount:
|
||||
return float64(b.count), true
|
||||
case AggFirst:
|
||||
return b.firstVal, true
|
||||
case AggLast:
|
||||
return b.lastVal, true
|
||||
case AggStdDev:
|
||||
if b.count == 0 {
|
||||
return 0, true
|
||||
}
|
||||
mean := b.sum / float64(b.count)
|
||||
variance := b.sumSq/float64(b.count) - mean*mean
|
||||
if variance < 0 {
|
||||
variance = 0 // floating-point guard
|
||||
}
|
||||
return math.Sqrt(variance), true
|
||||
case AggRate:
|
||||
return 0, false // rate needs the ordered raw series; not in a rollup
|
||||
default:
|
||||
if frac, ok := parsePercentile(agg); ok {
|
||||
if b.digest == nil || b.digest.Count() == 0 {
|
||||
return 0, true
|
||||
}
|
||||
return b.digest.Quantile(frac), true
|
||||
}
|
||||
return 0, false
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,184 @@
|
||||
package metric
|
||||
|
||||
import (
|
||||
"context"
|
||||
"math"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// TestSeriesHybridKeepsUncompactedRawNearCutoff verifies the hybrid read does
|
||||
// not lose raw points that fall between the (finest-aligned) raw cutoff and the
|
||||
// next bucket boundary, including a point written AFTER the last compaction.
|
||||
//
|
||||
// Reproduction: now=12:00:30, RawRetention=30m so the cutoff is 11:30:30, which
|
||||
// the policy floors to 11:30:00 (finest = 1m). A point at 11:30:15 written
|
||||
// before compaction is kept (>= aligned cutoff) and NOT rolled up; a point at
|
||||
// 11:30:45 written after compaction is also raw. Both belong to the 11:00 1h
|
||||
// output bucket. The old code aligned the split UP to 11:31:00 and read raw from
|
||||
// there, dropping both boundary points (count=1, the stale rollup half only).
|
||||
// Serving raw from the aligned cutoff fixes it (count=2).
|
||||
//
|
||||
// TestSeriesHybridKeepsUncompactedRawNearCutoff 验证混合读取不会丢失落在
|
||||
// (最细对齐后的)原始 cutoff 与下一个桶边界之间的 raw 点,包括上次 compact 之后
|
||||
// 才写入的点。
|
||||
func TestSeriesHybridKeepsUncompactedRawNearCutoff(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
policy := RollupPolicy{
|
||||
RawRetention: 30 * time.Minute,
|
||||
Tiers: []RollupTier{{Interval: time.Minute, Retention: 24 * time.Hour}},
|
||||
}
|
||||
s := newRollupStore(t, policy)
|
||||
if err := s.CreateMetric(ctx, Definition{Name: "nearcut", Type: TypeGauge, RetentionDays: 30}); err != nil {
|
||||
t.Fatalf("create: %v", err)
|
||||
}
|
||||
now := time.Date(2026, 6, 18, 12, 0, 30, 0, time.UTC)
|
||||
// 11:30:15 is >= the finest-aligned cutoff (11:30:00) -> survives compaction
|
||||
// as raw and is not rolled up.
|
||||
before := time.Date(2026, 6, 18, 11, 30, 15, 0, time.UTC)
|
||||
if err := s.Write(ctx, Point{MetricName: "nearcut", EntityID: "n1", Timestamp: before, Value: 10}); err != nil {
|
||||
t.Fatalf("write before: %v", err)
|
||||
}
|
||||
if _, err := s.Compact(ctx, now); err != nil {
|
||||
t.Fatalf("compact: %v", err)
|
||||
}
|
||||
// Written after compaction: only ever lives in raw.
|
||||
after := time.Date(2026, 6, 18, 11, 30, 45, 0, time.UTC)
|
||||
if err := s.Write(ctx, Point{MetricName: "nearcut", EntityID: "n1", Timestamp: after, Value: 100}); err != nil {
|
||||
t.Fatalf("write after: %v", err)
|
||||
}
|
||||
|
||||
got, err := s.Series(ctx, AggregateQuery{
|
||||
Query: Query{MetricName: "nearcut", EntityID: "n1", Start: time.Date(2026, 6, 18, 11, 0, 0, 0, time.UTC), End: now},
|
||||
Aggregation: AggAvg,
|
||||
Interval: time.Hour,
|
||||
}, now)
|
||||
if err != nil {
|
||||
t.Fatalf("series: %v", err)
|
||||
}
|
||||
bucket := alignTime(time.Date(2026, 6, 18, 11, 0, 0, 0, time.UTC), time.Hour)
|
||||
var found bool
|
||||
for _, p := range got {
|
||||
if p.Bucket.Equal(bucket) {
|
||||
found = true
|
||||
if p.Count != 2 {
|
||||
t.Fatalf("near-cutoff bucket count: got %d want 2 (%#v)", p.Count, p)
|
||||
}
|
||||
if math.Abs(p.Value-55) > 1e-9 {
|
||||
t.Fatalf("near-cutoff bucket avg: got %v want 55 (%#v)", p.Value, p)
|
||||
}
|
||||
}
|
||||
}
|
||||
if !found {
|
||||
t.Fatalf("expected 11:00 bucket with both raw points: %#v", got)
|
||||
}
|
||||
}
|
||||
|
||||
// TestSeriesHybridEndEqualsCutoff verifies a query whose End lands exactly on the
|
||||
// raw cutoff still includes the raw point sitting at the cutoff, instead of
|
||||
// returning empty.
|
||||
//
|
||||
// Reproduction: now=12:00:00, RawRetention=30m -> cutoff 11:30:00. A point at
|
||||
// exactly 11:30:00 is kept by compaction (not < cutoff) and lives in raw. An old
|
||||
// point at 11:00:10 is rolled up and its raw deleted. Querying [11:00, 11:30:00]
|
||||
// (End == cutoff) must combine the rolled-up old point and the raw cutoff point.
|
||||
// The old entry gate required End strictly AFTER the cutoff, so this fell through
|
||||
// to the pure-rollup path and returned the cutoff point's bucket empty.
|
||||
//
|
||||
// TestSeriesHybridEndEqualsCutoff 验证 End 恰好等于原始 cutoff 的查询仍会包含
|
||||
// 落在 cutoff 上的 raw 点,而不是返回空。
|
||||
func TestSeriesHybridEndEqualsCutoff(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
policy := RollupPolicy{
|
||||
RawRetention: 30 * time.Minute,
|
||||
Tiers: []RollupTier{{Interval: time.Minute, Retention: 24 * time.Hour}},
|
||||
}
|
||||
s := newRollupStore(t, policy)
|
||||
if err := s.CreateMetric(ctx, Definition{Name: "endcut", Type: TypeGauge, RetentionDays: 30}); err != nil {
|
||||
t.Fatalf("create: %v", err)
|
||||
}
|
||||
now := time.Date(2026, 6, 18, 12, 0, 0, 0, time.UTC)
|
||||
cutoff := time.Date(2026, 6, 18, 11, 30, 0, 0, time.UTC)
|
||||
old := time.Date(2026, 6, 18, 11, 0, 10, 0, time.UTC) // rolled up, raw deleted
|
||||
if err := s.Write(ctx, Point{MetricName: "endcut", EntityID: "n1", Timestamp: old, Value: 10}); err != nil {
|
||||
t.Fatalf("write old: %v", err)
|
||||
}
|
||||
// Point exactly at the cutoff: kept as raw (not < cutoff) after compaction.
|
||||
if err := s.Write(ctx, Point{MetricName: "endcut", EntityID: "n1", Timestamp: cutoff, Value: 100}); err != nil {
|
||||
t.Fatalf("write cutoff: %v", err)
|
||||
}
|
||||
if _, err := s.Compact(ctx, now); err != nil {
|
||||
t.Fatalf("compact: %v", err)
|
||||
}
|
||||
|
||||
got, err := s.Series(ctx, AggregateQuery{
|
||||
Query: Query{MetricName: "endcut", EntityID: "n1", Start: time.Date(2026, 6, 18, 11, 0, 0, 0, time.UTC), End: cutoff},
|
||||
Aggregation: AggSum,
|
||||
Interval: time.Hour,
|
||||
}, now)
|
||||
if err != nil {
|
||||
t.Fatalf("series: %v", err)
|
||||
}
|
||||
bucket := alignTime(time.Date(2026, 6, 18, 11, 0, 0, 0, time.UTC), time.Hour)
|
||||
var found bool
|
||||
for _, p := range got {
|
||||
if p.Bucket.Equal(bucket) {
|
||||
found = true
|
||||
if p.Count != 2 || math.Abs(p.Value-110) > 1e-9 {
|
||||
t.Fatalf("End==cutoff bucket: got count=%d value=%v want count=2 value=110", p.Count, p.Value)
|
||||
}
|
||||
}
|
||||
}
|
||||
if !found {
|
||||
t.Fatalf("expected 11:00 bucket including the cutoff raw point: %#v", got)
|
||||
}
|
||||
}
|
||||
|
||||
// TestSeriesHybridUsesLastCompactionWatermark verifies that a query does not
|
||||
// advance the raw/rollup split beyond the last successful compaction. The raw
|
||||
// point at 11:32 is still present after the 12:00 compaction, but a query at
|
||||
// 12:04 would calculate a dynamic cutoff of 11:34 and skip it without the
|
||||
// persisted watermark at 11:30.
|
||||
func TestSeriesHybridUsesLastCompactionWatermark(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
policy := RollupPolicy{
|
||||
RawRetention: 30 * time.Minute,
|
||||
Tiers: []RollupTier{{Interval: time.Minute, Retention: 24 * time.Hour}},
|
||||
}
|
||||
s := newRollupStore(t, policy)
|
||||
if err := s.CreateMetric(ctx, Definition{Name: "watermark", Type: TypeGauge, RetentionDays: 30}); err != nil {
|
||||
t.Fatalf("create: %v", err)
|
||||
}
|
||||
|
||||
compactAt := time.Date(2026, 6, 18, 12, 0, 0, 0, time.UTC)
|
||||
if err := s.WriteBatch(ctx, []Point{
|
||||
{MetricName: "watermark", EntityID: "n1", Timestamp: time.Date(2026, 6, 18, 11, 20, 0, 0, time.UTC), Value: 10},
|
||||
{MetricName: "watermark", EntityID: "n1", Timestamp: time.Date(2026, 6, 18, 11, 32, 0, 0, time.UTC), Value: 100},
|
||||
}); err != nil {
|
||||
t.Fatalf("write: %v", err)
|
||||
}
|
||||
if _, err := s.Compact(ctx, compactAt); err != nil {
|
||||
t.Fatalf("compact: %v", err)
|
||||
}
|
||||
|
||||
queryAt := compactAt.Add(4 * time.Minute)
|
||||
got, err := s.Series(ctx, AggregateQuery{
|
||||
Query: Query{MetricName: "watermark", EntityID: "n1", Start: time.Date(2026, 6, 18, 11, 0, 0, 0, time.UTC), End: queryAt},
|
||||
Aggregation: AggCount,
|
||||
Interval: time.Hour,
|
||||
}, queryAt)
|
||||
if err != nil {
|
||||
t.Fatalf("series: %v", err)
|
||||
}
|
||||
if len(got) != 1 || got[0].Count != 2 {
|
||||
t.Fatalf("series should include rollup and raw data before the next compaction, got %#v", got)
|
||||
}
|
||||
|
||||
watermark, found, err := s.compactionWatermark(ctx, "watermark")
|
||||
if err != nil {
|
||||
t.Fatalf("read watermark: %v", err)
|
||||
}
|
||||
if !found || !watermark.Equal(policy.rawCutoff(compactAt)) {
|
||||
t.Fatalf("watermark = %v, found=%v, want %v", watermark, found, policy.rawCutoff(compactAt))
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,387 @@
|
||||
package metric
|
||||
|
||||
import (
|
||||
"context"
|
||||
"math"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// TestAggregateRollupExcludesPartialBucket verifies the rollup query no longer
|
||||
// drags an out-of-window sample in through a partially-overlapping bucket.
|
||||
//
|
||||
// Two samples share one 1m rollup bucket [00:00:00, 00:01:00): 00:00:10=100 and
|
||||
// 00:00:50=10. A query window of [00:00:40, 00:01:30] only overlaps part of that
|
||||
// bucket. Because a rollup bucket is an indivisible summary, the old "any
|
||||
// overlap" rule folded the whole bucket in and counted 00:00:10 (value 100),
|
||||
// which lies OUTSIDE the window. Full containment excludes the straddling
|
||||
// bucket entirely, so the out-of-window sample can no longer leak in.
|
||||
//
|
||||
// TestAggregateRollupExcludesPartialBucket 验证 rollup 查询不再通过部分重叠的桶
|
||||
// 把窗口外的样本带进来。
|
||||
func TestAggregateRollupExcludesPartialBucket(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
policy := RollupPolicy{Tiers: []RollupTier{{Interval: time.Minute, Retention: 24 * time.Hour}}}
|
||||
s := newRollupStore(t, policy)
|
||||
if err := s.CreateMetric(ctx, Definition{Name: "win", Type: TypeGauge, RetentionDays: 30}); err != nil {
|
||||
t.Fatalf("create: %v", err)
|
||||
}
|
||||
base := time.Date(2026, 6, 18, 0, 0, 0, 0, time.UTC)
|
||||
batch := []Point{
|
||||
{MetricName: "win", EntityID: "n1", Timestamp: base.Add(10 * time.Second), Value: 100}, // out of window
|
||||
{MetricName: "win", EntityID: "n1", Timestamp: base.Add(50 * time.Second), Value: 10}, // in window
|
||||
}
|
||||
if err := s.WriteBatch(ctx, batch); err != nil {
|
||||
t.Fatalf("write: %v", err)
|
||||
}
|
||||
if _, err := s.Compact(ctx, base.Add(time.Hour)); err != nil {
|
||||
t.Fatalf("compact: %v", err)
|
||||
}
|
||||
|
||||
// Window [00:00:40, 00:01:30] partially overlaps the single 1m bucket.
|
||||
res, err := s.AggregateRollup(ctx, AggregateQuery{
|
||||
Query: Query{MetricName: "win", EntityID: "n1", Start: base.Add(40 * time.Second), End: base.Add(90 * time.Second)},
|
||||
Aggregation: AggSum,
|
||||
Interval: time.Minute,
|
||||
}, time.Minute)
|
||||
if err != nil {
|
||||
t.Fatalf("aggregate rollup: %v", err)
|
||||
}
|
||||
// The partially-overlapping bucket is excluded under full containment, so the
|
||||
// out-of-window 00:00:10 sample (value 100) must not appear. Concretely: no
|
||||
// returned bucket may carry the out-of-window value or a count of 2.
|
||||
for _, p := range res {
|
||||
if p.Count >= 2 || p.Value == 100 {
|
||||
t.Fatalf("partial bucket leaked out-of-window sample: %#v", p)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestAggregateRollupEndBoundaryInclusive verifies the rollup query end boundary
|
||||
// is inclusive (matching the raw query's ts <= end) and that a window aligned to
|
||||
// resolution boundaries returns every whole bucket it covers.
|
||||
//
|
||||
// Buckets at 00:00 and 00:01 are both whole-contained in [00:00:00, 00:02:00]:
|
||||
// the second bucket's window is [00:01:00, 00:02:00), whose last covered nano is
|
||||
// < 00:02:00, so an inclusive end keeps it. The previous half-open (< end) rule
|
||||
// could drop a bucket sitting exactly on the end edge.
|
||||
//
|
||||
// TestAggregateRollupEndBoundaryInclusive 验证 rollup 查询 end 边界为闭区间
|
||||
// (与 raw 的 ts <= end 一致),且对齐到分辨率边界的窗口会返回它覆盖的每个完整桶。
|
||||
func TestAggregateRollupEndBoundaryInclusive(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
policy := RollupPolicy{Tiers: []RollupTier{{Interval: time.Minute, Retention: 24 * time.Hour}}}
|
||||
s := newRollupStore(t, policy)
|
||||
if err := s.CreateMetric(ctx, Definition{Name: "edge", Type: TypeGauge, RetentionDays: 30}); err != nil {
|
||||
t.Fatalf("create: %v", err)
|
||||
}
|
||||
base := time.Date(2026, 6, 18, 0, 0, 0, 0, time.UTC)
|
||||
batch := []Point{
|
||||
{MetricName: "edge", EntityID: "n1", Timestamp: base.Add(10 * time.Second), Value: 1}, // bucket 00:00
|
||||
{MetricName: "edge", EntityID: "n1", Timestamp: base.Add(time.Minute + 10*time.Second), Value: 2}, // bucket 00:01
|
||||
}
|
||||
if err := s.WriteBatch(ctx, batch); err != nil {
|
||||
t.Fatalf("write: %v", err)
|
||||
}
|
||||
if _, err := s.Compact(ctx, base.Add(time.Hour)); err != nil {
|
||||
t.Fatalf("compact: %v", err)
|
||||
}
|
||||
|
||||
res, err := s.AggregateRollup(ctx, AggregateQuery{
|
||||
Query: Query{MetricName: "edge", EntityID: "n1", Start: base, End: base.Add(2 * time.Minute)},
|
||||
Aggregation: AggSum,
|
||||
Interval: time.Minute,
|
||||
}, time.Minute)
|
||||
if err != nil {
|
||||
t.Fatalf("aggregate rollup: %v", err)
|
||||
}
|
||||
if len(res) != 2 {
|
||||
t.Fatalf("aligned window should return both whole buckets, got %d: %#v", len(res), res)
|
||||
}
|
||||
if res[0].Value != 1 || res[1].Value != 2 {
|
||||
t.Fatalf("unexpected bucket values: %#v", res)
|
||||
}
|
||||
}
|
||||
|
||||
// TestSeriesHybridMergesStraddlingBucket verifies a query output bucket that
|
||||
// straddles the raw-retention boundary aggregates BOTH its rollup half and its
|
||||
// raw half, instead of letting raw replace the rollup half.
|
||||
//
|
||||
// With a 30m raw retention and now=12:00 the boundary is 11:30. A 1h output
|
||||
// bucket [11:00, 12:00) therefore straddles it. An 11:05 sample (value 10, older
|
||||
// than the boundary -> rolled up, raw deleted) and an 11:35 sample (value 100,
|
||||
// within retention -> still raw) both belong to that 11:00 bucket. The correct
|
||||
// answer folds both: count=2, avg=(10+100)/2=55. The old merge-by-bucket-time
|
||||
// path kept only the raw half and reported count=1, avg=100.
|
||||
//
|
||||
// TestSeriesHybridMergesStraddlingBucket 验证跨越原始保留期边界的输出桶会同时
|
||||
// 聚合 rollup 半边和 raw 半边,而不是让 raw 覆盖 rollup 半边。
|
||||
func TestSeriesHybridMergesStraddlingBucket(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
policy := RollupPolicy{
|
||||
RawRetention: 30 * time.Minute,
|
||||
Tiers: []RollupTier{{Interval: time.Minute, Retention: 24 * time.Hour}},
|
||||
}
|
||||
s := newRollupStore(t, policy)
|
||||
if err := s.CreateMetric(ctx, Definition{Name: "straddle", Type: TypeGauge, RetentionDays: 30}); err != nil {
|
||||
t.Fatalf("create: %v", err)
|
||||
}
|
||||
now := time.Date(2026, 6, 18, 12, 0, 0, 0, time.UTC)
|
||||
// 11:05 is older than the 11:30 cutoff (rolled up, raw deleted on compact);
|
||||
// 11:35 is within retention (kept raw).
|
||||
old := now.Add(-55 * time.Minute) // 11:05
|
||||
recent := now.Add(-25 * time.Minute) // 11:35
|
||||
batch := []Point{
|
||||
{MetricName: "straddle", EntityID: "n1", Timestamp: old, Value: 10},
|
||||
{MetricName: "straddle", EntityID: "n1", Timestamp: recent, Value: 100},
|
||||
}
|
||||
if err := s.WriteBatch(ctx, batch); err != nil {
|
||||
t.Fatalf("write: %v", err)
|
||||
}
|
||||
if _, err := s.Compact(ctx, now); err != nil {
|
||||
t.Fatalf("compact: %v", err)
|
||||
}
|
||||
|
||||
// Confirm the old raw point was deleted and the recent one survives, so the
|
||||
// two halves genuinely come from different sources.
|
||||
oldRaw, err := s.Query(ctx, Query{MetricName: "straddle", EntityID: "n1", Start: old.Add(-time.Minute), End: old.Add(time.Minute)})
|
||||
if err != nil {
|
||||
t.Fatalf("query old raw: %v", err)
|
||||
}
|
||||
if len(oldRaw) != 0 {
|
||||
t.Fatalf("expected old raw point deleted by retention, got %d", len(oldRaw))
|
||||
}
|
||||
|
||||
got, err := s.Series(ctx, AggregateQuery{
|
||||
Query: Query{MetricName: "straddle", EntityID: "n1", Start: now.Add(-time.Hour), End: now},
|
||||
Aggregation: AggAvg,
|
||||
Interval: time.Hour,
|
||||
}, now)
|
||||
if err != nil {
|
||||
t.Fatalf("series: %v", err)
|
||||
}
|
||||
|
||||
bucket := alignTime(now.Add(-time.Hour), time.Hour) // 11:00
|
||||
var found bool
|
||||
for _, p := range got {
|
||||
if p.Bucket.Equal(bucket) {
|
||||
found = true
|
||||
if p.Count != 2 {
|
||||
t.Fatalf("straddling bucket count: got %d want 2 (%#v)", p.Count, p)
|
||||
}
|
||||
if math.Abs(p.Value-55) > 1e-9 {
|
||||
t.Fatalf("straddling bucket avg: got %v want 55 (%#v)", p.Value, p)
|
||||
}
|
||||
}
|
||||
}
|
||||
if !found {
|
||||
t.Fatalf("expected a merged 11:00 bucket in result: %#v", got)
|
||||
}
|
||||
}
|
||||
|
||||
// TestSeriesHybridSumAcrossBoundary is the AggSum analogue: the straddling 1h
|
||||
// bucket must sum both halves (10 + 100 = 110), proving the merge is at the
|
||||
// summary level rather than a bucket-time replacement.
|
||||
//
|
||||
// TestSeriesHybridSumAcrossBoundary 是 AggSum 版本:跨边界的 1h 桶必须把两半求和
|
||||
// (10 + 100 = 110),证明合并发生在摘要层面,而不是按桶时间替换。
|
||||
func TestSeriesHybridSumAcrossBoundary(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
policy := RollupPolicy{
|
||||
RawRetention: 30 * time.Minute,
|
||||
Tiers: []RollupTier{{Interval: time.Minute, Retention: 24 * time.Hour}},
|
||||
}
|
||||
s := newRollupStore(t, policy)
|
||||
if err := s.CreateMetric(ctx, Definition{Name: "sumstraddle", Type: TypeGauge, RetentionDays: 30}); err != nil {
|
||||
t.Fatalf("create: %v", err)
|
||||
}
|
||||
now := time.Date(2026, 6, 18, 12, 0, 0, 0, time.UTC)
|
||||
batch := []Point{
|
||||
{MetricName: "sumstraddle", EntityID: "n1", Timestamp: now.Add(-55 * time.Minute), Value: 10},
|
||||
{MetricName: "sumstraddle", EntityID: "n1", Timestamp: now.Add(-25 * time.Minute), Value: 100},
|
||||
}
|
||||
if err := s.WriteBatch(ctx, batch); err != nil {
|
||||
t.Fatalf("write: %v", err)
|
||||
}
|
||||
if _, err := s.Compact(ctx, now); err != nil {
|
||||
t.Fatalf("compact: %v", err)
|
||||
}
|
||||
got, err := s.Series(ctx, AggregateQuery{
|
||||
Query: Query{MetricName: "sumstraddle", EntityID: "n1", Start: now.Add(-time.Hour), End: now},
|
||||
Aggregation: AggSum,
|
||||
Interval: time.Hour,
|
||||
}, now)
|
||||
if err != nil {
|
||||
t.Fatalf("series: %v", err)
|
||||
}
|
||||
bucket := alignTime(now.Add(-time.Hour), time.Hour)
|
||||
var found bool
|
||||
for _, p := range got {
|
||||
if p.Bucket.Equal(bucket) {
|
||||
found = true
|
||||
if p.Count != 2 || math.Abs(p.Value-110) > 1e-9 {
|
||||
t.Fatalf("straddling sum bucket: got count=%d value=%v want count=2 value=110", p.Count, p.Value)
|
||||
}
|
||||
}
|
||||
}
|
||||
if !found {
|
||||
t.Fatalf("expected merged sum bucket: %#v", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSeriesHybridIncludesRawBetweenCutoffAndNextBucket(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
policy := RollupPolicy{
|
||||
RawRetention: 30 * time.Minute,
|
||||
Tiers: []RollupTier{{Interval: time.Minute, Retention: 24 * time.Hour}},
|
||||
}
|
||||
s := newRollupStore(t, policy)
|
||||
if err := s.CreateMetric(ctx, Definition{Name: "cutgap", Type: TypeGauge, RetentionDays: 30}); err != nil {
|
||||
t.Fatalf("create: %v", err)
|
||||
}
|
||||
|
||||
now := time.Date(2026, 6, 18, 12, 0, 30, 0, time.UTC)
|
||||
old := time.Date(2026, 6, 18, 11, 30, 10, 0, time.UTC)
|
||||
if err := s.Write(ctx, Point{MetricName: "cutgap", EntityID: "n1", Timestamp: old, Value: 10}); err != nil {
|
||||
t.Fatalf("write old: %v", err)
|
||||
}
|
||||
if _, err := s.Compact(ctx, now); err != nil {
|
||||
t.Fatalf("compact: %v", err)
|
||||
}
|
||||
recentUncompacted := time.Date(2026, 6, 18, 11, 30, 45, 0, time.UTC)
|
||||
if err := s.Write(ctx, Point{MetricName: "cutgap", EntityID: "n1", Timestamp: recentUncompacted, Value: 100}); err != nil {
|
||||
t.Fatalf("write recent: %v", err)
|
||||
}
|
||||
|
||||
got, err := s.Series(ctx, AggregateQuery{
|
||||
Query: Query{MetricName: "cutgap", EntityID: "n1", Start: time.Date(2026, 6, 18, 11, 0, 0, 0, time.UTC), End: now},
|
||||
Aggregation: AggCount,
|
||||
Interval: time.Hour,
|
||||
}, now)
|
||||
if err != nil {
|
||||
t.Fatalf("series: %v", err)
|
||||
}
|
||||
if len(got) != 1 || got[0].Count != 2 {
|
||||
t.Fatalf("series should include rollup plus raw at the aligned cutoff, got %#v", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSeriesHybridEndAtCutoffIncludesRawBoundary(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
policy := RollupPolicy{
|
||||
RawRetention: 30 * time.Minute,
|
||||
Tiers: []RollupTier{{Interval: time.Minute, Retention: 24 * time.Hour}},
|
||||
}
|
||||
s := newRollupStore(t, policy)
|
||||
if err := s.CreateMetric(ctx, Definition{Name: "cutedge", Type: TypeGauge, RetentionDays: 30}); err != nil {
|
||||
t.Fatalf("create: %v", err)
|
||||
}
|
||||
|
||||
now := time.Date(2026, 6, 18, 12, 0, 30, 0, time.UTC)
|
||||
cutoff := policy.rawCutoff(now)
|
||||
if err := s.Write(ctx, Point{MetricName: "cutedge", EntityID: "n1", Timestamp: cutoff, Value: 7}); err != nil {
|
||||
t.Fatalf("write: %v", err)
|
||||
}
|
||||
if _, err := s.Compact(ctx, now); err != nil {
|
||||
t.Fatalf("compact: %v", err)
|
||||
}
|
||||
|
||||
got, err := s.Series(ctx, AggregateQuery{
|
||||
Query: Query{MetricName: "cutedge", EntityID: "n1", Start: cutoff.Add(-time.Hour), End: cutoff},
|
||||
Aggregation: AggCount,
|
||||
Interval: time.Hour,
|
||||
}, now)
|
||||
if err != nil {
|
||||
t.Fatalf("series: %v", err)
|
||||
}
|
||||
var total int
|
||||
for _, p := range got {
|
||||
total += p.Count
|
||||
}
|
||||
if total != 1 {
|
||||
t.Fatalf("series should include raw point exactly at cutoff/end, got %#v", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCompactDoesNotDoubleCountRawBucketAfterCutoffAdvances(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
policy := RollupPolicy{
|
||||
RawRetention: 30 * time.Minute,
|
||||
Tiers: []RollupTier{{Interval: time.Minute, Retention: 24 * time.Hour}},
|
||||
}
|
||||
s := newRollupStore(t, policy)
|
||||
if err := s.CreateMetric(ctx, Definition{Name: "nodup", Type: TypeGauge, RetentionDays: 30}); err != nil {
|
||||
t.Fatalf("create: %v", err)
|
||||
}
|
||||
|
||||
now := time.Date(2026, 6, 18, 12, 0, 30, 0, time.UTC)
|
||||
if err := s.Write(ctx, Point{MetricName: "nodup", EntityID: "n1", Timestamp: time.Date(2026, 6, 18, 11, 30, 10, 0, time.UTC), Value: 10}); err != nil {
|
||||
t.Fatalf("write old: %v", err)
|
||||
}
|
||||
if _, err := s.Compact(ctx, now); err != nil {
|
||||
t.Fatalf("compact 1: %v", err)
|
||||
}
|
||||
if err := s.Write(ctx, Point{MetricName: "nodup", EntityID: "n1", Timestamp: time.Date(2026, 6, 18, 11, 30, 45, 0, time.UTC), Value: 100}); err != nil {
|
||||
t.Fatalf("write recent: %v", err)
|
||||
}
|
||||
now = now.Add(time.Minute)
|
||||
if _, err := s.Compact(ctx, now); err != nil {
|
||||
t.Fatalf("compact 2: %v", err)
|
||||
}
|
||||
|
||||
got, err := s.Series(ctx, AggregateQuery{
|
||||
Query: Query{MetricName: "nodup", EntityID: "n1", Start: time.Date(2026, 6, 18, 11, 0, 0, 0, time.UTC), End: now},
|
||||
Aggregation: AggCount,
|
||||
Interval: time.Hour,
|
||||
}, now)
|
||||
if err != nil {
|
||||
t.Fatalf("series: %v", err)
|
||||
}
|
||||
var total int
|
||||
for _, p := range got {
|
||||
total += p.Count
|
||||
}
|
||||
if total != 2 {
|
||||
t.Fatalf("compaction should not double-count a bucket after cutoff advances, got %#v", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSeriesHybridIncludesPartialCoarseBucketAtRawCutoff(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
policy := RollupPolicy{
|
||||
RawRetention: 10 * time.Minute,
|
||||
Tiers: []RollupTier{
|
||||
{Interval: time.Minute, Retention: 20 * time.Minute},
|
||||
{Interval: time.Hour, Retention: 24 * time.Hour},
|
||||
},
|
||||
}
|
||||
s := newRollupStore(t, policy)
|
||||
if err := s.CreateMetric(ctx, Definition{Name: "coarse-cutoff", Type: TypeGauge, RetentionDays: 30}); err != nil {
|
||||
t.Fatalf("create: %v", err)
|
||||
}
|
||||
|
||||
now := time.Date(2026, 6, 18, 12, 50, 0, 0, time.UTC)
|
||||
old := now.Add(-40 * time.Minute)
|
||||
recent := now.Add(-5 * time.Minute)
|
||||
if err := s.WriteBatch(ctx, []Point{
|
||||
{MetricName: "coarse-cutoff", EntityID: "n1", Timestamp: old, Value: 10},
|
||||
{MetricName: "coarse-cutoff", EntityID: "n1", Timestamp: recent, Value: 100},
|
||||
}); err != nil {
|
||||
t.Fatalf("write: %v", err)
|
||||
}
|
||||
if _, err := s.Compact(ctx, now); err != nil {
|
||||
t.Fatalf("compact: %v", err)
|
||||
}
|
||||
|
||||
got, err := s.Series(ctx, AggregateQuery{
|
||||
Query: Query{MetricName: "coarse-cutoff", EntityID: "n1", Start: now.Add(-2 * time.Hour), End: now},
|
||||
Aggregation: AggAvg,
|
||||
Interval: time.Hour,
|
||||
}, now)
|
||||
if err != nil {
|
||||
t.Fatalf("series: %v", err)
|
||||
}
|
||||
if len(got) != 1 || got[0].Count != 2 || math.Abs(got[0].Value-55) > 1e-9 {
|
||||
t.Fatalf("expected coarse rollup and recent raw point, got %#v", got)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,625 @@
|
||||
package metric
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// AggregateRollup answers an AggregateQuery from a stored rollup tier instead of
|
||||
// raw points. resolution names which tier to read (it must match a tier
|
||||
// Interval that Compact has materialized). The query Interval must be a positive
|
||||
// integer multiple of resolution, so each output bucket is composed of whole
|
||||
// rollup buckets.
|
||||
//
|
||||
// query.Tags is honored: only rollup series whose stored tag set matches the
|
||||
// filter are folded in, so a tag filter selects the same data it would over raw
|
||||
// points. By default, matched series are merged into each output bucket; callers
|
||||
// that need entity/tag identities preserved can set PreserveSeries.
|
||||
//
|
||||
// Every aggregation works except AggRate, which needs the ordered raw series and
|
||||
// is therefore raw-only. Percentiles (p50, p95, p99, and arbitrary pXX) are
|
||||
// answered by merging the per-bucket t-digests, so they survive downsampling
|
||||
// with bounded error.
|
||||
//
|
||||
// AggregateRollup 从已存储的 rollup 层回答 AggregateQuery,而不是读取原始点。
|
||||
// resolution 指定要读取的层级(必须匹配 Compact 已物化的某个层级 Interval)。
|
||||
// 查询的 Interval 必须是 resolution 的正整数倍,因此每个输出桶都由完整的
|
||||
// rollup 桶组成。
|
||||
//
|
||||
// query.Tags 会被遵守:只有存储标签集合匹配过滤条件的 rollup 序列会被合入,
|
||||
// 因此标签过滤会选中与原始点查询相同的数据。默认会把匹配序列合并进每个
|
||||
// 输出桶;需要保留 entity/tag 身份的调用方可以设置 PreserveSeries。
|
||||
//
|
||||
// 除 AggRate 外,每种聚合都可用;AggRate 需要有序原始序列,因此只能基于原始点。
|
||||
// 百分位(p50、p95、p99 和任意 pXX)通过合并每桶 t-digest 回答,因此能在
|
||||
// 降采样后以有界误差保留下来。
|
||||
func (s *Store) AggregateRollup(ctx context.Context, query AggregateQuery, resolution time.Duration) ([]AggregatePoint, error) {
|
||||
if err := s.ensureOpen(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := query.Validate(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if resolution <= 0 {
|
||||
return nil, fmt.Errorf("%w: rollup resolution must be positive", ErrInvalidArgument)
|
||||
}
|
||||
if query.Interval < resolution || query.Interval%resolution != 0 {
|
||||
return nil, fmt.Errorf("%w: query interval must be a positive multiple of the rollup resolution", ErrInvalidArgument)
|
||||
}
|
||||
if query.Aggregation == AggRate {
|
||||
return nil, fmt.Errorf("%w: rate is not derivable from rollups (raw only)", ErrInvalidArgument)
|
||||
}
|
||||
|
||||
q := query.Query.normalized()
|
||||
comp := s.cfg.RollupPolicy.compression()
|
||||
|
||||
// Read rollup buckets at this resolution that are FULLY contained in the
|
||||
// inclusive window [Start, End] (entity and tag filters pushed into SQL),
|
||||
// then fold them into query.Interval-wide output buckets. Full containment
|
||||
// (rather than mere overlap) is what keeps a partially-overlapping bucket's
|
||||
// out-of-window samples from leaking into the result: a rollup bucket is an
|
||||
// indivisible summary, so a bucket straddling a window edge cannot be
|
||||
// trimmed to the window and is therefore excluded. Callers that need every
|
||||
// sample in a sub-bucket window must align the window to resolution
|
||||
// boundaries (or query raw points).
|
||||
rows, err := s.scanRollupRowsContained(ctx, q.MetricName, q.EntityID, q.Tags, resolution, q.Start, q.End)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
groups := foldRollupRows(nil, rows, query.Interval, comp, query.PreserveSeries)
|
||||
|
||||
out, err := rollupGroupsToPoints(groups, query)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return pageBuckets(out, query.BucketLimit, query.BucketOffset), nil
|
||||
}
|
||||
|
||||
// rollupGroupsToPoints turns the merged output buckets into ordered
|
||||
// AggregatePoints, computing the requested aggregation from each bucket's
|
||||
// summaries/digest.
|
||||
//
|
||||
// rollupGroupsToPoints 将合并后的输出桶转换为有序 AggregatePoint,并根据
|
||||
// 每个桶的摘要或 digest 计算请求的聚合。
|
||||
func rollupGroupsToPoints(groups map[rollupKey]*rollupBucket, query AggregateQuery) ([]AggregatePoint, error) {
|
||||
if !query.PreserveSeries {
|
||||
return mergedRollupGroupsToPoints(groups, query)
|
||||
}
|
||||
|
||||
keys := make([]rollupKey, 0, len(groups))
|
||||
for key := range groups {
|
||||
keys = append(keys, key)
|
||||
}
|
||||
sortRollupKeys(keys)
|
||||
|
||||
out := make([]AggregatePoint, 0, len(keys))
|
||||
for _, key := range keys {
|
||||
b := groups[key]
|
||||
v, ok := b.value(query.Aggregation)
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("%w: aggregation %q not supported over rollups", ErrInvalidArgument, query.Aggregation)
|
||||
}
|
||||
tags, err := rollupTagsFromJSON(b.tagsJSON)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out = append(out, AggregatePoint{
|
||||
MetricName: query.MetricName,
|
||||
EntityID: key.entityID,
|
||||
Bucket: time.Unix(0, key.bucket).UTC(),
|
||||
Value: v,
|
||||
Count: int(b.count),
|
||||
Tags: tags,
|
||||
})
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func mergedRollupGroupsToPoints(groups map[rollupKey]*rollupBucket, query AggregateQuery) ([]AggregatePoint, error) {
|
||||
keys := make([]rollupKey, 0, len(groups))
|
||||
for key := range groups {
|
||||
keys = append(keys, key)
|
||||
}
|
||||
sortRollupKeys(keys)
|
||||
|
||||
tags := cloneStringMap(query.Tags)
|
||||
out := make([]AggregatePoint, 0, len(keys))
|
||||
for _, key := range keys {
|
||||
b := groups[key]
|
||||
v, ok := b.value(query.Aggregation)
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("%w: aggregation %q not supported over rollups", ErrInvalidArgument, query.Aggregation)
|
||||
}
|
||||
out = append(out, AggregatePoint{
|
||||
MetricName: query.MetricName,
|
||||
EntityID: query.EntityID,
|
||||
Bucket: time.Unix(0, key.bucket).UTC(),
|
||||
Value: v,
|
||||
Count: int(b.count),
|
||||
Tags: cloneStringMap(tags),
|
||||
})
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// scanRollupRowsContained loads rollup rows for one resolution whose whole
|
||||
// bucket window [bucket, bucket+resolution) lies inside the inclusive query
|
||||
// window [start, end], with optional entity and tag filters pushed into SQL.
|
||||
//
|
||||
// Full containment (not mere overlap) is the boundary rule that keeps a rollup
|
||||
// query from over-counting: a rollup bucket is an indivisible summary, so a
|
||||
// bucket that only partially overlaps the window would drag its out-of-window
|
||||
// samples in if it were included. The end boundary is inclusive to match the
|
||||
// raw query semantics (raw uses ts <= end); a bucket whose last covered nano is
|
||||
// exactly end is therefore still contained. A degenerate zero-width window
|
||||
// (start == end) contains no whole bucket and yields an empty result — callers
|
||||
// that need sub-bucket precision must query raw points.
|
||||
//
|
||||
// scanRollupRowsContained 读取某分辨率下整桶窗口 [bucket, bucket+resolution)
|
||||
// 完整落在闭区间 [start, end] 内的 rollup 行,并可把实体和标签过滤下推到 SQL。
|
||||
//
|
||||
// 采用“完整包含”(而非仅重叠)作为边界规则,避免 rollup 查询过度计数:rollup
|
||||
// 桶是不可分割的摘要,只与窗口部分重叠的桶若被纳入,会把窗口外的样本一起带入。
|
||||
// end 边界为闭区间,以匹配原始查询语义(raw 使用 ts <= end);因此最后覆盖纳秒
|
||||
// 恰为 end 的桶仍算被包含。零宽窗口(start == end)不包含任何完整桶,返回空结果,
|
||||
// 需要亚桶精度的调用方应查询原始点。
|
||||
func (s *Store) scanRollupRowsContained(ctx context.Context, metricName, entityID string, tags map[string]string, resolution time.Duration, start, end time.Time) ([]storedRollup, error) {
|
||||
resNano := resolution.Nanoseconds()
|
||||
startNano := start.UTC().UnixNano()
|
||||
endNano := end.UTC().UnixNano()
|
||||
// A fully-contained bucket has start >= startNano and end (inclusive,
|
||||
// bucket+resNano-1) <= endNano, i.e. bucket in [startNano, endNano-resNano+1].
|
||||
// Push that exact closed range into SQL; no post-filter is then required.
|
||||
lower := startNano
|
||||
upper := endNano - resNano + 1 // inclusive upper bound for bucket_nano
|
||||
if upper < lower {
|
||||
return nil, nil // window narrower than one bucket: nothing is contained
|
||||
}
|
||||
rows, err := s.scanRollupRowsBetween(ctx, metricName, entityID, tags, resNano, lower, upper)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return rows, nil
|
||||
}
|
||||
|
||||
// scanRollupRowsBetween loads rollup rows for one resolution whose bucket start
|
||||
// falls in the inclusive nano range [lowerBucket, upperBucket], with optional
|
||||
// entity and tag filters pushed into SQL. It is the shared SQL primitive behind
|
||||
// the containment and hybrid scans; the bucket-window semantics are imposed by
|
||||
// the caller through the bounds it passes.
|
||||
//
|
||||
// scanRollupRowsBetween 读取某分辨率下桶起点落在闭区间
|
||||
// [lowerBucket, upperBucket] 内的 rollup 行,并可把实体和标签过滤下推到 SQL。
|
||||
// 它是包含扫描和混合扫描共用的 SQL 原语;桶窗口语义由调用方通过传入的边界决定。
|
||||
func (s *Store) scanRollupRowsBetween(ctx context.Context, metricName, entityID string, tags map[string]string, resNano, lowerBucket, upperBucket int64) ([]storedRollup, error) {
|
||||
args := []any{metricName, resNano, lowerBucket, upperBucket}
|
||||
parts := []string{
|
||||
"metric_name = " + s.dialect.placeholder(1),
|
||||
"resolution_nano = " + s.dialect.placeholder(2),
|
||||
"bucket_nano >= " + s.dialect.placeholder(3),
|
||||
"bucket_nano <= " + s.dialect.placeholder(4),
|
||||
}
|
||||
if strings.TrimSpace(entityID) != "" {
|
||||
args = append(args, entityID)
|
||||
parts = append(parts, "entity_id = "+s.dialect.placeholder(len(args)))
|
||||
}
|
||||
for _, k := range sortedKeys(tags) {
|
||||
args = append(args, tags[k])
|
||||
parts = append(parts, s.dialect.jsonExtractEquals("tags", k, s.dialect.placeholder(len(args))))
|
||||
}
|
||||
sqlText := fmt.Sprintf(
|
||||
`SELECT entity_id, tags_hash, tags, bucket_nano, count, sum, sum_sq, min_val, max_val, first_val, first_ts, last_val, last_ts, digest
|
||||
FROM %s WHERE %s ORDER BY bucket_nano ASC`,
|
||||
s.tables.rollups, strings.Join(parts, " AND "),
|
||||
)
|
||||
rows, err := s.reader().QueryContext(ctx, sqlText, args...)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
return scanStoredRollups(rows)
|
||||
}
|
||||
|
||||
// foldRollupRows folds stored rollup rows into interval-wide output buckets,
|
||||
// merging summaries (including the t-digest) per bucket. groups may be nil, in
|
||||
// which case a fresh map is allocated; passing an existing map lets a caller
|
||||
// accumulate rollup and raw contributions into the same output buckets.
|
||||
//
|
||||
// foldRollupRows 将存储的 rollup 行折叠进 interval 宽的输出桶,按桶合并摘要
|
||||
// (包括 t-digest)。groups 可为 nil,此时会分配新 map;传入已有 map 可让调用方
|
||||
// 把 rollup 和 raw 的贡献累加到同一批输出桶中。
|
||||
func foldRollupRows(groups map[rollupKey]*rollupBucket, rows []storedRollup, interval time.Duration, comp float64, preserveSeries bool) map[rollupKey]*rollupBucket {
|
||||
if groups == nil {
|
||||
groups = make(map[rollupKey]*rollupBucket)
|
||||
}
|
||||
size := interval.Nanoseconds()
|
||||
for _, r := range rows {
|
||||
bkt := floorDivNano(r.bucket, size)
|
||||
key := foldedRollupKey(r.entityID, r.bucketData.tagsHash, bkt, preserveSeries)
|
||||
b := groups[key]
|
||||
if b == nil {
|
||||
b = newRollupBucket(comp)
|
||||
b.tagsHash = r.bucketData.tagsHash
|
||||
b.tagsJSON = r.bucketData.tagsJSON
|
||||
groups[key] = b
|
||||
}
|
||||
b.mergeStored(r.bucketData)
|
||||
}
|
||||
return groups
|
||||
}
|
||||
|
||||
// foldRawPoints folds raw points into interval-wide output buckets, adding each
|
||||
// observation into the matching bucket's accumulator. It shares the bucket map
|
||||
// with foldRollupRows so a hybrid query can combine an old rollup half and a
|
||||
// recent raw half into the same output buckets and aggregate them together
|
||||
// (correct count/avg/percentile across the boundary).
|
||||
//
|
||||
// foldRawPoints 将原始点折叠进 interval 宽的输出桶,把每个观测值加入对应桶的
|
||||
// 累加器。它与 foldRollupRows 共用桶 map,因此混合查询可以把旧 rollup 半边和
|
||||
// 近期 raw 半边合并到同一批输出桶中并一起聚合(跨边界的 count/avg/百分位正确)。
|
||||
func foldRawPoints(groups map[rollupKey]*rollupBucket, points []Point, interval time.Duration, comp float64, preserveSeries bool) (map[rollupKey]*rollupBucket, error) {
|
||||
if groups == nil {
|
||||
groups = make(map[rollupKey]*rollupBucket)
|
||||
}
|
||||
size := interval.Nanoseconds()
|
||||
for _, p := range points {
|
||||
p = p.normalized()
|
||||
tagsHash, tagsJSON, err := tagsFingerprint(p.Tags)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
ts := p.Timestamp.UTC().UnixNano()
|
||||
bkt := floorDivNano(ts, size)
|
||||
key := foldedRollupKey(p.EntityID, tagsHash, bkt, preserveSeries)
|
||||
b := groups[key]
|
||||
if b == nil {
|
||||
b = newRollupBucket(comp)
|
||||
b.tagsHash = tagsHash
|
||||
b.tagsJSON = tagsJSON
|
||||
groups[key] = b
|
||||
}
|
||||
b.addPoint(p.Value, ts)
|
||||
}
|
||||
return groups, nil
|
||||
}
|
||||
|
||||
func foldedRollupKey(entityID, tagsHash string, bucket int64, preserveSeries bool) rollupKey {
|
||||
if !preserveSeries {
|
||||
return rollupKey{bucket: bucket}
|
||||
}
|
||||
return rollupKey{entityID: entityID, tagsHash: tagsHash, bucket: bucket}
|
||||
}
|
||||
|
||||
func sortRollupKeys(keys []rollupKey) {
|
||||
sort.Slice(keys, func(i, j int) bool {
|
||||
if keys[i].bucket != keys[j].bucket {
|
||||
return keys[i].bucket < keys[j].bucket
|
||||
}
|
||||
if keys[i].entityID != keys[j].entityID {
|
||||
return keys[i].entityID < keys[j].entityID
|
||||
}
|
||||
return keys[i].tagsHash < keys[j].tagsHash
|
||||
})
|
||||
}
|
||||
|
||||
func rollupTagsFromJSON(tagsJSON string) (map[string]string, error) {
|
||||
tags, err := decodeMapString(tagsJSON)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return cloneStringMap(tags), nil
|
||||
}
|
||||
|
||||
// scanStoredRollups reconstructs storedRollup rows (including tag identity and
|
||||
// the decoded t-digest) from a result set whose columns are, in order:
|
||||
// entity_id, tags_hash, tags, bucket_nano, count, sum, sum_sq, min_val, max_val,
|
||||
// first_val, first_ts, last_val, last_ts, digest.
|
||||
//
|
||||
// scanStoredRollups 从结果集中还原 storedRollup 行(包括标签身份和解码后的
|
||||
// t-digest)。结果集的列顺序必须是:entity_id、tags_hash、tags、bucket_nano、
|
||||
// count、sum、sum_sq、min_val、max_val、first_val、first_ts、last_val、
|
||||
// last_ts、digest。
|
||||
func scanStoredRollups(rows *sql.Rows) ([]storedRollup, error) {
|
||||
var out []storedRollup
|
||||
for rows.Next() {
|
||||
var (
|
||||
eid string
|
||||
tagsHash string
|
||||
rawTags any
|
||||
bucketNano int64
|
||||
count int64
|
||||
sum, sumSq, minV, maxV, firstV, lastV float64
|
||||
firstTS, lastTS int64
|
||||
digestBlob []byte
|
||||
)
|
||||
if err := rows.Scan(&eid, &tagsHash, &rawTags, &bucketNano, &count, &sum, &sumSq, &minV, &maxV, &firstV, &firstTS, &lastV, &lastTS, &digestBlob); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
td, err := DecodeTDigest(digestBlob)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
tagsJSON, err := rawTagsToJSON(rawTags)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out = append(out, storedRollup{
|
||||
entityID: eid,
|
||||
bucket: bucketNano,
|
||||
bucketData: &rollupBucket{
|
||||
count: count, sum: sum, sumSq: sumSq,
|
||||
min: minV, max: maxV,
|
||||
firstVal: firstV, firstTS: firstTS,
|
||||
lastVal: lastV, lastTS: lastTS,
|
||||
digest: td,
|
||||
tagsHash: tagsHash,
|
||||
tagsJSON: tagsJSON,
|
||||
},
|
||||
})
|
||||
}
|
||||
return out, rows.Err()
|
||||
}
|
||||
|
||||
// rawTagsToJSON normalizes a scanned tags column (string or []byte) into the
|
||||
// canonical JSON string used when the bucket is re-written by a coarser tier.
|
||||
//
|
||||
// rawTagsToJSON 将扫描出的 tags 列(string 或 []byte)规范化为 JSON 字符串,
|
||||
// 供更粗层级重写桶时复用。
|
||||
func rawTagsToJSON(v any) (string, error) {
|
||||
switch x := v.(type) {
|
||||
case nil:
|
||||
return "{}", nil
|
||||
case string:
|
||||
if x == "" {
|
||||
return "{}", nil
|
||||
}
|
||||
return x, nil
|
||||
case []byte:
|
||||
if len(x) == 0 {
|
||||
return "{}", nil
|
||||
}
|
||||
return string(x), nil
|
||||
default:
|
||||
return "", fmt.Errorf("unsupported tags column type %T", v)
|
||||
}
|
||||
}
|
||||
|
||||
// CompatibleSeriesInterval raises an output interval when the requested window
|
||||
// is only covered by a coarser rollup tier. If the requested start predates all
|
||||
// retained tiers, it uses the longest-retained tier so the available portion of
|
||||
// the window can still be returned.
|
||||
func (s *Store) CompatibleSeriesInterval(start, now time.Time, interval time.Duration) time.Duration {
|
||||
policy := s.cfg.RollupPolicy
|
||||
if interval <= 0 || !policy.Enabled() {
|
||||
return interval
|
||||
}
|
||||
|
||||
start = start.UTC()
|
||||
now = now.UTC()
|
||||
rawCutoff := policy.rawCutoff(now)
|
||||
if rawCutoff.IsZero() || !start.Before(rawCutoff) {
|
||||
return interval
|
||||
}
|
||||
|
||||
var fallback *RollupTier
|
||||
for i := range policy.Tiers {
|
||||
tier := &policy.Tiers[i]
|
||||
fallback = tier
|
||||
if now.Add(-tier.Retention).After(start) {
|
||||
continue
|
||||
}
|
||||
if interval < tier.Interval {
|
||||
return tier.Interval
|
||||
}
|
||||
if remainder := interval % tier.Interval; remainder != 0 {
|
||||
return interval + tier.Interval - remainder
|
||||
}
|
||||
return interval
|
||||
}
|
||||
|
||||
if fallback != nil {
|
||||
if interval < fallback.Interval {
|
||||
return fallback.Interval
|
||||
}
|
||||
if remainder := interval % fallback.Interval; remainder != 0 {
|
||||
return interval + fallback.Interval - remainder
|
||||
}
|
||||
}
|
||||
return interval
|
||||
}
|
||||
|
||||
// bestRollupTier returns the finest compatible tier that covers start. When
|
||||
// start is older than every retained tier, it returns the longest-retained
|
||||
// compatible tier so callers still receive the retained intersection.
|
||||
func bestRollupTier(policy RollupPolicy, interval time.Duration, start, now time.Time) *RollupTier {
|
||||
var fallback *RollupTier
|
||||
for i := range policy.Tiers {
|
||||
tier := &policy.Tiers[i]
|
||||
if interval < tier.Interval || interval%tier.Interval != 0 {
|
||||
continue
|
||||
}
|
||||
fallback = tier
|
||||
if !now.Add(-tier.Retention).After(start) {
|
||||
return tier
|
||||
}
|
||||
}
|
||||
return fallback
|
||||
}
|
||||
|
||||
// Series answers an AggregateQuery by transparently choosing the best data
|
||||
// source for the requested window, given `now`:
|
||||
//
|
||||
// - If rollups are disabled, or the whole window still lies within raw
|
||||
// retention, it reads raw points (Aggregate) for full fidelity.
|
||||
// - Otherwise it picks the FINEST rollup tier that both (a) has an Interval
|
||||
// dividing query.Interval and (b) whose retention reaches back to the start
|
||||
// of the window, and serves the query from that tier.
|
||||
// - If the query spans the raw retention boundary, it uses a hybrid approach:
|
||||
// read rollups for the old part and raw for the recent part, then merge
|
||||
// the results to avoid losing uncompacted recent data.
|
||||
// - If the start predates every retained tier, it reads the longest-retained
|
||||
// compatible tier and returns the available intersection of the window.
|
||||
// - It falls back to raw only when no rollup tier is interval-compatible.
|
||||
//
|
||||
// query.Tags is honored on both branches: the raw path already filters by tag,
|
||||
// and the rollup path filters by the stored tag set, so a tag filter selects the
|
||||
// same series regardless of which source answers. This is the "downsampling
|
||||
// TSDB" read path: recent ranges answer from raw at full resolution, older
|
||||
// ranges answer from progressively coarser rollups.
|
||||
//
|
||||
// Series 会在给定 `now` 的情况下,通过透明选择最佳数据源来回答 AggregateQuery:
|
||||
//
|
||||
// - 如果 rollup 已禁用,或整个窗口仍在原始保留期内,它会读取原始点
|
||||
// (Aggregate)以获得完整保真度。
|
||||
// - 否则它会选择最细的 rollup 层,该层必须同时满足 (a) Interval 能整除
|
||||
// query.Interval,且 (b) 保留时间能覆盖到窗口起点,然后从该层服务查询。
|
||||
// - 如果查询跨越原始数据保留期边界,它使用混合方式:读取旧部分的 rollup
|
||||
// 和最近部分的原始点,然后合并结果,避免丢掉未 compact 的最近数据。
|
||||
// - 如果起点早于所有层级的保留窗口,它会读取保留时间最长的兼容层级,返回请求
|
||||
// 窗口与实际保留数据的交集。
|
||||
// - 只有没有任何层级与输出间隔兼容时才回退到原始点。
|
||||
//
|
||||
// query.Tags 在两条分支上都会被遵守:原始路径已按标签过滤,rollup 路径会按
|
||||
// 存储的标签集合过滤,因此无论由哪个数据源回答,标签过滤都会选择相同序列。
|
||||
// 这是“降采样 TSDB”的读取路径:近期范围以完整分辨率从原始点回答,旧范围从
|
||||
// 逐级更粗的 rollup 回答。
|
||||
func (s *Store) Series(ctx context.Context, query AggregateQuery, now time.Time) ([]AggregatePoint, error) {
|
||||
if err := s.ensureOpen(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := query.Validate(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
policy := s.cfg.RollupPolicy
|
||||
if !policy.Enabled() {
|
||||
return s.Aggregate(ctx, query)
|
||||
}
|
||||
q := query.Query.normalized()
|
||||
now = now.UTC()
|
||||
|
||||
rawCutoff := policy.rawCutoff(now)
|
||||
watermark, hasWatermark, err := s.compactionWatermark(ctx, q.MetricName)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Whole window inside raw retention (or raw kept forever) -> raw.
|
||||
if rawCutoff.IsZero() || !q.Start.Before(rawCutoff) {
|
||||
return s.Aggregate(ctx, query)
|
||||
}
|
||||
// Rate is raw-only; the caller asked for something rollups can't provide, so
|
||||
// answer from raw regardless of age.
|
||||
if query.Aggregation == AggRate {
|
||||
return s.Aggregate(ctx, query)
|
||||
}
|
||||
|
||||
servicingTier := bestRollupTier(policy, query.Interval, q.Start, now)
|
||||
if q.Start.Before(rawCutoff) && !q.End.Before(rawCutoff) && servicingTier != nil {
|
||||
if hasWatermark {
|
||||
boundary := rawCutoff
|
||||
if watermark.Before(boundary) {
|
||||
boundary = watermark
|
||||
}
|
||||
if !q.Start.Before(boundary) {
|
||||
return s.Aggregate(ctx, query)
|
||||
}
|
||||
return s.seriesHybrid(ctx, query, boundary, servicingTier)
|
||||
}
|
||||
|
||||
// Existing stores may have rollups but no watermark yet. Read all raw
|
||||
// data from the requested start and use the dynamic cutoff only as the
|
||||
// rollup scan's upper bound. Compaction's raw/rollup invariant makes
|
||||
// these two ranges disjoint; the next successful compact persists the
|
||||
// precise boundary for subsequent queries.
|
||||
return s.seriesHybridWithRollupEnd(ctx, query, q.Start, rawCutoff, servicingTier)
|
||||
}
|
||||
|
||||
if servicingTier == nil {
|
||||
return s.Aggregate(ctx, query)
|
||||
}
|
||||
|
||||
return s.AggregateRollup(ctx, query, servicingTier.Interval)
|
||||
}
|
||||
|
||||
// seriesHybrid answers a query that spans the raw-retention boundary by reading
|
||||
// rollups for the old part and raw for the recent part and folding BOTH into the
|
||||
// same query.Interval-wide output buckets before reducing them.
|
||||
//
|
||||
// The old lossy approach reduced each half to AggregatePoints independently and
|
||||
// then deduplicated by bucket time, letting raw fully replace a rollup point in
|
||||
// any output bucket they shared. That dropped the rollup half of a bucket that
|
||||
// straddles the boundary: e.g. with a 1h output bucket and the boundary mid-hour,
|
||||
// an 11:05 rollup sample and an 11:35 raw sample both belong to the 11:00 bucket,
|
||||
// yet the result kept only the raw sample (count/avg wrong). Folding both halves
|
||||
// into one rollupBucket per output bucket fixes that: count, sum/avg, min/max and
|
||||
// percentiles are computed over the union of the two halves.
|
||||
//
|
||||
// The rollup bucket containing the raw cutoff may be partial: compaction only
|
||||
// folds points older than the cutoff into it. Include that bucket, then add raw
|
||||
// points at or after the cutoff, so the two halves neither overlap nor leave a
|
||||
// gap when the selected rollup resolution is coarser than raw retention.
|
||||
//
|
||||
// seriesHybrid 回答跨越原始保留期边界的查询:读取旧部分的 rollup 和近期部分的
|
||||
// 原始点,并在归约前把两者折叠进同一批 query.Interval 宽的输出桶。
|
||||
//
|
||||
// 旧的有损做法会把两半各自归约成 AggregatePoint 再按桶时间去重,让 raw 在二者
|
||||
// 共有的输出桶里完全覆盖 rollup 点,从而丢掉跨边界桶的 rollup 半边(例如 1h 输出
|
||||
// 桶且边界落在小时中间时,11:05 的 rollup 样本和 11:35 的 raw 样本都属于 11:00 桶,
|
||||
// 结果却只剩 raw 样本,count/avg 错误)。把两半折叠进同一个 rollupBucket 即可修复:
|
||||
// count、sum/avg、min/max 和百分位都在两半的并集上计算。
|
||||
//
|
||||
// 包含 raw 截止点的 rollup 桶可能尚未完整;它只包含截止点之前已压缩的数据。读取
|
||||
// 该桶,再合入截止点及之后的 raw 点,可以在粗粒度层级上避免重叠和缺口。
|
||||
func (s *Store) seriesHybrid(ctx context.Context, query AggregateQuery, boundary time.Time, tier *RollupTier) ([]AggregatePoint, error) {
|
||||
return s.seriesHybridWithRollupEnd(ctx, query, boundary, boundary, tier)
|
||||
}
|
||||
|
||||
func (s *Store) seriesHybridWithRollupEnd(ctx context.Context, query AggregateQuery, boundary, rollupEnd time.Time, tier *RollupTier) ([]AggregatePoint, error) {
|
||||
q := query.Query.normalized()
|
||||
comp := s.cfg.RollupPolicy.compression()
|
||||
resNano := tier.Interval.Nanoseconds()
|
||||
startNano := q.Start.UnixNano()
|
||||
endNano := q.End.UnixNano()
|
||||
splitAt := boundary.UTC().UnixNano()
|
||||
|
||||
groups := make(map[rollupKey]*rollupBucket)
|
||||
|
||||
// Old half: include the rollup bucket containing splitAt. That bucket only
|
||||
// contains compacted points before splitAt; the raw half starts at splitAt.
|
||||
upperBucket := floorDivNano(rollupEnd.UTC().UnixNano(), resNano)
|
||||
if upperBucket >= startNano {
|
||||
rows, err := s.scanRollupRowsBetween(ctx, q.MetricName, q.EntityID, q.Tags, resNano, startNano, upperBucket)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
foldRollupRows(groups, rows, query.Interval, comp, query.PreserveSeries)
|
||||
}
|
||||
|
||||
// Recent half: raw points in [splitAt, End]. Reuse the raw Query path so the
|
||||
// entity and tag filters match the rollup half exactly, then fold the points
|
||||
// into the SAME output buckets so a bucket straddling splitAt aggregates both
|
||||
// halves. Skip when splitAt is past the window end (no recent raw to add).
|
||||
if splitAt <= endNano {
|
||||
rawQuery := q
|
||||
rawQuery.Start = time.Unix(0, splitAt).UTC()
|
||||
rawQuery.Limit = 0
|
||||
rawQuery.Offset = 0
|
||||
points, err := s.Query(ctx, rawQuery)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if _, err := foldRawPoints(groups, points, query.Interval, comp, query.PreserveSeries); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
|
||||
out, err := rollupGroupsToPoints(groups, query)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return pageBuckets(out, query.BucketLimit, query.BucketOffset), nil
|
||||
}
|
||||
@@ -0,0 +1,755 @@
|
||||
package metric
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"errors"
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Compact runs the store's RollupPolicy over every metric: it advances the
|
||||
// rollup tiers to `now` and enforces every retention window (raw points and each
|
||||
// tier). With finite raw retention, only newly eligible and late raw points are
|
||||
// propagated. It is safe to call repeatedly and is idempotent for unchanged
|
||||
// windows.
|
||||
//
|
||||
// Returns the number of rollup buckets written across all tiers and metrics.
|
||||
//
|
||||
// Compact 会对所有指标执行 Store 的 RollupPolicy:它会把 rollup 层推进到
|
||||
// `now`,并执行每个保留窗口(原始点和各层级)。原始点保留期有限时,只会级联
|
||||
// 本轮新到期及迟到的原始点。它可以重复安全调用,未变化窗口保持幂等。
|
||||
//
|
||||
// 返回所有指标、所有层级中写入的 rollup 桶数量。
|
||||
func (s *Store) Compact(ctx context.Context, now time.Time) (int, error) {
|
||||
if err := s.ensureOpen(); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
defs, err := s.ListMetrics(ctx)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
total := 0
|
||||
for _, def := range defs {
|
||||
n, err := s.CompactMetric(ctx, def.Name, now)
|
||||
if err != nil {
|
||||
return total, fmt.Errorf("compact metric %q: %w", def.Name, err)
|
||||
}
|
||||
total += n
|
||||
}
|
||||
return total, nil
|
||||
}
|
||||
|
||||
// CompactMetric compacts a single metric. With finite raw retention it consumes
|
||||
// newly eligible raw points as a delta and propagates that delta through every
|
||||
// tier. When raw is retained forever it falls back to rebuilding tiers. It then
|
||||
// deletes data that has aged out of each retention window.
|
||||
//
|
||||
// The entire compaction is performed within a SERIALIZABLE transaction to
|
||||
// guarantee that the raw scan and the raw deletion observe the same snapshot.
|
||||
// Without this, the default isolation level on PostgreSQL/MySQL lets a point
|
||||
// inserted between the scan and the delete be deleted by the cutoff DELETE
|
||||
// without ever being folded into a rollup: the scan never saw it (it ran on an
|
||||
// earlier read), yet the DELETE (a fresh read) does. SERIALIZABLE makes the
|
||||
// scan acquire predicate/range protection so a concurrent write that would
|
||||
// otherwise slip into that gap either is excluded from the delete or forces a
|
||||
// serialization failure that we retry on a fresh snapshot. SQLite serializes
|
||||
// writes on a single connection, so its default isolation already provides this
|
||||
// guarantee and needs no escalation.
|
||||
//
|
||||
// Rollups are keyed by tag set as well as entity, so each distinct tag
|
||||
// combination (e.g. a GPU device_index) is summarized into its own series and
|
||||
// can be queried independently after the raw points are gone.
|
||||
//
|
||||
// CompactMetric 会压缩单个指标。原始点保留期有限时,它把本轮新到期的原始点
|
||||
// 作为增量逐层传播;原始点永久保留时则回退到全量重建。最后删除各保留窗口中
|
||||
// 已过期的数据。
|
||||
//
|
||||
// 整个 compaction 在一个 SERIALIZABLE 事务内执行,以保证 raw 扫描和 raw 删除
|
||||
// 看到同一个快照。否则在 PostgreSQL/MySQL 的默认隔离级别下,扫描之后、删除
|
||||
// 之前写入的旧时间点仍可能被 cutoff 删除却没有进入 rollup:扫描(较早的读)
|
||||
// 没看到它,而删除(一次新的读)看到了。SERIALIZABLE 让扫描获得谓词/范围
|
||||
// 保护,使这种并发写要么被删除排除在外,要么触发可重试的序列化失败,由我们
|
||||
// 在新快照上重试。SQLite 在单连接上串行化写入,其默认隔离已提供该保证,
|
||||
// 无需提升隔离级别。
|
||||
func (s *Store) CompactMetric(ctx context.Context, metricName string, now time.Time) (int, error) {
|
||||
if err := s.ensureOpen(); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
s.retentionMu.RLock()
|
||||
defer s.retentionMu.RUnlock()
|
||||
policy := s.cfg.RollupPolicy
|
||||
def, err := s.GetMetric(ctx, metricName)
|
||||
if errors.Is(err, ErrNotFound) {
|
||||
return 0, nil
|
||||
}
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
if def.RetentionDays == 0 {
|
||||
_, err := s.DeleteSeries(ctx, Query{MetricName: metricName})
|
||||
return 0, err
|
||||
}
|
||||
if !policy.Enabled() {
|
||||
return 0, nil
|
||||
}
|
||||
effectivePolicy := policy.withMetricRetention(time.Duration(def.RetentionDays) * 24 * time.Hour)
|
||||
obsoleteIntervals := rollupIntervalsOutsidePolicy(policy.Tiers, effectivePolicy.Tiers)
|
||||
policy = effectivePolicy
|
||||
now = now.UTC()
|
||||
|
||||
// Retry the whole compaction on a transient serialization/deadlock failure.
|
||||
// Under SERIALIZABLE, a concurrent writer touching the same raw range can
|
||||
// abort our transaction; re-running on a fresh snapshot is the correct and
|
||||
// expected recovery, and Compact is idempotent so a retry is safe.
|
||||
const maxAttempts = 5
|
||||
var lastErr error
|
||||
for attempt := 0; attempt < maxAttempts; attempt++ {
|
||||
written, err := s.compactMetricOnce(ctx, metricName, now, policy, obsoleteIntervals)
|
||||
if err == nil {
|
||||
return written, nil
|
||||
}
|
||||
if !isRetryableSerializationError(err) {
|
||||
return written, err
|
||||
}
|
||||
lastErr = err
|
||||
}
|
||||
return 0, fmt.Errorf("compact metric %q: exhausted retries after serialization failures: %w", metricName, lastErr)
|
||||
}
|
||||
|
||||
// compactMetricOnce runs a single compaction attempt inside one transaction.
|
||||
//
|
||||
// compactMetricOnce 在单个事务内执行一次 compaction 尝试。
|
||||
func (s *Store) compactMetricOnce(ctx context.Context, metricName string, now time.Time, policy RollupPolicy, obsoleteIntervals []time.Duration) (int, error) {
|
||||
// Use a transaction to ensure consistency between raw scan, rollup write, and
|
||||
// raw deletion. The isolation level is backend-specific (SERIALIZABLE on
|
||||
// PostgreSQL/MySQL, default on SQLite) so late-arriving points cannot be
|
||||
// deleted without first being rolled up.
|
||||
tx, err := s.db.BeginTx(ctx, s.dialect.compactTxOptions())
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
defer func() { _ = tx.Rollback() }()
|
||||
|
||||
if err := s.deleteRollupsForIntervalsTx(ctx, metricName, obsoleteIntervals, tx); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
written, err := s.compactMetricWithinTx(ctx, metricName, now, policy, tx)
|
||||
if err != nil {
|
||||
return written, err
|
||||
}
|
||||
if err := s.persistCompactionWatermarkTx(ctx, metricName, policy.rawCutoff(now), tx); err != nil {
|
||||
return written, err
|
||||
}
|
||||
|
||||
if err := tx.Commit(); err != nil {
|
||||
return written, err
|
||||
}
|
||||
return written, nil
|
||||
}
|
||||
|
||||
func rollupIntervalsOutsidePolicy(configured, effective []RollupTier) []time.Duration {
|
||||
active := make(map[time.Duration]struct{}, len(effective))
|
||||
for _, tier := range effective {
|
||||
active[tier.Interval] = struct{}{}
|
||||
}
|
||||
obsolete := make([]time.Duration, 0, len(configured))
|
||||
for _, tier := range configured {
|
||||
if _, ok := active[tier.Interval]; !ok {
|
||||
obsolete = append(obsolete, tier.Interval)
|
||||
}
|
||||
}
|
||||
return obsolete
|
||||
}
|
||||
|
||||
// isRetryableSerializationError reports whether err is a transient
|
||||
// serialization failure or deadlock that should be retried on a fresh
|
||||
// transaction. It matches on portable SQLSTATE codes and driver error text so
|
||||
// the package stays free of driver-specific error type imports:
|
||||
//
|
||||
// - PostgreSQL: 40001 serialization_failure, 40P01 deadlock_detected.
|
||||
// - MySQL: 1213 deadlock, 1205 lock wait timeout.
|
||||
// - SQLite: SQLITE_BUSY / database is locked (only relevant with >1 conn).
|
||||
//
|
||||
// isRetryableSerializationError 判断 err 是否为应在新事务上重试的瞬时序列化
|
||||
// 失败或死锁。它通过可移植的 SQLSTATE 码和驱动错误文本匹配,使包不需要导入
|
||||
// 驱动专用错误类型。
|
||||
func isRetryableSerializationError(err error) bool {
|
||||
if err == nil {
|
||||
return false
|
||||
}
|
||||
msg := strings.ToLower(err.Error())
|
||||
switch {
|
||||
case strings.Contains(msg, "40001"): // postgres serialization_failure
|
||||
return true
|
||||
case strings.Contains(msg, "40p01"): // postgres deadlock_detected
|
||||
return true
|
||||
case strings.Contains(msg, "serialization failure"):
|
||||
return true
|
||||
case strings.Contains(msg, "could not serialize"):
|
||||
return true
|
||||
case strings.Contains(msg, "deadlock"): // mysql 1213, postgres text
|
||||
return true
|
||||
case strings.Contains(msg, "lock wait timeout"): // mysql 1205
|
||||
return true
|
||||
case strings.Contains(msg, "database is locked"): // sqlite busy
|
||||
return true
|
||||
case strings.Contains(msg, "database table is locked"): // sqlite busy
|
||||
return true
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
// compactMetricWithinTx is the actual compaction logic, meant to be called
|
||||
// within a transaction. The tx parameter is passed explicitly to all operations
|
||||
// that need transactional consistency.
|
||||
func (s *Store) compactMetricWithinTx(ctx context.Context, metricName string, now time.Time, policy RollupPolicy, tx *sql.Tx) (int, error) {
|
||||
if !policy.Enabled() {
|
||||
return 0, nil
|
||||
}
|
||||
if policy.RawRetention > 0 {
|
||||
return s.compactMetricIncrementalWithinTx(ctx, metricName, now, policy, tx)
|
||||
}
|
||||
return s.compactMetricFullWithinTx(ctx, metricName, now, policy, tx)
|
||||
}
|
||||
|
||||
// compactMetricIncrementalWithinTx rolls only raw points crossing the hot-data
|
||||
// cutoff and propagates that delta through every coarser tier. Because those raw
|
||||
// points are deleted in the same transaction, each sample is merged exactly
|
||||
// once, including late samples for buckets that already exist.
|
||||
func (s *Store) compactMetricIncrementalWithinTx(ctx context.Context, metricName string, now time.Time, policy RollupPolicy, tx *sql.Tx) (int, error) {
|
||||
comp := policy.compression()
|
||||
rawCutoff := policy.rawCutoff(now)
|
||||
delta, err := s.buildFinestTierBefore(ctx, tx, metricName, policy.Tiers[0].Interval, comp, rawCutoff)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
|
||||
written := 0
|
||||
for tierIdx, tier := range policy.Tiers {
|
||||
if tierIdx > 0 {
|
||||
delta = buildCoarserBucketsFromDelta(delta, tier.Interval, comp)
|
||||
}
|
||||
n, err := s.mergeRollupBucketsTx(ctx, metricName, tier.Interval, delta, tx)
|
||||
if err != nil {
|
||||
return written, err
|
||||
}
|
||||
written += n
|
||||
}
|
||||
|
||||
if err := s.enforceRetentionWithinTx(ctx, metricName, now, policy, tx); err != nil {
|
||||
return written, err
|
||||
}
|
||||
return written, nil
|
||||
}
|
||||
|
||||
// compactMetricFullWithinTx retains the rebuild behavior required when raw
|
||||
// points are kept forever and therefore cannot act as a consumed delta queue.
|
||||
func (s *Store) compactMetricFullWithinTx(ctx context.Context, metricName string, now time.Time, policy RollupPolicy, tx *sql.Tx) (int, error) {
|
||||
comp := policy.compression()
|
||||
|
||||
written := 0
|
||||
var prevInterval time.Duration
|
||||
var prevDelta map[rollupKey]*rollupBucket
|
||||
rawCutoff := policy.rawCutoff(now)
|
||||
|
||||
for tierIdx, tier := range policy.Tiers {
|
||||
var buckets map[rollupKey]*rollupBucket
|
||||
var currentDelta map[rollupKey]*rollupBucket
|
||||
var err error
|
||||
// Reads must run on the transaction itself, not s.reader(): with a
|
||||
// single-connection pool (SQLite MaxOpenConns=1) the tx already holds
|
||||
// the only connection, so a pool read here would deadlock.
|
||||
if tierIdx == 0 {
|
||||
buckets, err = s.buildFinestTier(ctx, tx, metricName, tier.Interval, comp)
|
||||
if err == nil && !rawCutoff.IsZero() {
|
||||
cutoffNano := rawCutoff.UnixNano()
|
||||
for k := range buckets {
|
||||
if k.bucket >= cutoffNano {
|
||||
delete(buckets, k)
|
||||
}
|
||||
}
|
||||
}
|
||||
if !rawCutoff.IsZero() {
|
||||
currentDelta = buckets
|
||||
}
|
||||
} else {
|
||||
buckets, err = s.buildCoarserTier(ctx, tx, metricName, prevInterval, tier.Interval, comp)
|
||||
if prevDelta != nil {
|
||||
currentDelta = buildCoarserBucketsFromDelta(prevDelta, tier.Interval, comp)
|
||||
}
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
return written, err
|
||||
}
|
||||
// For the finest tier only, merge buckets older than the raw cutoff:
|
||||
// those buckets may already have a retained rollup while only late raw
|
||||
// samples remain in the raw table.
|
||||
var n int
|
||||
if tierIdx == 0 && !rawCutoff.IsZero() {
|
||||
n, err = s.writeRollupBucketsWithMergePointTx(ctx, metricName, tier.Interval, buckets, rawCutoff, time.Time{}, nil, tx)
|
||||
} else if tierIdx > 0 {
|
||||
deltaCutoff := alignRollupRetentionCutoff(now.Add(-policy.Tiers[tierIdx-1].Retention), tier.Interval)
|
||||
n, err = s.writeRollupBucketsWithMergePointTx(ctx, metricName, tier.Interval, buckets, time.Time{}, deltaCutoff, currentDelta, tx)
|
||||
} else {
|
||||
n, err = s.writeRollupBucketsTx(ctx, metricName, tier.Interval, buckets, tx)
|
||||
}
|
||||
if err != nil {
|
||||
return written, err
|
||||
}
|
||||
written += n
|
||||
prevInterval = tier.Interval
|
||||
prevDelta = currentDelta
|
||||
}
|
||||
|
||||
if err := s.enforceRetentionWithinTx(ctx, metricName, now, policy, tx); err != nil {
|
||||
return written, err
|
||||
}
|
||||
return written, nil
|
||||
}
|
||||
|
||||
// enforceRetentionWithinTx trims raw and rollup tiers after coarser data has
|
||||
// been materialized. Cutoffs retain a bucket that straddles the boundary.
|
||||
func (s *Store) enforceRetentionWithinTx(ctx context.Context, metricName string, now time.Time, policy RollupPolicy, tx *sql.Tx) error {
|
||||
if policy.RawRetention > 0 {
|
||||
if _, err := s.DeleteBeforeTx(ctx, metricName, policy.rawCutoff(now), tx); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
for i, tier := range policy.Tiers {
|
||||
alignment := tier.Interval
|
||||
if i+1 < len(policy.Tiers) {
|
||||
alignment = policy.Tiers[i+1].Interval
|
||||
}
|
||||
cutoff := alignRollupRetentionCutoff(now.Add(-tier.Retention), alignment)
|
||||
if err := s.deleteRollupsBeforeTx(ctx, metricName, tier.Interval, cutoff, tx); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// deleteRollupsForIntervalsTx removes rows from resolutions that are no longer
|
||||
// part of a metric's effective retention policy.
|
||||
func (s *Store) deleteRollupsForIntervalsTx(ctx context.Context, metricName string, intervals []time.Duration, tx *sql.Tx) error {
|
||||
if len(intervals) == 0 {
|
||||
return nil
|
||||
}
|
||||
args := make([]any, 1, len(intervals)+1)
|
||||
args[0] = metricName
|
||||
placeholders := make([]string, len(intervals))
|
||||
for i, interval := range intervals {
|
||||
placeholders[i] = s.dialect.placeholder(i + 2)
|
||||
args = append(args, interval.Nanoseconds())
|
||||
}
|
||||
sqlText := fmt.Sprintf(
|
||||
`DELETE FROM %s WHERE metric_name = %s AND resolution_nano IN (%s)`,
|
||||
s.tables.rollups, s.dialect.placeholder(1), strings.Join(placeholders, ", "),
|
||||
)
|
||||
_, err := tx.ExecContext(ctx, sqlText, args...)
|
||||
return err
|
||||
}
|
||||
|
||||
func alignRollupRetentionCutoff(cutoff time.Time, nextInterval time.Duration) time.Time {
|
||||
if nextInterval <= 0 {
|
||||
return cutoff.UTC()
|
||||
}
|
||||
return time.Unix(0, floorDivNano(cutoff.UTC().UnixNano(), nextInterval.Nanoseconds())).UTC()
|
||||
}
|
||||
|
||||
// rollupKey identifies one rollup cell. The tag dimension (tagsHash) is part of
|
||||
// the key so points carrying different tags never collapse into the same bucket.
|
||||
//
|
||||
// rollupKey 标识一个 rollup 单元;tagsHash 是 key 的一部分,确保不同标签
|
||||
// 的点不会落入同一个桶。
|
||||
type rollupKey struct {
|
||||
// entityID is the entity dimension of the rollup cell.
|
||||
//
|
||||
// entityID 是 rollup 单元的实体维度。
|
||||
entityID string
|
||||
// tagsHash is the stable fingerprint of the tag set.
|
||||
//
|
||||
// tagsHash 是标签集合的稳定指纹。
|
||||
tagsHash string
|
||||
// bucket is the bucket start timestamp in nanoseconds.
|
||||
//
|
||||
// bucket 是桶起始时间的纳秒时间戳。
|
||||
bucket int64
|
||||
}
|
||||
|
||||
// buildFinestTier scans raw points for the metric and groups them into buckets
|
||||
// of the given interval, keyed by (entity, tag set, bucket-start). Each point's
|
||||
// tag map determines which series it belongs to.
|
||||
//
|
||||
// buildFinestTier 扫描某指标的原始点,并按给定 interval 分桶,key 为
|
||||
// (实体、标签集合、桶起点)。每个点的标签 map 决定它属于哪条序列。
|
||||
func (s *Store) buildFinestTier(ctx context.Context, q querier, metricName string, interval time.Duration, comp float64) (map[rollupKey]*rollupBucket, error) {
|
||||
return s.buildFinestTierBefore(ctx, q, metricName, interval, comp, time.Time{})
|
||||
}
|
||||
|
||||
// buildFinestTierBefore aggregates only raw points older than before. A zero
|
||||
// cutoff preserves the full-rebuild behavior used when raw is retained forever.
|
||||
func (s *Store) buildFinestTierBefore(ctx context.Context, q querier, metricName string, interval time.Duration, comp float64, before time.Time) (map[rollupKey]*rollupBucket, error) {
|
||||
size := interval.Nanoseconds()
|
||||
out := make(map[rollupKey]*rollupBucket)
|
||||
args := []any{metricName}
|
||||
where := "metric_name = " + s.dialect.placeholder(1)
|
||||
if !before.IsZero() {
|
||||
args = append(args, before.UTC().UnixNano())
|
||||
where += " AND ts_nano < " + s.dialect.placeholder(2)
|
||||
}
|
||||
sqlText := fmt.Sprintf(
|
||||
`SELECT entity_id, ts_nano, value, tags FROM %s WHERE %s ORDER BY ts_nano ASC`,
|
||||
s.tables.points, where,
|
||||
)
|
||||
rows, err := q.QueryContext(ctx, sqlText, args...)
|
||||
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
for rows.Next() {
|
||||
var entityID string
|
||||
var ts int64
|
||||
var value float64
|
||||
var rawTags any
|
||||
if err := rows.Scan(&entityID, &ts, &value, &rawTags); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
tags, err := decodeMap(rawTags)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
hash, canonical, err := tagsFingerprint(tags)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
bucket := floorDivNano(ts, size)
|
||||
k := rollupKey{entityID: entityID, tagsHash: hash, bucket: bucket}
|
||||
b := out[k]
|
||||
if b == nil {
|
||||
b = newRollupBucket(comp)
|
||||
b.tagsHash = hash
|
||||
b.tagsJSON = canonical
|
||||
out[k] = b
|
||||
}
|
||||
b.addPoint(value, ts)
|
||||
}
|
||||
return out, rows.Err()
|
||||
}
|
||||
|
||||
// buildCoarserTier composes a coarser tier from the already-stored finer tier:
|
||||
// it reads the finer rollup rows and merges every finer bucket into the coarse
|
||||
// bucket that shares its (entity, tag set). Tag identity is preserved end to
|
||||
// end, so a coarse series only ever merges finer buckets of the same tag set.
|
||||
//
|
||||
// buildCoarserTier 基于已存储的细层 rollup 合成更粗层:它读取细层 rollup 行,
|
||||
// 并把每个细桶合并进共享相同(实体、标签集合)的粗桶。标签身份会端到端保留,
|
||||
// 因此一条粗粒度序列只会合并相同标签集合的细桶。
|
||||
func (s *Store) buildCoarserTier(ctx context.Context, q querier, metricName string, fineInterval, coarseInterval time.Duration, comp float64) (map[rollupKey]*rollupBucket, error) {
|
||||
coarseSize := coarseInterval.Nanoseconds()
|
||||
out := make(map[rollupKey]*rollupBucket)
|
||||
fineRows, err := s.scanRollupRows(ctx, q, metricName, fineInterval)
|
||||
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for _, fr := range fineRows {
|
||||
bucket := floorDivNano(fr.bucket, coarseSize)
|
||||
k := rollupKey{entityID: fr.entityID, tagsHash: fr.bucketData.tagsHash, bucket: bucket}
|
||||
b := out[k]
|
||||
if b == nil {
|
||||
b = newRollupBucket(comp)
|
||||
b.tagsHash = fr.bucketData.tagsHash
|
||||
b.tagsJSON = fr.bucketData.tagsJSON
|
||||
out[k] = b
|
||||
}
|
||||
b.mergeStored(fr.bucketData)
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func buildCoarserBucketsFromDelta(delta map[rollupKey]*rollupBucket, coarseInterval time.Duration, comp float64) map[rollupKey]*rollupBucket {
|
||||
out := make(map[rollupKey]*rollupBucket)
|
||||
if len(delta) == 0 {
|
||||
return out
|
||||
}
|
||||
coarseSize := coarseInterval.Nanoseconds()
|
||||
for k, src := range delta {
|
||||
bucket := floorDivNano(k.bucket, coarseSize)
|
||||
ck := rollupKey{entityID: k.entityID, tagsHash: k.tagsHash, bucket: bucket}
|
||||
b := out[ck]
|
||||
if b == nil {
|
||||
b = newRollupBucket(comp)
|
||||
b.tagsHash = src.tagsHash
|
||||
b.tagsJSON = src.tagsJSON
|
||||
out[ck] = b
|
||||
}
|
||||
b.mergeStored(src)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// storedRollup represents a rollup row reconstructed from storage.
|
||||
//
|
||||
// storedRollup 表示从存储中还原的一行 rollup 数据。
|
||||
type storedRollup struct {
|
||||
// entityID is the entity stored on the rollup row.
|
||||
//
|
||||
// entityID 是 rollup 行中保存的实体。
|
||||
entityID string
|
||||
// bucket is the stored bucket start timestamp in nanoseconds.
|
||||
//
|
||||
// bucket 是存储的桶起始纳秒时间戳。
|
||||
bucket int64
|
||||
// bucketData is the reconstructed in-memory accumulator for the row.
|
||||
//
|
||||
// bucketData 是该行还原出的内存累加器。
|
||||
bucketData *rollupBucket
|
||||
}
|
||||
|
||||
// scanRollupRows loads all rollup rows for a metric at a given resolution and
|
||||
// reconstructs their in-memory accumulators (including tag identity and the
|
||||
// decoded t-digest).
|
||||
//
|
||||
// scanRollupRows 读取某指标在给定分辨率下的所有 rollup 行,并还原它们的
|
||||
// 内存累加器(包括标签身份和解码后的 t-digest)。
|
||||
func (s *Store) scanRollupRows(ctx context.Context, q querier, metricName string, interval time.Duration) ([]storedRollup, error) {
|
||||
sqlText := fmt.Sprintf(
|
||||
`SELECT entity_id, tags_hash, tags, bucket_nano, count, sum, sum_sq, min_val, max_val, first_val, first_ts, last_val, last_ts, digest
|
||||
FROM %s WHERE metric_name = %s AND resolution_nano = %s ORDER BY bucket_nano ASC`,
|
||||
s.tables.rollups, s.dialect.placeholder(1), s.dialect.placeholder(2),
|
||||
)
|
||||
rows, err := q.QueryContext(ctx, sqlText, metricName, interval.Nanoseconds())
|
||||
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
return scanStoredRollups(rows)
|
||||
}
|
||||
|
||||
// mergeRollupBucketsTx merges a consumed raw-data delta into the affected
|
||||
// rollup buckets. Existing buckets are read by their indexed primary key, so a
|
||||
// compact run performs work proportional to new/late data rather than to the
|
||||
// complete retained history.
|
||||
func (s *Store) mergeRollupBucketsTx(ctx context.Context, metricName string, interval time.Duration, buckets map[rollupKey]*rollupBucket, tx *sql.Tx) (int, error) {
|
||||
if len(buckets) == 0 {
|
||||
return 0, nil
|
||||
}
|
||||
|
||||
keys := make([]rollupKey, 0, len(buckets))
|
||||
for key := range buckets {
|
||||
keys = append(keys, key)
|
||||
}
|
||||
sort.Slice(keys, func(i, j int) bool {
|
||||
if keys[i].entityID != keys[j].entityID {
|
||||
return keys[i].entityID < keys[j].entityID
|
||||
}
|
||||
if keys[i].tagsHash != keys[j].tagsHash {
|
||||
return keys[i].tagsHash < keys[j].tagsHash
|
||||
}
|
||||
return keys[i].bucket < keys[j].bucket
|
||||
})
|
||||
|
||||
stmt := s.dialect.upsertRollupSQL(s.tables)
|
||||
resolution := interval.Nanoseconds()
|
||||
createdAt := time.Now().UTC().UnixNano()
|
||||
for _, key := range keys {
|
||||
bucket := buckets[key]
|
||||
existing, err := s.readRollupBucketTx(ctx, metricName, key.entityID, key.tagsHash, interval, key.bucket, tx)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
if existing != nil {
|
||||
existing.mergeStored(bucket)
|
||||
bucket = existing
|
||||
}
|
||||
tagsJSON := bucket.tagsJSON
|
||||
if tagsJSON == "" {
|
||||
tagsJSON = "{}"
|
||||
}
|
||||
if _, err := tx.ExecContext(ctx, stmt,
|
||||
metricName, key.entityID, key.tagsHash, tagsJSON, resolution, key.bucket,
|
||||
bucket.count, bucket.sum, bucket.sumSq, bucket.min, bucket.max,
|
||||
bucket.firstVal, bucket.firstTS, bucket.lastVal, bucket.lastTS,
|
||||
bucket.digest.Encode(), createdAt,
|
||||
); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
}
|
||||
return len(keys), nil
|
||||
}
|
||||
|
||||
// writeRollupBucketsTx upserts buckets within an existing transaction.
|
||||
func (s *Store) writeRollupBucketsTx(ctx context.Context, metricName string, interval time.Duration, buckets map[rollupKey]*rollupBucket, tx *sql.Tx) (int, error) {
|
||||
return s.writeRollupBucketsWithMergePointTx(ctx, metricName, interval, buckets, time.Time{}, time.Time{}, nil, tx)
|
||||
}
|
||||
|
||||
// writeRollupBucketsWithMergePointTx is the transactional version that executes
|
||||
// all operations within an existing transaction. deltaCutoff marks coarser
|
||||
// buckets whose source tier may already have aged out; a smaller recomputed
|
||||
// bucket in that region is treated as late-arriving delta data and merged into
|
||||
// the retained coarse bucket instead of replacing it.
|
||||
func (s *Store) writeRollupBucketsWithMergePointTx(ctx context.Context, metricName string, interval time.Duration, buckets map[rollupKey]*rollupBucket, mergeCutoff, protectCutoff time.Time, deltaBuckets map[rollupKey]*rollupBucket, tx *sql.Tx) (int, error) {
|
||||
if len(buckets) == 0 {
|
||||
return 0, nil
|
||||
}
|
||||
keys := make([]rollupKey, 0, len(buckets))
|
||||
for k := range buckets {
|
||||
keys = append(keys, k)
|
||||
}
|
||||
sort.Slice(keys, func(i, j int) bool {
|
||||
if keys[i].entityID != keys[j].entityID {
|
||||
return keys[i].entityID < keys[j].entityID
|
||||
}
|
||||
if keys[i].tagsHash != keys[j].tagsHash {
|
||||
return keys[i].tagsHash < keys[j].tagsHash
|
||||
}
|
||||
return keys[i].bucket < keys[j].bucket
|
||||
})
|
||||
|
||||
stmt := s.dialect.upsertRollupSQL(s.tables)
|
||||
resNano := interval.Nanoseconds()
|
||||
now := time.Now().UTC().UnixNano()
|
||||
mergeCutoffNano := mergeCutoff.UnixNano()
|
||||
deltaCutoffNano := protectCutoff.UnixNano()
|
||||
written := 0
|
||||
|
||||
for _, k := range keys {
|
||||
b := buckets[k]
|
||||
tagsJSON := b.tagsJSON
|
||||
if tagsJSON == "" {
|
||||
tagsJSON = "{}"
|
||||
}
|
||||
|
||||
// If mergeCutoff is set and this bucket is older than the cutoff,
|
||||
// merge with existing data instead of replacing it.
|
||||
if !mergeCutoff.IsZero() && k.bucket < mergeCutoffNano {
|
||||
// Read existing row if it exists
|
||||
existing, err := s.readRollupBucketTx(ctx, metricName, k.entityID, k.tagsHash, interval, k.bucket, tx)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
if existing != nil {
|
||||
// Merge the new bucket into the existing one
|
||||
existing.mergeStored(b)
|
||||
b = existing
|
||||
}
|
||||
}
|
||||
if !protectCutoff.IsZero() && k.bucket < deltaCutoffNano {
|
||||
existing, err := s.readRollupBucketTx(ctx, metricName, k.entityID, k.tagsHash, interval, k.bucket, tx)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
if existing != nil {
|
||||
if deltaBuckets == nil {
|
||||
// Without delta tracking (raw is retained forever), the
|
||||
// source tier is rebuilt from full raw data and can replace.
|
||||
} else if delta := deltaBuckets[k]; delta != nil && delta.count > 0 {
|
||||
existing.mergeStored(delta)
|
||||
b = existing
|
||||
} else {
|
||||
continue
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Column order must match rollupColumns in dialect_rollup.go
|
||||
_, err := tx.ExecContext(ctx, stmt,
|
||||
metricName, k.entityID, k.tagsHash, tagsJSON, resNano, k.bucket,
|
||||
b.count, b.sum, b.sumSq, b.min, b.max,
|
||||
b.firstVal, b.firstTS, b.lastVal, b.lastTS,
|
||||
b.digest.Encode(), now,
|
||||
)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
written++
|
||||
}
|
||||
return written, nil
|
||||
}
|
||||
|
||||
// deleteRollupsBeforeTx deletes stored rollup rows within a transaction.
|
||||
func (s *Store) deleteRollupsBeforeTx(ctx context.Context, metricName string, interval time.Duration, before time.Time, tx *sql.Tx) error {
|
||||
sqlText := fmt.Sprintf(
|
||||
`DELETE FROM %s WHERE metric_name = %s AND resolution_nano = %s AND bucket_nano < %s`,
|
||||
s.tables.rollups, s.dialect.placeholder(1), s.dialect.placeholder(2), s.dialect.placeholder(3),
|
||||
)
|
||||
_, err := tx.ExecContext(ctx, sqlText, metricName, interval.Nanoseconds(), before.UTC().UnixNano())
|
||||
return err
|
||||
}
|
||||
|
||||
// DeleteBeforeTx deletes raw points before a cutoff within a transaction.
|
||||
func (s *Store) DeleteBeforeTx(ctx context.Context, metricName string, before time.Time, tx *sql.Tx) (int64, error) {
|
||||
sqlText := fmt.Sprintf(
|
||||
`DELETE FROM %s WHERE metric_name = %s AND ts_nano < %s`,
|
||||
s.tables.points, s.dialect.placeholder(1), s.dialect.placeholder(2),
|
||||
)
|
||||
result, err := tx.ExecContext(ctx, sqlText, metricName, before.UTC().UnixNano())
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return result.RowsAffected()
|
||||
}
|
||||
|
||||
// floorDivNano floors ts to the start of its size-wide bucket, handling
|
||||
// negative timestamps (pre-epoch) toward negative infinity so buckets align
|
||||
// consistently. Mirrors alignTime but operates on raw nanos.
|
||||
//
|
||||
// floorDivNano 将 ts 向下对齐到 size 宽桶的起点,并把负时间戳(Unix epoch 前)
|
||||
// 朝负无穷取整,让桶保持一致对齐。它与 alignTime 逻辑一致,但直接操作纳秒值。
|
||||
func floorDivNano(ts, size int64) int64 {
|
||||
rem := ((ts % size) + size) % size
|
||||
return ts - rem
|
||||
}
|
||||
|
||||
// readRollupBucketTx reads a single rollup bucket from a transaction.
|
||||
func (s *Store) readRollupBucketTx(ctx context.Context, metricName, entityID, tagsHash string, interval time.Duration, bucketNano int64, tx *sql.Tx) (*rollupBucket, error) {
|
||||
sqlText := fmt.Sprintf(
|
||||
`SELECT count, sum, sum_sq, min_val, max_val, first_val, first_ts, last_val, last_ts, digest, tags
|
||||
FROM %s WHERE metric_name = %s AND resolution_nano = %s AND entity_id = %s
|
||||
AND tags_hash = %s AND bucket_nano = %s`,
|
||||
s.tables.rollups,
|
||||
s.dialect.placeholder(1), s.dialect.placeholder(2), s.dialect.placeholder(3),
|
||||
s.dialect.placeholder(4), s.dialect.placeholder(5),
|
||||
)
|
||||
row := tx.QueryRowContext(ctx, sqlText, metricName, interval.Nanoseconds(), entityID, tagsHash, bucketNano)
|
||||
|
||||
var count int64
|
||||
var sum, sumSq, minV, maxV, firstV, lastV float64
|
||||
var firstTS, lastTS int64
|
||||
var digestBlob []byte
|
||||
var rawTags any
|
||||
|
||||
err := row.Scan(&count, &sum, &sumSq, &minV, &maxV, &firstV, &firstTS, &lastV, &lastTS, &digestBlob, &rawTags)
|
||||
if err == sql.ErrNoRows {
|
||||
return nil, nil
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
td, err := DecodeTDigest(digestBlob)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
tagsJSON, err := rawTagsToJSON(rawTags)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return &rollupBucket{
|
||||
count: count, sum: sum, sumSq: sumSq,
|
||||
min: minV, max: maxV,
|
||||
firstVal: firstV, firstTS: firstTS,
|
||||
lastVal: lastV, lastTS: lastTS,
|
||||
digest: td,
|
||||
tagsHash: tagsHash,
|
||||
tagsJSON: tagsJSON,
|
||||
}, nil
|
||||
}
|
||||
@@ -0,0 +1,366 @@
|
||||
package metric
|
||||
|
||||
import (
|
||||
"context"
|
||||
"math"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// TestRollupKeepsTagSeriesSeparate is the regression test for the bug where the
|
||||
// rollup dropped the tag dimension: two points in the SAME metric/entity/bucket
|
||||
// but with different tags must become two independent rollup series, and a tag
|
||||
// filter must return that tag's own value rather than a cross-tag merge.
|
||||
//
|
||||
// TestRollupKeepsTagSeriesSeparate 是针对 rollup 曾丢失标签维度这一问题的
|
||||
// 回归测试:同一个指标、实体和桶内带有不同标签的两个点,必须成为两条独立的
|
||||
// rollup 序列;标签过滤必须返回该标签自己的值,而不是跨标签合并结果。
|
||||
func TestRollupKeepsTagSeriesSeparate(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
policy := RollupPolicy{Tiers: []RollupTier{{Interval: time.Minute, Retention: 24 * time.Hour}}}
|
||||
s := newRollupStore(t, policy)
|
||||
if err := s.CreateMetric(ctx, Definition{Name: "util", Type: TypeGauge, RetentionDays: 30}); err != nil {
|
||||
t.Fatalf("create: %v", err)
|
||||
}
|
||||
base := time.Date(2026, 6, 18, 0, 0, 0, 0, time.UTC)
|
||||
// Same entity, same 1-minute bucket; two GPUs distinguished only by tag.
|
||||
// device 0 carries values 10..19, device 1 carries 80..89.
|
||||
var batch []Point
|
||||
for i := 0; i < 10; i++ {
|
||||
batch = append(batch,
|
||||
Point{MetricName: "util", EntityID: "host-1", Timestamp: base.Add(time.Duration(i) * time.Second), Value: float64(10 + i), Tags: map[string]string{"device_index": "0"}},
|
||||
Point{MetricName: "util", EntityID: "host-1", Timestamp: base.Add(time.Duration(i) * time.Second), Value: float64(80 + i), Tags: map[string]string{"device_index": "1"}},
|
||||
)
|
||||
}
|
||||
if err := s.WriteBatch(ctx, batch); err != nil {
|
||||
t.Fatalf("write: %v", err)
|
||||
}
|
||||
written, err := s.Compact(ctx, base.Add(10*time.Minute))
|
||||
if err != nil {
|
||||
t.Fatalf("compact: %v", err)
|
||||
}
|
||||
// Two distinct tag series in the one bucket => two rollup rows.
|
||||
if written != 2 {
|
||||
t.Fatalf("expected 2 rollup series (one per device_index), got %d", written)
|
||||
}
|
||||
|
||||
q := func(dev string) AggregateQuery {
|
||||
return AggregateQuery{
|
||||
Query: Query{
|
||||
MetricName: "util", EntityID: "host-1",
|
||||
Start: base, End: base.Add(10 * time.Minute),
|
||||
Tags: map[string]string{"device_index": dev},
|
||||
},
|
||||
Aggregation: AggAvg,
|
||||
Interval: time.Minute,
|
||||
}
|
||||
}
|
||||
// device 0: avg of 10..19 = 14.5
|
||||
res0, err := s.AggregateRollup(ctx, q("0"), time.Minute)
|
||||
if err != nil {
|
||||
t.Fatalf("rollup dev0: %v", err)
|
||||
}
|
||||
if len(res0) != 1 || res0[0].Count != 10 || math.Abs(res0[0].Value-14.5) > 1e-9 {
|
||||
t.Fatalf("device 0 should be its own series avg=14.5 count=10, got %#v", res0)
|
||||
}
|
||||
if res0[0].Tags["device_index"] != "0" {
|
||||
t.Fatalf("device 0 rollup should keep tags, got %#v", res0[0])
|
||||
}
|
||||
// device 1: avg of 80..89 = 84.5
|
||||
res1, err := s.AggregateRollup(ctx, q("1"), time.Minute)
|
||||
if err != nil {
|
||||
t.Fatalf("rollup dev1: %v", err)
|
||||
}
|
||||
if len(res1) != 1 || res1[0].Count != 10 || math.Abs(res1[0].Value-84.5) > 1e-9 {
|
||||
t.Fatalf("device 1 should be its own series avg=84.5 count=10, got %#v", res1)
|
||||
}
|
||||
if res1[0].Tags["device_index"] != "1" {
|
||||
t.Fatalf("device 1 rollup should keep tags, got %#v", res1[0])
|
||||
}
|
||||
|
||||
// No tag filter => both tag series are returned independently, matching
|
||||
// raw Aggregate and preserving the tag dimension for public query callers.
|
||||
all, err := s.AggregateRollup(ctx, AggregateQuery{
|
||||
Query: Query{MetricName: "util", EntityID: "host-1", Start: base, End: base.Add(10 * time.Minute)},
|
||||
Aggregation: AggAvg,
|
||||
Interval: time.Minute,
|
||||
PreserveSeries: true,
|
||||
}, time.Minute)
|
||||
if err != nil {
|
||||
t.Fatalf("rollup all: %v", err)
|
||||
}
|
||||
byDevice := map[string]AggregatePoint{}
|
||||
for _, point := range all {
|
||||
byDevice[point.Tags["device_index"]] = point
|
||||
}
|
||||
if len(byDevice) != 2 {
|
||||
t.Fatalf("no-filter should keep both tag series, got %#v", all)
|
||||
}
|
||||
if got := byDevice["0"]; got.Count != 10 || math.Abs(got.Value-14.5) > 1e-9 {
|
||||
t.Fatalf("unexpected no-filter device 0 rollup: %#v", got)
|
||||
}
|
||||
if got := byDevice["1"]; got.Count != 10 || math.Abs(got.Value-84.5) > 1e-9 {
|
||||
t.Fatalf("unexpected no-filter device 1 rollup: %#v", got)
|
||||
}
|
||||
}
|
||||
|
||||
// TestRollupTagPercentileSurvivesRetention proves the headline TSDB property
|
||||
// still holds per tag: after raw is deleted, a tag-filtered percentile is still
|
||||
// answerable from that tag's surviving rollup series (and not contaminated by
|
||||
// the other tag).
|
||||
//
|
||||
// TestRollupTagPercentileSurvivesRetention 验证核心 TSDB 特性在每个标签上
|
||||
// 仍然成立:原始点删除后,按标签过滤的百分位仍能从该标签保留下来的 rollup 序列
|
||||
// 回答,并且不会被另一个标签污染。
|
||||
func TestRollupTagPercentileSurvivesRetention(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
policy := RollupPolicy{
|
||||
RawRetention: 2 * time.Minute,
|
||||
Tiers: []RollupTier{{Interval: time.Minute, Retention: 365 * 24 * time.Hour}},
|
||||
}
|
||||
s := newRollupStore(t, policy)
|
||||
if err := s.CreateMetric(ctx, Definition{Name: "lat", Type: TypeGauge, RetentionDays: 30}); err != nil {
|
||||
t.Fatalf("create: %v", err)
|
||||
}
|
||||
base := time.Date(2026, 6, 18, 0, 0, 0, 0, time.UTC)
|
||||
// One 1m bucket. region=ap: values 0..99; region=eu: values 1000..1099.
|
||||
var batch []Point
|
||||
for i := 0; i < 100; i++ {
|
||||
ts := base.Add(time.Duration(i) * time.Millisecond * 500)
|
||||
batch = append(batch,
|
||||
Point{MetricName: "lat", EntityID: "n1", Timestamp: ts, Value: float64(i), Tags: map[string]string{"region": "ap"}},
|
||||
Point{MetricName: "lat", EntityID: "n1", Timestamp: ts, Value: float64(1000 + i), Tags: map[string]string{"region": "eu"}},
|
||||
)
|
||||
}
|
||||
if err := s.WriteBatch(ctx, batch); err != nil {
|
||||
t.Fatalf("write: %v", err)
|
||||
}
|
||||
if _, err := s.Compact(ctx, base.Add(48*time.Hour)); err != nil {
|
||||
t.Fatalf("compact: %v", err)
|
||||
}
|
||||
// Raw gone.
|
||||
raw, err := s.Query(ctx, Query{MetricName: "lat", EntityID: "n1", Start: base, End: base.Add(time.Hour)})
|
||||
if err != nil {
|
||||
t.Fatalf("query raw: %v", err)
|
||||
}
|
||||
if len(raw) != 0 {
|
||||
t.Fatalf("expected raw deleted, got %d", len(raw))
|
||||
}
|
||||
// p90 of region=ap must be near the ap-only exact (≈89), NOT pulled up toward
|
||||
// the eu values (which would happen if tags were merged).
|
||||
res, err := s.AggregateRollup(ctx, AggregateQuery{
|
||||
Query: Query{
|
||||
MetricName: "lat", EntityID: "n1",
|
||||
Start: base, End: base.Add(time.Hour),
|
||||
Tags: map[string]string{"region": "ap"},
|
||||
},
|
||||
Aggregation: Pxx(90),
|
||||
Interval: time.Minute,
|
||||
}, time.Minute)
|
||||
if err != nil {
|
||||
t.Fatalf("rollup ap p90: %v", err)
|
||||
}
|
||||
if len(res) != 1 {
|
||||
t.Fatalf("expected 1 ap bucket, got %d", len(res))
|
||||
}
|
||||
exactAP := percentileSortedRange(0, 99, 0.90)
|
||||
if math.Abs(res[0].Value-exactAP)/exactAP > 0.05 {
|
||||
t.Fatalf("ap p90 %v contaminated or wrong (exact ap ≈ %v)", res[0].Value, exactAP)
|
||||
}
|
||||
if res[0].Value > 500 {
|
||||
t.Fatalf("ap p90 %v is contaminated by eu series (>500)", res[0].Value)
|
||||
}
|
||||
}
|
||||
|
||||
// TestSeriesTagFilterRoutesPerTag checks the auto-routing read path honors tags
|
||||
// on both the raw branch (recent) and the rollup branch (old).
|
||||
//
|
||||
// TestSeriesTagFilterRoutesPerTag 检查自动路由读取路径在原始分支(近期数据)和
|
||||
// rollup 分支(旧数据)上都遵守标签。
|
||||
func TestSeriesTagFilterRoutesPerTag(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
policy := RollupPolicy{
|
||||
RawRetention: time.Hour,
|
||||
Tiers: []RollupTier{{Interval: time.Minute, Retention: 30 * 24 * time.Hour}},
|
||||
}
|
||||
s := newRollupStore(t, policy)
|
||||
if err := s.CreateMetric(ctx, Definition{Name: "m", Type: TypeGauge, RetentionDays: 30}); err != nil {
|
||||
t.Fatalf("create: %v", err)
|
||||
}
|
||||
now := time.Date(2026, 6, 18, 12, 0, 0, 0, time.UTC)
|
||||
old := now.Add(-48 * time.Hour)
|
||||
// Old bucket: env=prod values 0..9, env=stage values 500..509.
|
||||
var batch []Point
|
||||
for i := 0; i < 10; i++ {
|
||||
ts := old.Add(time.Duration(i) * time.Second)
|
||||
batch = append(batch,
|
||||
Point{MetricName: "m", EntityID: "n1", Timestamp: ts, Value: float64(i), Tags: map[string]string{"env": "prod"}},
|
||||
Point{MetricName: "m", EntityID: "n1", Timestamp: ts, Value: float64(500 + i), Tags: map[string]string{"env": "stage"}},
|
||||
)
|
||||
}
|
||||
if err := s.WriteBatch(ctx, batch); err != nil {
|
||||
t.Fatalf("write: %v", err)
|
||||
}
|
||||
if _, err := s.Compact(ctx, now); err != nil {
|
||||
t.Fatalf("compact: %v", err)
|
||||
}
|
||||
// Old window, raw aged out -> Series must use rollup, filtered to env=prod
|
||||
// (avg of 0..9 = 4.5), not the merged 252.
|
||||
got, err := s.Series(ctx, AggregateQuery{
|
||||
Query: Query{
|
||||
MetricName: "m", EntityID: "n1",
|
||||
Start: old.Add(-time.Minute), End: old.Add(2 * time.Minute),
|
||||
Tags: map[string]string{"env": "prod"},
|
||||
},
|
||||
Aggregation: AggAvg,
|
||||
Interval: time.Minute,
|
||||
}, now)
|
||||
if err != nil {
|
||||
t.Fatalf("series old prod: %v", err)
|
||||
}
|
||||
var found bool
|
||||
for _, p := range got {
|
||||
if p.Count == 10 && math.Abs(p.Value-4.5) < 1e-9 {
|
||||
found = true
|
||||
}
|
||||
if p.Count > 10 {
|
||||
t.Fatalf("env=prod bucket contaminated by stage: %#v", p)
|
||||
}
|
||||
}
|
||||
if !found {
|
||||
t.Fatalf("env=prod old series not served correctly from rollup: %#v", got)
|
||||
}
|
||||
}
|
||||
|
||||
// TestDeleteSeriesRemovesTaggedTaskAcrossAgents verifies a task_id tag can be
|
||||
// deleted across every agent, including stored rollups.
|
||||
//
|
||||
// TestDeleteSeriesRemovesTaggedTaskAcrossAgents 验证可以跨所有 agent 删除某个
|
||||
// task_id 标签,包括已存储的 rollup。
|
||||
func TestDeleteSeriesRemovesTaggedTaskAcrossAgents(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
policy := RollupPolicy{
|
||||
RawRetention: 2 * time.Minute,
|
||||
Tiers: []RollupTier{{Interval: time.Minute, Retention: 24 * time.Hour}},
|
||||
}
|
||||
s := newRollupStore(t, policy)
|
||||
if err := s.CreateMetric(ctx, Definition{Name: "ping.latency_ms", Type: TypeGauge, RetentionDays: 30}); err != nil {
|
||||
t.Fatalf("create: %v", err)
|
||||
}
|
||||
base := time.Date(2026, 6, 18, 0, 0, 0, 0, time.UTC)
|
||||
var batch []Point
|
||||
for _, agent := range []string{"agent-001", "agent-002"} {
|
||||
for i := 0; i < 60; i++ {
|
||||
ts := base.Add(time.Duration(i) * time.Second)
|
||||
batch = append(batch,
|
||||
Point{MetricName: "ping.latency_ms", EntityID: agent, Timestamp: ts, Value: float64(20 + i), Tags: map[string]string{"task_id": "pingtask1"}},
|
||||
Point{MetricName: "ping.latency_ms", EntityID: agent, Timestamp: ts, Value: float64(60 + i), Tags: map[string]string{"task_id": "pingtask2"}},
|
||||
)
|
||||
}
|
||||
}
|
||||
if err := s.WriteBatch(ctx, batch); err != nil {
|
||||
t.Fatalf("write: %v", err)
|
||||
}
|
||||
if _, err := s.Compact(ctx, base.Add(time.Hour)); err != nil {
|
||||
t.Fatalf("compact: %v", err)
|
||||
}
|
||||
if _, err := s.DeleteSeries(ctx, Query{
|
||||
MetricName: "ping.latency_ms",
|
||||
Tags: map[string]string{"task_id": "pingtask2"},
|
||||
}); err != nil {
|
||||
t.Fatalf("delete series: %v", err)
|
||||
}
|
||||
|
||||
task2, err := s.AggregateRollup(ctx, AggregateQuery{
|
||||
Query: Query{
|
||||
MetricName: "ping.latency_ms",
|
||||
Start: base, End: base.Add(time.Minute),
|
||||
Tags: map[string]string{"task_id": "pingtask2"},
|
||||
},
|
||||
Aggregation: AggAvg,
|
||||
Interval: time.Minute,
|
||||
}, time.Minute)
|
||||
if err != nil {
|
||||
t.Fatalf("rollup task2: %v", err)
|
||||
}
|
||||
if len(task2) != 0 {
|
||||
t.Fatalf("deleted task still has rollup buckets: %#v", task2)
|
||||
}
|
||||
task1, err := s.AggregateRollup(ctx, AggregateQuery{
|
||||
Query: Query{
|
||||
MetricName: "ping.latency_ms",
|
||||
Start: base, End: base.Add(time.Minute),
|
||||
Tags: map[string]string{"task_id": "pingtask1"},
|
||||
},
|
||||
Aggregation: AggAvg,
|
||||
Interval: time.Minute,
|
||||
}, time.Minute)
|
||||
if err != nil {
|
||||
t.Fatalf("rollup task1: %v", err)
|
||||
}
|
||||
if len(task1) != 1 || task1[0].Count != 120 {
|
||||
t.Fatalf("remaining task should keep both agents, got %#v", task1)
|
||||
}
|
||||
}
|
||||
|
||||
// TestDeleteMetricRemovesRollups verifies metric deletion clears rollup rows.
|
||||
//
|
||||
// TestDeleteMetricRemovesRollups 验证删除指标会清理 rollup 行。
|
||||
func TestDeleteMetricRemovesRollups(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
s := newRollupStore(t, RollupPolicy{
|
||||
Tiers: []RollupTier{{Interval: time.Minute, Retention: 24 * time.Hour}},
|
||||
})
|
||||
if err := s.CreateMetric(ctx, Definition{Name: "gone", Type: TypeGauge, RetentionDays: 30}); err != nil {
|
||||
t.Fatalf("create: %v", err)
|
||||
}
|
||||
base := time.Date(2026, 6, 18, 0, 0, 0, 0, time.UTC)
|
||||
if err := s.Write(ctx, Point{MetricName: "gone", EntityID: "agent-001", Timestamp: base, Value: 1}); err != nil {
|
||||
t.Fatalf("write: %v", err)
|
||||
}
|
||||
if _, err := s.Compact(ctx, base.Add(time.Hour)); err != nil {
|
||||
t.Fatalf("compact: %v", err)
|
||||
}
|
||||
if err := s.DeleteMetric(ctx, "gone"); err != nil {
|
||||
t.Fatalf("delete metric: %v", err)
|
||||
}
|
||||
rows, err := s.scanRollupRows(ctx, s.reader(), "gone", time.Minute)
|
||||
|
||||
if err != nil {
|
||||
t.Fatalf("scan rollups: %v", err)
|
||||
}
|
||||
if len(rows) != 0 {
|
||||
t.Fatalf("deleted metric still has rollups: %#v", rows)
|
||||
}
|
||||
}
|
||||
|
||||
// TestTagsFingerprintStable verifies equal tag maps fingerprint identically
|
||||
// regardless of construction order, and different maps differ.
|
||||
//
|
||||
// TestTagsFingerprintStable 验证相同标签 map 无论构造顺序如何都会生成相同指纹,
|
||||
// 而不同 map 会得到不同指纹。
|
||||
func TestTagsFingerprintStable(t *testing.T) {
|
||||
a := map[string]string{"region": "ap", "device_index": "3"}
|
||||
b := map[string]string{"device_index": "3", "region": "ap"}
|
||||
ha, _, err := tagsFingerprint(a)
|
||||
if err != nil {
|
||||
t.Fatalf("fp a: %v", err)
|
||||
}
|
||||
hb, _, err := tagsFingerprint(b)
|
||||
if err != nil {
|
||||
t.Fatalf("fp b: %v", err)
|
||||
}
|
||||
if ha != hb {
|
||||
t.Fatalf("equal tag maps must fingerprint identically: %s vs %s", ha, hb)
|
||||
}
|
||||
hc, _, _ := tagsFingerprint(map[string]string{"region": "eu", "device_index": "3"})
|
||||
if ha == hc {
|
||||
t.Fatalf("different tag maps must fingerprint differently")
|
||||
}
|
||||
// nil and empty map agree.
|
||||
hn, _, _ := tagsFingerprint(nil)
|
||||
he, _, _ := tagsFingerprint(map[string]string{})
|
||||
if hn != he {
|
||||
t.Fatalf("nil and empty tag maps must agree: %s vs %s", hn, he)
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large.
Load diff
+1295
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,146 @@
|
||||
package metric
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"math"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// TestSQLiteStoreWriteQueryAggregate verifies SQLite write, query, aggregate, and stats.
|
||||
//
|
||||
// TestSQLiteStoreWriteQueryAggregate 验证 SQLite 写入、查询、聚合和统计。
|
||||
func TestSQLiteStoreWriteQueryAggregate(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
store, err := Open(ctx, SQLite("file:test-metric?mode=memory&cache=shared"))
|
||||
if err != nil {
|
||||
t.Fatalf("open sqlite store: %v", err)
|
||||
}
|
||||
defer store.Close()
|
||||
|
||||
if err := store.CreateMetric(ctx, Definition{Name: "cpu.usage", Type: TypeGauge, Unit: "%", RetentionDays: 30}); err != nil {
|
||||
t.Fatalf("create metric: %v", err)
|
||||
}
|
||||
|
||||
base := time.Date(2026, 6, 18, 10, 0, 0, 0, time.UTC)
|
||||
points := []Point{
|
||||
{MetricName: "cpu.usage", EntityID: "server-1", Timestamp: base, Value: 10, Tags: map[string]string{"region": "ap"}},
|
||||
{MetricName: "cpu.usage", EntityID: "server-1", Timestamp: base.Add(time.Minute), Value: 20, Tags: map[string]string{"region": "ap"}},
|
||||
{MetricName: "cpu.usage", EntityID: "server-1", Timestamp: base.Add(2 * time.Minute), Value: 30, Tags: map[string]string{"region": "ap"}},
|
||||
{MetricName: "cpu.usage", EntityID: "server-2", Timestamp: base.Add(time.Minute), Value: 99, Tags: map[string]string{"region": "eu"}},
|
||||
}
|
||||
if err := store.WriteBatch(ctx, points); err != nil {
|
||||
t.Fatalf("write batch: %v", err)
|
||||
}
|
||||
|
||||
got, err := store.Query(ctx, Query{
|
||||
MetricName: "cpu.usage",
|
||||
EntityID: "server-1",
|
||||
Start: base.Add(-time.Second),
|
||||
End: base.Add(3 * time.Minute),
|
||||
Tags: map[string]string{"region": "ap"},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("query: %v", err)
|
||||
}
|
||||
if len(got) != 3 {
|
||||
t.Fatalf("expected 3 points, got %d", len(got))
|
||||
}
|
||||
if got[0].Value != 10 || got[2].Value != 30 {
|
||||
t.Fatalf("unexpected ordered values: %#v", got)
|
||||
}
|
||||
|
||||
agg, err := store.Aggregate(ctx, AggregateQuery{
|
||||
Query: Query{
|
||||
MetricName: "cpu.usage",
|
||||
EntityID: "server-1",
|
||||
Start: base,
|
||||
End: base.Add(3 * time.Minute),
|
||||
},
|
||||
Aggregation: AggAvg,
|
||||
Interval: 2 * time.Minute,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("aggregate: %v", err)
|
||||
}
|
||||
if len(agg) != 2 {
|
||||
t.Fatalf("expected 2 aggregate buckets, got %d", len(agg))
|
||||
}
|
||||
if agg[0].Value != 15 || agg[0].Count != 2 {
|
||||
t.Fatalf("unexpected first aggregate: %#v", agg[0])
|
||||
}
|
||||
|
||||
stats, err := store.Stats(ctx, Query{
|
||||
MetricName: "cpu.usage",
|
||||
EntityID: "server-1",
|
||||
Start: base,
|
||||
End: base.Add(3 * time.Minute),
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("stats: %v", err)
|
||||
}
|
||||
if stats.Count != 3 || stats.Avg != 20 || stats.P95 != 29 {
|
||||
t.Fatalf("unexpected stats: %#v", stats)
|
||||
}
|
||||
}
|
||||
|
||||
func TestWriteRejectsNonFiniteValues(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
store, err := Open(ctx, SQLite("file:test-non-finite?mode=memory&cache=shared"))
|
||||
if err != nil {
|
||||
t.Fatalf("open sqlite store: %v", err)
|
||||
}
|
||||
defer store.Close()
|
||||
|
||||
for _, value := range []float64{math.NaN(), math.Inf(1), math.Inf(-1)} {
|
||||
err := store.Write(ctx, Point{
|
||||
MetricName: "bad",
|
||||
EntityID: "server-1",
|
||||
Timestamp: time.Now(),
|
||||
Value: value,
|
||||
})
|
||||
if !errors.Is(err, ErrInvalidArgument) {
|
||||
t.Fatalf("expected ErrInvalidArgument for %v, got %v", value, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestSQLiteInDirCreatesDirectoryAndAppliesPragmas verifies SQLite file setup and PRAGMAs.
|
||||
//
|
||||
// TestSQLiteInDirCreatesDirectoryAndAppliesPragmas 验证 SQLite 文件初始化和 PRAGMA 设置。
|
||||
func TestSQLiteInDirCreatesDirectoryAndAppliesPragmas(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
dir := filepath.Join(t.TempDir(), "metrics")
|
||||
store, err := Open(ctx, SQLiteInDir(
|
||||
dir,
|
||||
WithSQLiteProfile(SQLiteProfilePerformance),
|
||||
WithSQLiteCacheSizeKB(32*1024),
|
||||
))
|
||||
if err != nil {
|
||||
t.Fatalf("open sqlite dir store: %v", err)
|
||||
}
|
||||
defer store.Close()
|
||||
|
||||
if _, err := os.Stat(filepath.Join(dir, "metrics.db")); err != nil {
|
||||
t.Fatalf("expected sqlite database file to be created: %v", err)
|
||||
}
|
||||
|
||||
var journalMode string
|
||||
if err := store.db.QueryRowContext(ctx, "PRAGMA journal_mode").Scan(&journalMode); err != nil {
|
||||
t.Fatalf("query journal mode: %v", err)
|
||||
}
|
||||
if journalMode != "wal" {
|
||||
t.Fatalf("expected WAL journal mode, got %q", journalMode)
|
||||
}
|
||||
|
||||
var synchronous int
|
||||
if err := store.db.QueryRowContext(ctx, "PRAGMA synchronous").Scan(&synchronous); err != nil {
|
||||
t.Fatalf("query synchronous: %v", err)
|
||||
}
|
||||
if synchronous != 0 {
|
||||
t.Fatalf("expected performance profile synchronous=OFF(0), got %d", synchronous)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,331 @@
|
||||
package metric
|
||||
|
||||
import (
|
||||
"encoding/binary"
|
||||
"errors"
|
||||
"math"
|
||||
"sort"
|
||||
)
|
||||
|
||||
// TDigest is a mergeable sketch for estimating arbitrary quantiles of a stream
|
||||
// of float64 values. It is the piece that makes downsampling lossless *enough*
|
||||
// for percentiles: count/sum/min/max can be re-aggregated exactly when rolling
|
||||
// finer buckets into coarser ones, but a percentile cannot be recovered from
|
||||
// those scalars. Storing a small t-digest per rollup bucket lets a coarse
|
||||
// bucket's pXX be computed by merging the digests of the finer buckets it
|
||||
// covers, with bounded error and bounded size.
|
||||
//
|
||||
// This is the "merging" variant of Dunning & Ertl's t-digest. Centroids near
|
||||
// the median are allowed to absorb more weight (lower resolution where the CDF
|
||||
// is flat) while centroids in the tails stay small (high resolution where
|
||||
// accuracy matters), giving relative error that is small at the extremes — the
|
||||
// regime that matters for p95/p99/p99.9 latency work.
|
||||
//
|
||||
// TDigest 是用于估算 float64 数据流任意分位数的可合并 sketch。它让降采样对
|
||||
// 百分位来说“足够无损”:count/sum/min/max 可以在把细桶合入粗桶时精确重聚合,
|
||||
// 但仅靠这些标量无法恢复百分位。每个 rollup 桶保存一个小型 t-digest 后,
|
||||
// 粗桶的 pXX 就可以通过合并其覆盖的细桶 digest 来计算,并保持有界误差和有界大小。
|
||||
//
|
||||
// 这是 Dunning 与 Ertl 的 t-digest 的“merging”变体。靠近中位数的质心允许吸收
|
||||
// 更多权重(CDF 平坦处使用较低分辨率),尾部质心则保持较小(精度重要处使用
|
||||
// 较高分辨率),从而在极端位置取得较小相对误差,这正是 p95/p99/p99.9 延迟分析
|
||||
// 最关心的区间。
|
||||
type TDigest struct {
|
||||
// compression controls the size/accuracy tradeoff.
|
||||
//
|
||||
// compression 控制大小与精度之间的取舍。
|
||||
compression float64
|
||||
// centroids stores the digest's weighted clusters.
|
||||
//
|
||||
// centroids 保存 digest 的加权簇。
|
||||
centroids []centroid
|
||||
// count is the total observed weight.
|
||||
//
|
||||
// count 是已观测的总权重。
|
||||
count float64
|
||||
// min is the minimum observed value.
|
||||
//
|
||||
// min 是已观测的最小值。
|
||||
min float64
|
||||
// max is the maximum observed value.
|
||||
//
|
||||
// max 是已观测的最大值。
|
||||
max float64
|
||||
// processed reports whether centroids is sorted+merged. Add() appends
|
||||
// unprocessed singletons and flips this false; process() restores it.
|
||||
//
|
||||
// processed 表示 centroids 是否已排序并合并。Add() 会追加未处理的单点并把它
|
||||
// 置为 false;process() 会恢复它。
|
||||
processed bool
|
||||
}
|
||||
|
||||
// centroid stores one t-digest centroid.
|
||||
//
|
||||
// centroid 保存一个 t-digest 质心。
|
||||
type centroid struct {
|
||||
// mean is the centroid's weighted mean.
|
||||
//
|
||||
// mean 是质心的加权均值。
|
||||
mean float64
|
||||
// weight is the total weight represented by the centroid.
|
||||
//
|
||||
// weight 是该质心代表的总权重。
|
||||
weight float64
|
||||
}
|
||||
|
||||
const (
|
||||
// defaultTDigestCompression is used when a caller supplies no useful value.
|
||||
//
|
||||
// defaultTDigestCompression 在调用方未提供有效值时使用。
|
||||
defaultTDigestCompression = 100.0
|
||||
// tdigestMagic0 is the first magic byte in the binary format.
|
||||
//
|
||||
// tdigestMagic0 是二进制格式的第一个 magic 字节。
|
||||
tdigestMagic0 = 'T'
|
||||
// tdigestMagic1 is the second magic byte in the binary format.
|
||||
//
|
||||
// tdigestMagic1 是二进制格式的第二个 magic 字节。
|
||||
tdigestMagic1 = 'D'
|
||||
// tdigestVersion is the current binary encoding version.
|
||||
//
|
||||
// tdigestVersion 是当前二进制编码版本。
|
||||
tdigestVersion = 1
|
||||
)
|
||||
|
||||
// NewTDigest returns an empty digest. compression trades size for accuracy;
|
||||
// higher keeps more centroids. Values <= 1 fall back to the default (100),
|
||||
// which keeps each digest to a few KB while holding tail error to well under
|
||||
// 1% for typical distributions.
|
||||
//
|
||||
// NewTDigest 返回一个空 digest。compression 在大小和精度之间取舍;值越高会保留
|
||||
// 更多质心。小于等于 1 的值会回退到默认值(100),这通常能把每个 digest 控制在
|
||||
// 几 KB,同时让典型分布的尾部误差远低于 1%。
|
||||
func NewTDigest(compression float64) *TDigest {
|
||||
if compression <= 1 {
|
||||
compression = defaultTDigestCompression
|
||||
}
|
||||
return &TDigest{
|
||||
compression: compression,
|
||||
min: math.Inf(1),
|
||||
max: math.Inf(-1),
|
||||
processed: true,
|
||||
}
|
||||
}
|
||||
|
||||
// Add folds a single observation with weight w (w must be > 0) into the digest.
|
||||
//
|
||||
// Add 将一个观测值及其权重合入摘要;权重必须大于 0。
|
||||
func (t *TDigest) Add(x, w float64) {
|
||||
if w <= 0 || math.IsNaN(x) || math.IsInf(x, 0) {
|
||||
return
|
||||
}
|
||||
t.centroids = append(t.centroids, centroid{mean: x, weight: w})
|
||||
t.count += w
|
||||
if x < t.min {
|
||||
t.min = x
|
||||
}
|
||||
if x > t.max {
|
||||
t.max = x
|
||||
}
|
||||
t.processed = false
|
||||
// Bound the unprocessed buffer so a long stream cannot grow memory without
|
||||
// limit; process() collapses it back to ~compression centroids.
|
||||
if len(t.centroids) > int(8*t.compression)+16 {
|
||||
t.process()
|
||||
}
|
||||
}
|
||||
|
||||
// Merge folds every centroid of other into t. This is the operation rollup
|
||||
// composition relies on: a coarse bucket merges the digests of the finer
|
||||
// buckets it spans.
|
||||
//
|
||||
// Merge 将 other 的每个质心合入 t。rollup 合成依赖这个操作:粗桶会合并它所覆盖的
|
||||
// 细桶 digest。
|
||||
func (t *TDigest) Merge(other *TDigest) {
|
||||
if other == nil || other.count == 0 {
|
||||
return
|
||||
}
|
||||
other.process()
|
||||
for _, c := range other.centroids {
|
||||
t.centroids = append(t.centroids, c)
|
||||
}
|
||||
t.count += other.count
|
||||
if other.min < t.min {
|
||||
t.min = other.min
|
||||
}
|
||||
if other.max > t.max {
|
||||
t.max = other.max
|
||||
}
|
||||
t.processed = false
|
||||
t.process()
|
||||
}
|
||||
|
||||
// Count returns the total weight observed.
|
||||
//
|
||||
// Count 返回已观测样本的总权重。
|
||||
func (t *TDigest) Count() float64 { return t.count }
|
||||
|
||||
// process sorts the buffered centroids by mean and merges adjacent ones while
|
||||
// the merged weight stays under the quantile-dependent size limit
|
||||
// 4*N*q*(1-q)/compression. That limit is generous near q=0.5 and tightens to
|
||||
// near zero in the tails, which is exactly the t-digest accuracy profile.
|
||||
//
|
||||
// process 将缓冲质心按均值排序,并在合并后权重仍低于分位数相关大小限制
|
||||
// 4*N*q*(1-q)/compression 时合并相邻质心。这个限制在 q=0.5 附近较宽松,
|
||||
// 到尾部会收紧到接近零,这正是 t-digest 的精度分布特征。
|
||||
func (t *TDigest) process() {
|
||||
if t.processed {
|
||||
return
|
||||
}
|
||||
if len(t.centroids) == 0 {
|
||||
t.processed = true
|
||||
return
|
||||
}
|
||||
sort.Slice(t.centroids, func(i, j int) bool {
|
||||
return t.centroids[i].mean < t.centroids[j].mean
|
||||
})
|
||||
total := t.count
|
||||
merged := t.centroids[:0:0] // fresh backing array; don't alias input mid-merge
|
||||
cur := t.centroids[0]
|
||||
weightBefore := 0.0
|
||||
for i := 1; i < len(t.centroids); i++ {
|
||||
next := t.centroids[i]
|
||||
proposed := cur.weight + next.weight
|
||||
// Quantile at the center of the proposed combined centroid.
|
||||
q := (weightBefore + proposed/2) / total
|
||||
limit := 4 * total * q * (1 - q) / t.compression
|
||||
if proposed <= limit || limit < 1 && proposed <= 1 {
|
||||
// Weighted-mean update keeps the centroid's mean exact.
|
||||
cur.mean += next.weight * (next.mean - cur.mean) / proposed
|
||||
cur.weight = proposed
|
||||
} else {
|
||||
merged = append(merged, cur)
|
||||
weightBefore += cur.weight
|
||||
cur = next
|
||||
}
|
||||
}
|
||||
merged = append(merged, cur)
|
||||
t.centroids = merged
|
||||
t.processed = true
|
||||
}
|
||||
|
||||
// Quantile estimates the value at q in [0,1] using linear interpolation between
|
||||
// centroid centers, with the extreme tails anchored to the observed min/max.
|
||||
//
|
||||
// Quantile 用质心中心之间的线性插值估算 [0,1] 分位点,极端尾部锚定到
|
||||
// 已观测的最小值和最大值。
|
||||
func (t *TDigest) Quantile(q float64) float64 {
|
||||
t.process()
|
||||
n := len(t.centroids)
|
||||
if n == 0 {
|
||||
return math.NaN()
|
||||
}
|
||||
if q <= 0 {
|
||||
return t.min
|
||||
}
|
||||
if q >= 1 {
|
||||
return t.max
|
||||
}
|
||||
if n == 1 {
|
||||
return t.centroids[0].mean
|
||||
}
|
||||
index := q * t.count
|
||||
|
||||
// Head: between the observed min and the first centroid's center.
|
||||
c0 := t.centroids[0]
|
||||
if index < c0.weight/2 {
|
||||
z := index / (c0.weight / 2)
|
||||
return t.min + (c0.mean-t.min)*z
|
||||
}
|
||||
weightSoFar := c0.weight / 2
|
||||
for i := 0; i < n-1; i++ {
|
||||
c := t.centroids[i]
|
||||
next := t.centroids[i+1]
|
||||
dw := (c.weight + next.weight) / 2
|
||||
if index < weightSoFar+dw {
|
||||
z := (index - weightSoFar) / dw
|
||||
return c.mean*(1-z) + next.mean*z
|
||||
}
|
||||
weightSoFar += dw
|
||||
}
|
||||
// Tail: between the last centroid's center and the observed max.
|
||||
cl := t.centroids[n-1]
|
||||
z := (index - weightSoFar) / (cl.weight / 2)
|
||||
if z > 1 {
|
||||
z = 1
|
||||
}
|
||||
return cl.mean + (t.max-cl.mean)*z
|
||||
}
|
||||
|
||||
// Encode serializes the (processed) digest to a compact little-endian blob:
|
||||
// magic[2] version[1] compression[8] min[8] max[8] count[8] nCentroids[4]
|
||||
// then nCentroids * (mean[8] weight[8]).
|
||||
//
|
||||
// Encode 将处理后的 digest 序列化为紧凑的小端二进制 blob:
|
||||
// magic[2] version[1] compression[8] min[8] max[8] count[8] nCentroids[4],
|
||||
// 后接 nCentroids * (mean[8] weight[8])。
|
||||
func (t *TDigest) Encode() []byte {
|
||||
t.process()
|
||||
n := len(t.centroids)
|
||||
buf := make([]byte, 0, 3+8*4+4+n*16)
|
||||
buf = append(buf, tdigestMagic0, tdigestMagic1, tdigestVersion)
|
||||
var tmp [8]byte
|
||||
putF := func(f float64) {
|
||||
binary.LittleEndian.PutUint64(tmp[:], math.Float64bits(f))
|
||||
buf = append(buf, tmp[:]...)
|
||||
}
|
||||
putF(t.compression)
|
||||
putF(t.min)
|
||||
putF(t.max)
|
||||
putF(t.count)
|
||||
var u32 [4]byte
|
||||
binary.LittleEndian.PutUint32(u32[:], uint32(n))
|
||||
buf = append(buf, u32[:]...)
|
||||
for _, c := range t.centroids {
|
||||
putF(c.mean)
|
||||
putF(c.weight)
|
||||
}
|
||||
return buf
|
||||
}
|
||||
|
||||
// DecodeTDigest reconstructs a digest produced by Encode. A nil/empty blob
|
||||
// yields an empty digest so callers can treat "no sketch stored" uniformly.
|
||||
//
|
||||
// DecodeTDigest 还原 Encode 生成的摘要;nil 或空 blob 会返回空摘要,
|
||||
// 方便调用方统一处理“没有保存 sketch”的情况。
|
||||
func DecodeTDigest(b []byte) (*TDigest, error) {
|
||||
if len(b) == 0 {
|
||||
return NewTDigest(defaultTDigestCompression), nil
|
||||
}
|
||||
if len(b) < 3+8*4+4 || b[0] != tdigestMagic0 || b[1] != tdigestMagic1 {
|
||||
return nil, errors.New("metric: invalid t-digest blob")
|
||||
}
|
||||
if b[2] != tdigestVersion {
|
||||
return nil, errors.New("metric: unsupported t-digest version")
|
||||
}
|
||||
off := 3
|
||||
getF := func() float64 {
|
||||
v := math.Float64frombits(binary.LittleEndian.Uint64(b[off : off+8]))
|
||||
off += 8
|
||||
return v
|
||||
}
|
||||
t := &TDigest{processed: true}
|
||||
t.compression = getF()
|
||||
t.min = getF()
|
||||
t.max = getF()
|
||||
t.count = getF()
|
||||
n := int(binary.LittleEndian.Uint32(b[off : off+4]))
|
||||
off += 4
|
||||
if n < 0 || off+n*16 > len(b) {
|
||||
return nil, errors.New("metric: truncated t-digest blob")
|
||||
}
|
||||
t.centroids = make([]centroid, n)
|
||||
for i := 0; i < n; i++ {
|
||||
t.centroids[i].mean = getF()
|
||||
t.centroids[i].weight = getF()
|
||||
}
|
||||
if t.compression <= 1 {
|
||||
t.compression = defaultTDigestCompression
|
||||
}
|
||||
return t, nil
|
||||
}
|
||||
@@ -0,0 +1,101 @@
|
||||
package metric
|
||||
|
||||
import (
|
||||
"math"
|
||||
"math/rand"
|
||||
"sort"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// exactQuantile computes an exact quantile for a sample slice.
|
||||
//
|
||||
// exactQuantile 为样本切片计算精确分位数。
|
||||
func exactQuantile(xs []float64, q float64) float64 {
|
||||
s := append([]float64(nil), xs...)
|
||||
sort.Float64s(s)
|
||||
return percentileSorted(s, q)
|
||||
}
|
||||
|
||||
// TestTDigestAccuracySmoke checks basic t-digest accuracy.
|
||||
//
|
||||
// TestTDigestAccuracySmoke 检查 t-digest 的基本精度。
|
||||
func TestTDigestAccuracySmoke(t *testing.T) {
|
||||
rng := rand.New(rand.NewSource(42))
|
||||
var xs []float64
|
||||
td := NewTDigest(100)
|
||||
for i := 0; i < 100000; i++ {
|
||||
// Exponential-ish: skewed, heavy right tail (latency-like).
|
||||
x := math.Abs(rng.NormFloat64())*100 + rng.ExpFloat64()*50
|
||||
xs = append(xs, x)
|
||||
td.Add(x, 1)
|
||||
}
|
||||
for _, q := range []float64{0.5, 0.9, 0.95, 0.99, 0.999} {
|
||||
exact := exactQuantile(xs, q)
|
||||
est := td.Quantile(q)
|
||||
relErr := math.Abs(est-exact) / math.Abs(exact)
|
||||
t.Logf("q=%.3f exact=%.3f est=%.3f relErr=%.4f", q, exact, est, relErr)
|
||||
if relErr > 0.02 {
|
||||
t.Errorf("q=%.3f relErr %.4f exceeds 2%%", q, relErr)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Rollup composition: many finer buckets, each a digest over points drawn from
|
||||
// the SAME distribution, merged into one coarse digest. Its quantiles must
|
||||
// track the quantiles of all the raw points combined.
|
||||
//
|
||||
// 该测试模拟 rollup 合成:多个细桶的 digest 合并成一个粗桶 digest,合并后
|
||||
// 的分位数应接近所有原始点合在一起计算出的精确分位数。
|
||||
func TestTDigestMergeMatchesCombined(t *testing.T) {
|
||||
rng := rand.New(rand.NewSource(7))
|
||||
var all []float64
|
||||
coarse := NewTDigest(100)
|
||||
for b := 0; b < 24; b++ { // 24 fine buckets -> 1 coarse
|
||||
fine := NewTDigest(100)
|
||||
for i := 0; i < 5000; i++ {
|
||||
x := math.Abs(rng.NormFloat64())*80 + 100
|
||||
all = append(all, x)
|
||||
fine.Add(x, 1)
|
||||
}
|
||||
coarse.Merge(fine)
|
||||
}
|
||||
if coarse.Count() != float64(len(all)) {
|
||||
t.Fatalf("merged weight %v != %d points", coarse.Count(), len(all))
|
||||
}
|
||||
for _, q := range []float64{0.5, 0.9, 0.95, 0.99, 0.999} {
|
||||
exact := exactQuantile(all, q)
|
||||
est := coarse.Quantile(q)
|
||||
relErr := math.Abs(est-exact) / math.Abs(exact)
|
||||
t.Logf("merged q=%.3f exact=%.3f est=%.3f relErr=%.4f", q, exact, est, relErr)
|
||||
if relErr > 0.02 {
|
||||
t.Errorf("merged q=%.3f relErr %.4f exceeds 2%%", q, relErr)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestTDigestEncodeRoundTrip verifies t-digest serialization.
|
||||
//
|
||||
// TestTDigestEncodeRoundTrip 验证 t-digest 编码和解码往返。
|
||||
func TestTDigestEncodeRoundTrip(t *testing.T) {
|
||||
td := NewTDigest(50)
|
||||
for i := 0; i < 2000; i++ {
|
||||
td.Add(float64(i%137)+0.5, 1)
|
||||
}
|
||||
blob := td.Encode()
|
||||
back, err := DecodeTDigest(blob)
|
||||
if err != nil {
|
||||
t.Fatalf("decode: %v", err)
|
||||
}
|
||||
if back.Count() != td.Count() {
|
||||
t.Fatalf("count mismatch: %v vs %v", back.Count(), td.Count())
|
||||
}
|
||||
for _, q := range []float64{0.1, 0.5, 0.9, 0.99} {
|
||||
if math.Abs(back.Quantile(q)-td.Quantile(q)) > 1e-9 {
|
||||
t.Fatalf("q=%v mismatch after round-trip: %v vs %v", q, back.Quantile(q), td.Quantile(q))
|
||||
}
|
||||
}
|
||||
// Empty blob -> empty digest, no error.
|
||||
if _, err := DecodeTDigest(nil); err != nil {
|
||||
t.Fatalf("decode nil: %v", err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,486 @@
|
||||
package metric
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"math"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Driver names a supported database backend.
|
||||
//
|
||||
// Driver 表示 metric store 使用的数据库后端类型。
|
||||
type Driver string
|
||||
|
||||
const (
|
||||
// DriverSQLite selects the SQLite backend.
|
||||
//
|
||||
// DriverSQLite 选择 SQLite 后端。
|
||||
DriverSQLite Driver = "sqlite"
|
||||
// DriverMySQL selects the MySQL backend.
|
||||
//
|
||||
// DriverMySQL 选择 MySQL 后端。
|
||||
DriverMySQL Driver = "mysql"
|
||||
// DriverPostgreSQL selects the PostgreSQL backend.
|
||||
//
|
||||
// DriverPostgreSQL 选择 PostgreSQL 后端。
|
||||
DriverPostgreSQL Driver = "postgresql"
|
||||
)
|
||||
|
||||
// MetricType describes the semantic type of a metric.
|
||||
//
|
||||
// MetricType 表示指标的语义类型。
|
||||
type MetricType string
|
||||
|
||||
const (
|
||||
// TypeGauge represents a point-in-time value.
|
||||
//
|
||||
// TypeGauge 表示某一时刻的数值。
|
||||
TypeGauge MetricType = "gauge"
|
||||
// TypeCounter represents a monotonically increasing counter.
|
||||
//
|
||||
// TypeCounter 表示单调递增计数器。
|
||||
TypeCounter MetricType = "counter"
|
||||
// TypeHistogram represents histogram-style measurements.
|
||||
//
|
||||
// TypeHistogram 表示直方图类度量。
|
||||
TypeHistogram MetricType = "histogram"
|
||||
// TypeSummary represents pre-summarized measurements.
|
||||
//
|
||||
// TypeSummary 表示预汇总类度量。
|
||||
TypeSummary MetricType = "summary"
|
||||
)
|
||||
|
||||
// Aggregation names a supported aggregation operation.
|
||||
//
|
||||
// Aggregation 表示聚合方式,例如 avg、p95 或 rate。
|
||||
type Aggregation string
|
||||
|
||||
const (
|
||||
// AggAvg computes the arithmetic mean.
|
||||
//
|
||||
// AggAvg 计算算术平均值。
|
||||
AggAvg Aggregation = "avg"
|
||||
// AggMin computes the minimum value.
|
||||
//
|
||||
// AggMin 计算最小值。
|
||||
AggMin Aggregation = "min"
|
||||
// AggMax computes the maximum value.
|
||||
//
|
||||
// AggMax 计算最大值。
|
||||
AggMax Aggregation = "max"
|
||||
// AggSum computes the sum of values.
|
||||
//
|
||||
// AggSum 计算值的总和。
|
||||
AggSum Aggregation = "sum"
|
||||
// AggCount counts the number of points.
|
||||
//
|
||||
// AggCount 计算点数量。
|
||||
AggCount Aggregation = "count"
|
||||
// AggP50 computes the 50th percentile.
|
||||
//
|
||||
// AggP50 计算第 50 百分位。
|
||||
AggP50 Aggregation = "p50"
|
||||
// AggP95 computes the 95th percentile.
|
||||
//
|
||||
// AggP95 计算第 95 百分位。
|
||||
AggP95 Aggregation = "p95"
|
||||
// AggP99 computes the 99th percentile.
|
||||
//
|
||||
// AggP99 计算第 99 百分位。
|
||||
AggP99 Aggregation = "p99"
|
||||
// AggFirst returns the first value in time order.
|
||||
//
|
||||
// AggFirst 返回时间顺序上的第一个值。
|
||||
AggFirst Aggregation = "first"
|
||||
// AggLast returns the last value in time order.
|
||||
//
|
||||
// AggLast 返回时间顺序上的最后一个值。
|
||||
AggLast Aggregation = "last"
|
||||
// AggRate computes the reset-aware per-second rate.
|
||||
//
|
||||
// AggRate 计算可处理重置的每秒速率。
|
||||
AggRate Aggregation = "rate"
|
||||
// AggStdDev computes the population standard deviation.
|
||||
//
|
||||
// AggStdDev 计算总体标准差。
|
||||
AggStdDev Aggregation = "stddev"
|
||||
)
|
||||
|
||||
// Order controls chronological query ordering.
|
||||
//
|
||||
// Order 表示查询结果按时间升序或降序排列。
|
||||
type Order string
|
||||
|
||||
const (
|
||||
// OrderAsc orders points from oldest to newest.
|
||||
//
|
||||
// OrderAsc 按从旧到新的顺序排列点。
|
||||
OrderAsc Order = "asc"
|
||||
// OrderDesc orders points from newest to oldest.
|
||||
//
|
||||
// OrderDesc 按从新到旧的顺序排列点。
|
||||
OrderDesc Order = "desc"
|
||||
)
|
||||
|
||||
// Definition describes a metric and its metadata.
|
||||
//
|
||||
// Definition 描述一个指标的元数据和保留策略。
|
||||
type Definition struct {
|
||||
// Name is the unique metric name.
|
||||
//
|
||||
// Name 是唯一指标名称。
|
||||
Name string `json:"name"`
|
||||
// Description is optional human-readable metric text.
|
||||
//
|
||||
// Description 是可选的人类可读指标说明。
|
||||
Description string `json:"description,omitempty"`
|
||||
// Type describes the metric's semantic type.
|
||||
//
|
||||
// Type 描述指标的语义类型。
|
||||
Type MetricType `json:"type"`
|
||||
// Unit names the value unit, such as bytes or percent.
|
||||
//
|
||||
// Unit 表示数值单位,例如 bytes 或 percent。
|
||||
Unit string `json:"unit,omitempty"`
|
||||
// RetentionDays controls historical data retention for this metric. A value
|
||||
// of zero disables persistence and removes existing metric data.
|
||||
//
|
||||
// RetentionDays 控制该指标历史数据的保留天数;零表示禁用持久化并清除已有数据。
|
||||
RetentionDays int `json:"retention_days,omitempty"`
|
||||
// Metadata stores caller-defined metric metadata.
|
||||
//
|
||||
// Metadata 保存调用方定义的指标元数据。
|
||||
Metadata map[string]string `json:"metadata,omitempty"`
|
||||
// CreatedAt records when the metric definition was created.
|
||||
//
|
||||
// CreatedAt 记录指标定义创建时间。
|
||||
CreatedAt time.Time `json:"created_at,omitempty"`
|
||||
// UpdatedAt records when the metric definition was last updated.
|
||||
//
|
||||
// UpdatedAt 记录指标定义最后更新时间。
|
||||
UpdatedAt time.Time `json:"updated_at,omitempty"`
|
||||
}
|
||||
|
||||
// withDefaults fills default values on a metric definition.
|
||||
//
|
||||
// withDefaults 为指标定义填充默认类型。
|
||||
func (d Definition) withDefaults() Definition {
|
||||
if d.Type == "" {
|
||||
d.Type = TypeGauge
|
||||
}
|
||||
return d
|
||||
}
|
||||
|
||||
// Validate checks whether the value is well formed.
|
||||
//
|
||||
// Validate 检查指标定义是否合法。
|
||||
func (d Definition) Validate() error {
|
||||
if strings.TrimSpace(d.Name) == "" {
|
||||
return fmt.Errorf("%w: metric name is required", ErrInvalidArgument)
|
||||
}
|
||||
switch d.Type {
|
||||
case "", TypeGauge, TypeCounter, TypeHistogram, TypeSummary:
|
||||
default:
|
||||
return fmt.Errorf("%w: unsupported metric type %q", ErrInvalidArgument, d.Type)
|
||||
}
|
||||
if d.RetentionDays < 0 {
|
||||
return fmt.Errorf("%w: retention days cannot be negative", ErrInvalidArgument)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Point stores one metric sample.
|
||||
//
|
||||
// Point 表示某个实体在某一时刻的一条指标采样。
|
||||
type Point struct {
|
||||
// MetricName names the metric this sample belongs to.
|
||||
//
|
||||
// MetricName 表示该采样所属的指标名称。
|
||||
MetricName string `json:"metric_name"`
|
||||
// EntityID identifies the entity that emitted the sample.
|
||||
//
|
||||
// EntityID 标识产生该采样的实体。
|
||||
EntityID string `json:"entity_id"`
|
||||
// Timestamp is the sample time.
|
||||
//
|
||||
// Timestamp 是采样时间。
|
||||
Timestamp time.Time `json:"timestamp"`
|
||||
// Value is the numeric sample value.
|
||||
//
|
||||
// Value 是采样的数值。
|
||||
Value float64 `json:"value"`
|
||||
// Tags identify the logical series within a metric and entity.
|
||||
//
|
||||
// Tags 标识同一指标和实体下的逻辑序列。
|
||||
Tags map[string]string `json:"tags,omitempty"`
|
||||
// Labels carry extra metadata that does not define series identity.
|
||||
//
|
||||
// Labels 携带不参与序列身份判定的额外元数据。
|
||||
Labels map[string]string `json:"labels,omitempty"`
|
||||
}
|
||||
|
||||
// Validate checks whether the value is well formed.
|
||||
//
|
||||
// Validate 检查采样点是否包含必要字段。
|
||||
func (p Point) Validate() error {
|
||||
if strings.TrimSpace(p.MetricName) == "" {
|
||||
return fmt.Errorf("%w: metric name is required", ErrInvalidArgument)
|
||||
}
|
||||
if strings.TrimSpace(p.EntityID) == "" {
|
||||
return fmt.Errorf("%w: entity id is required", ErrInvalidArgument)
|
||||
}
|
||||
if p.Timestamp.IsZero() {
|
||||
return fmt.Errorf("%w: timestamp is required", ErrInvalidArgument)
|
||||
}
|
||||
if math.IsNaN(p.Value) || math.IsInf(p.Value, 0) {
|
||||
return fmt.Errorf("%w: value must be finite", ErrInvalidArgument)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// normalized returns a canonical form of the value.
|
||||
//
|
||||
// normalized 将采样点时间规范化为 UTC,并补齐空标签和标签说明 map。
|
||||
func (p Point) normalized() Point {
|
||||
p.Timestamp = p.Timestamp.UTC()
|
||||
if p.Tags == nil {
|
||||
p.Tags = map[string]string{}
|
||||
}
|
||||
if p.Labels == nil {
|
||||
p.Labels = map[string]string{}
|
||||
}
|
||||
return p
|
||||
}
|
||||
|
||||
// Query loads raw metric points matching a query.
|
||||
//
|
||||
// Query 描述原始点查询条件,包括指标、实体、时间范围、标签和分页。
|
||||
type Query struct {
|
||||
// MetricName restricts the query to one metric.
|
||||
//
|
||||
// MetricName 将查询限定到单个指标。
|
||||
MetricName string `json:"metric_name"`
|
||||
// EntityID optionally restricts the query to one entity.
|
||||
//
|
||||
// EntityID 可选地将查询限定到单个实体。
|
||||
EntityID string `json:"entity_id,omitempty"`
|
||||
// Start is the inclusive query start time.
|
||||
//
|
||||
// Start 是包含边界的查询起始时间。
|
||||
Start time.Time `json:"start"`
|
||||
// End is the inclusive query end time.
|
||||
//
|
||||
// End 是包含边界的查询结束时间。
|
||||
End time.Time `json:"end"`
|
||||
// Tags filters points by exact tag key/value matches.
|
||||
//
|
||||
// Tags 按标签键值精确匹配过滤采样点。
|
||||
Tags map[string]string `json:"tags,omitempty"`
|
||||
// Limit limits the number of raw points returned.
|
||||
//
|
||||
// Limit 限制返回的原始点数量。
|
||||
Limit int `json:"limit,omitempty"`
|
||||
// Offset skips this many raw points before returning results.
|
||||
//
|
||||
// Offset 在返回结果前跳过指定数量的原始点。
|
||||
Offset int `json:"offset,omitempty"`
|
||||
// Order controls chronological result ordering.
|
||||
//
|
||||
// Order 控制结果的时间顺序。
|
||||
Order Order `json:"order,omitempty"`
|
||||
}
|
||||
|
||||
// Validate checks whether the value is well formed.
|
||||
//
|
||||
// Validate 检查原始查询条件是否合法。
|
||||
func (q Query) Validate() error {
|
||||
if strings.TrimSpace(q.MetricName) == "" {
|
||||
return fmt.Errorf("%w: metric name is required", ErrInvalidArgument)
|
||||
}
|
||||
if q.Start.IsZero() || q.End.IsZero() {
|
||||
return fmt.Errorf("%w: start and end time are required", ErrInvalidArgument)
|
||||
}
|
||||
if q.End.Before(q.Start) {
|
||||
return fmt.Errorf("%w: end time cannot be before start time", ErrInvalidArgument)
|
||||
}
|
||||
if q.Limit < 0 || q.Offset < 0 {
|
||||
return fmt.Errorf("%w: limit and offset cannot be negative", ErrInvalidArgument)
|
||||
}
|
||||
switch q.Order {
|
||||
case "", OrderAsc, OrderDesc:
|
||||
default:
|
||||
return fmt.Errorf("%w: unsupported order %q", ErrInvalidArgument, q.Order)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// normalized returns a canonical form of the value.
|
||||
//
|
||||
// normalized 将查询时间规范化为 UTC,并设置默认排序。
|
||||
func (q Query) normalized() Query {
|
||||
q.Start = q.Start.UTC()
|
||||
q.End = q.End.UTC()
|
||||
if q.Order == "" {
|
||||
q.Order = OrderAsc
|
||||
}
|
||||
return q
|
||||
}
|
||||
|
||||
// AggregateQuery describes a bucketed aggregate query.
|
||||
//
|
||||
// AggregateQuery 描述按固定时间间隔分桶后的聚合查询。
|
||||
type AggregateQuery struct {
|
||||
// Query supplies the raw series filter and time window.
|
||||
//
|
||||
// Query 提供原始序列过滤条件和时间窗口。
|
||||
Query
|
||||
// Aggregation selects the bucket aggregation to compute.
|
||||
//
|
||||
// Aggregation 选择要计算的桶聚合方式。
|
||||
Aggregation Aggregation `json:"aggregation"`
|
||||
// Interval is the width of each aggregate bucket.
|
||||
//
|
||||
// Interval 是每个聚合桶的宽度。
|
||||
Interval time.Duration `json:"interval"`
|
||||
// PreserveSeries keeps entity/tag identities as separate aggregate series on
|
||||
// rollup-backed reads. The default preserves the historical rollup behavior
|
||||
// of merging all matched series into each output bucket.
|
||||
//
|
||||
// PreserveSeries 在基于 rollup 的读取中保留 entity/tag 维度为独立聚合序列。
|
||||
// 默认值保留历史 rollup 行为:把匹配到的序列合并进同一个输出桶。
|
||||
PreserveSeries bool `json:"preserve_series,omitempty"`
|
||||
// BucketLimit and BucketOffset page over the produced aggregate buckets, not
|
||||
// the underlying raw points. They are applied consistently across every
|
||||
// backend and aggregation type. The embedded Query.Limit/Query.Offset are
|
||||
// ignored for aggregation (they describe raw-point paging, which would mean
|
||||
// something different depending on whether the aggregation is pushed down to
|
||||
// SQL or computed in memory).
|
||||
//
|
||||
// BucketLimit 和 BucketOffset 对生成的聚合桶分页,而不是对底层原始点分页。
|
||||
// 它们会在所有后端和聚合类型上保持一致。嵌入的 Query.Limit/Query.Offset
|
||||
// 在聚合中会被忽略,因为它们描述的是原始点分页;根据聚合是下推到 SQL 还是在
|
||||
// 内存计算,这会产生不同语义。
|
||||
BucketLimit int `json:"bucket_limit,omitempty"`
|
||||
// BucketOffset skips this many aggregate buckets before returning results.
|
||||
//
|
||||
// BucketOffset 在返回结果前跳过指定数量的聚合桶。
|
||||
BucketOffset int `json:"bucket_offset,omitempty"`
|
||||
}
|
||||
|
||||
// Validate checks whether the value is well formed.
|
||||
//
|
||||
// Validate 检查聚合查询的时间间隔、分页和聚合类型是否合法。
|
||||
func (q AggregateQuery) Validate() error {
|
||||
if err := q.Query.Validate(); err != nil {
|
||||
return err
|
||||
}
|
||||
if q.Interval <= 0 {
|
||||
return fmt.Errorf("%w: aggregate interval must be positive", ErrInvalidArgument)
|
||||
}
|
||||
if q.BucketLimit < 0 || q.BucketOffset < 0 {
|
||||
return fmt.Errorf("%w: bucket limit and offset cannot be negative", ErrInvalidArgument)
|
||||
}
|
||||
switch q.Aggregation {
|
||||
case AggAvg, AggMin, AggMax, AggSum, AggCount, AggFirst, AggLast, AggRate, AggStdDev:
|
||||
default:
|
||||
// Any percentile (p50, p95, p99, p99.9, ...) is also valid. The fixed
|
||||
// AggP50/AggP95/AggP99 constants fall through to here as well.
|
||||
if !isPercentile(q.Aggregation) {
|
||||
return fmt.Errorf("%w: unsupported aggregation %q", ErrInvalidArgument, q.Aggregation)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// AggregatePoint stores one aggregate bucket result.
|
||||
//
|
||||
// AggregatePoint 表示一个聚合桶的结果。
|
||||
type AggregatePoint struct {
|
||||
// MetricName is the metric represented by the bucket.
|
||||
//
|
||||
// MetricName 是该桶代表的指标名称。
|
||||
MetricName string `json:"metric_name"`
|
||||
// EntityID is the entity represented by the bucket when one was requested.
|
||||
//
|
||||
// EntityID 是请求限定实体时该桶代表的实体。
|
||||
EntityID string `json:"entity_id,omitempty"`
|
||||
// Bucket is the bucket start time.
|
||||
//
|
||||
// Bucket 是桶起始时间。
|
||||
Bucket time.Time `json:"bucket"`
|
||||
// Value is the computed aggregate value.
|
||||
//
|
||||
// Value 是计算出的聚合值。
|
||||
Value float64 `json:"value"`
|
||||
// Count is the number of points represented by the bucket.
|
||||
//
|
||||
// Count 是该桶代表的点数量。
|
||||
Count int `json:"count"`
|
||||
// Tags identify the logical series represented by the bucket.
|
||||
//
|
||||
// Tags 标识该聚合桶所属的逻辑序列。
|
||||
Tags map[string]string `json:"tags,omitempty"`
|
||||
}
|
||||
|
||||
// Stats stores or computes summary statistics for a point series.
|
||||
//
|
||||
// Stats 表示一段原始点序列的统计摘要。
|
||||
type Stats struct {
|
||||
// Count is the number of points in the series.
|
||||
//
|
||||
// Count 是序列中的点数量。
|
||||
Count int `json:"count"`
|
||||
// Min is the minimum value.
|
||||
//
|
||||
// Min 是最小值。
|
||||
Min float64 `json:"min"`
|
||||
// Max is the maximum value.
|
||||
//
|
||||
// Max 是最大值。
|
||||
Max float64 `json:"max"`
|
||||
// Avg is the arithmetic mean.
|
||||
//
|
||||
// Avg 是算术平均值。
|
||||
Avg float64 `json:"avg"`
|
||||
// Sum is the sum of all values.
|
||||
//
|
||||
// Sum 是所有值的总和。
|
||||
Sum float64 `json:"sum"`
|
||||
// P50 is the 50th percentile.
|
||||
//
|
||||
// P50 是第 50 百分位。
|
||||
P50 float64 `json:"p50"`
|
||||
// P95 is the 95th percentile.
|
||||
//
|
||||
// P95 是第 95 百分位。
|
||||
P95 float64 `json:"p95"`
|
||||
// P99 is the 99th percentile.
|
||||
//
|
||||
// P99 是第 99 百分位。
|
||||
P99 float64 `json:"p99"`
|
||||
// First is the first value in time order.
|
||||
//
|
||||
// First 是时间顺序上的第一个值。
|
||||
First float64 `json:"first"`
|
||||
// Last is the last value in time order.
|
||||
//
|
||||
// Last 是时间顺序上的最后一个值。
|
||||
Last float64 `json:"last"`
|
||||
// Rate is the reset-aware per-second rate.
|
||||
//
|
||||
// Rate 是可处理重置的每秒速率。
|
||||
Rate float64 `json:"rate"`
|
||||
// Start is the first point timestamp.
|
||||
//
|
||||
// Start 是第一个点的时间戳。
|
||||
Start time.Time `json:"start"`
|
||||
// End is the last point timestamp.
|
||||
//
|
||||
// End 是最后一个点的时间戳。
|
||||
End time.Time `json:"end"`
|
||||
// StdDev is the population standard deviation.
|
||||
//
|
||||
// StdDev 是总体标准差。
|
||||
StdDev float64 `json:"std_dev"`
|
||||
}
|
||||
+21
-21
@@ -4,27 +4,27 @@ import "time"
|
||||
|
||||
// ClientInfo is the legacy table shape for migrating pre-Client model data.
|
||||
type ClientInfo struct {
|
||||
UUID string `json:"uuid,omitempty" gorm:"type:varchar(36);primaryKey;foreignKey:ClientUUID;references:UUID;constraint:OnDelete:CASCADE"`
|
||||
Name string `json:"name" gorm:"type:varchar(100);not null"`
|
||||
CpuName string `json:"cpu_name" gorm:"type:varchar(100)"`
|
||||
Virtualization string `json:"virtualization" gorm:"type:varchar(50)"`
|
||||
Arch string `json:"arch" gorm:"type:varchar(50)"`
|
||||
CpuCores int `json:"cpu_cores" gorm:"type:int"`
|
||||
OS string `json:"os" gorm:"type:varchar(100)"`
|
||||
GpuName string `json:"gpu_name" gorm:"type:varchar(100)"`
|
||||
IPv4 string `json:"ipv4,omitempty" gorm:"type:varchar(100)"`
|
||||
IPv6 string `json:"ipv6,omitempty" gorm:"type:varchar(100)"`
|
||||
Region string `json:"region" gorm:"type:varchar(100)"`
|
||||
Remark string `json:"remark,omitempty" gorm:"type:longtext"`
|
||||
PublicRemark string `json:"public_remark,omitempty" gorm:"type:longtext"`
|
||||
MemTotal int64 `json:"mem_total" gorm:"type:bigint"`
|
||||
SwapTotal int64 `json:"swap_total" gorm:"type:bigint"`
|
||||
DiskTotal int64 `json:"disk_total" gorm:"type:bigint"`
|
||||
Version string `json:"version,omitempty" gorm:"type:varchar(100)"`
|
||||
Weight int `json:"weight" gorm:"type:int"`
|
||||
Price float64 `json:"price"`
|
||||
BillingCycle int `json:"billing_cycle"`
|
||||
ExpiredAt time.Time `json:"expired_at" gorm:"type:timestamp"`
|
||||
UUID string `json:"uuid,omitempty" gorm:"type:varchar(36);primaryKey;foreignKey:ClientUUID;references:UUID;constraint:OnDelete:CASCADE"`
|
||||
Name string `json:"name" gorm:"type:varchar(100);not null"`
|
||||
CpuName string `json:"cpu_name" gorm:"type:varchar(100)"`
|
||||
Virtualization string `json:"virtualization" gorm:"type:varchar(50)"`
|
||||
Arch string `json:"arch" gorm:"type:varchar(50)"`
|
||||
CpuCores int `json:"cpu_cores" gorm:"type:int"`
|
||||
OS string `json:"os" gorm:"type:varchar(100)"`
|
||||
GpuName string `json:"gpu_name" gorm:"type:varchar(100)"`
|
||||
IPv4 string `json:"ipv4,omitempty" gorm:"type:varchar(100)"`
|
||||
IPv6 string `json:"ipv6,omitempty" gorm:"type:varchar(100)"`
|
||||
Region string `json:"region" gorm:"type:varchar(100)"`
|
||||
Remark string `json:"remark,omitempty" gorm:"type:longtext"`
|
||||
PublicRemark string `json:"public_remark,omitempty" gorm:"type:longtext"`
|
||||
MemTotal int64 `json:"mem_total" gorm:"type:bigint"`
|
||||
SwapTotal int64 `json:"swap_total" gorm:"type:bigint"`
|
||||
DiskTotal int64 `json:"disk_total" gorm:"type:bigint"`
|
||||
Version string `json:"version,omitempty" gorm:"type:varchar(100)"`
|
||||
Weight int `json:"weight" gorm:"type:int"`
|
||||
Price float64 `json:"price"`
|
||||
BillingCycle int `json:"billing_cycle"`
|
||||
ExpiredAt *time.Time `json:"expired_at" gorm:"type:timestamp"`
|
||||
CreatedAt time.Time
|
||||
UpdatedAt time.Time
|
||||
}
|
||||
@@ -0,0 +1,562 @@
|
||||
package migrations
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"log"
|
||||
"math"
|
||||
"time"
|
||||
|
||||
"github.com/komari-monitor/komari/database/metricstore"
|
||||
"github.com/komari-monitor/komari/database/models"
|
||||
appconfig "github.com/komari-monitor/komari/pkg/config"
|
||||
"github.com/komari-monitor/komari/pkg/metric"
|
||||
"gorm.io/gorm"
|
||||
)
|
||||
|
||||
const (
|
||||
legacyMonitoringMigrationDoneKey = "migration_legacy_monitoring_to_metric_store_done"
|
||||
legacyMonitoringBatchSize = 500
|
||||
)
|
||||
|
||||
var legacyMonitoringTables = []string{"records", "records_long_term", "gpu_records", "ping_records"}
|
||||
|
||||
type MetricContext struct {
|
||||
DB *gorm.DB
|
||||
Store *metric.Store
|
||||
}
|
||||
|
||||
type LegacyMonitoringStats struct {
|
||||
Records int64
|
||||
GPU int64
|
||||
Ping int64
|
||||
}
|
||||
|
||||
// LegacyMonitoringSummary describes the legacy source data shown by the
|
||||
// pre-start upgrade wizard. Row counts deliberately stay separate from the
|
||||
// estimated metric point count because one legacy row expands into multiple
|
||||
// metric points.
|
||||
type LegacyMonitoringSummary struct {
|
||||
LoadRows int64 `json:"load_rows"`
|
||||
GPURows int64 `json:"gpu_rows"`
|
||||
LatencyRows int64 `json:"latency_rows"`
|
||||
MonitoringRows int64 `json:"monitoring_rows"`
|
||||
EstimatedPoints int64 `json:"estimated_points"`
|
||||
ServerCount int64 `json:"server_count"`
|
||||
RetentionDays int `json:"retention_days"`
|
||||
OldestAt *time.Time `json:"oldest_at,omitempty"`
|
||||
NewestAt *time.Time `json:"newest_at,omitempty"`
|
||||
}
|
||||
|
||||
type LegacyMonitoringProgress struct {
|
||||
Phase string `json:"phase"`
|
||||
Table string `json:"table,omitempty"`
|
||||
SourceRowsDone int64 `json:"source_rows_done"`
|
||||
SourceRowsTotal int64 `json:"source_rows_total"`
|
||||
WrittenPoints int64 `json:"written_points"`
|
||||
}
|
||||
|
||||
type LegacyMonitoringDeleted struct {
|
||||
LoadRows int64 `json:"load_rows"`
|
||||
GPURows int64 `json:"gpu_rows"`
|
||||
LatencyRows int64 `json:"latency_rows"`
|
||||
}
|
||||
|
||||
type legacyMonitoringBounds struct {
|
||||
Oldest string `gorm:"column:oldest"`
|
||||
Newest string `gorm:"column:newest"`
|
||||
}
|
||||
|
||||
type legacyBatchProgress func(table string, rows, points int64)
|
||||
|
||||
type legacyRecordRow struct {
|
||||
RowID int64 `gorm:"column:row_id"`
|
||||
models.Record
|
||||
}
|
||||
|
||||
type legacyGPURecordRow struct {
|
||||
RowID int64 `gorm:"column:row_id"`
|
||||
models.GPURecord
|
||||
}
|
||||
|
||||
type legacyPingRecordRow struct {
|
||||
RowID int64 `gorm:"column:row_id"`
|
||||
models.PingRecord
|
||||
}
|
||||
|
||||
// RunMetricStoreMigrations executes one-shot migrations that need the metric store
|
||||
// to be initialized. These cannot run inside Run because Run executes before the
|
||||
// metric store is opened.
|
||||
func RunMetricStoreMigrations(ctx MetricContext) error {
|
||||
done, err := appconfig.GetAs[bool](legacyMonitoringMigrationDoneKey, false)
|
||||
if err != nil {
|
||||
return fmt.Errorf("read legacy monitoring migration marker: %w", err)
|
||||
}
|
||||
|
||||
stats, err := runLegacyMonitoringMigration(context.Background(), ctx.DB, ctx.Store, done, func() error {
|
||||
return appconfig.Set(legacyMonitoringMigrationDoneKey, true)
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if stats.Records > 0 || stats.GPU > 0 || stats.Ping > 0 {
|
||||
log.Printf("Legacy monitoring table migration completed (records=%d, gpu=%d, ping=%d)", stats.Records, stats.GPU, stats.Ping)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// LegacyMonitoringMigrationRequired reports whether startup must enter the
|
||||
// restricted 1.2.7 upgrade flow. Empty legacy tables are handled by the normal
|
||||
// one-shot migration so fresh installations never see the wizard.
|
||||
func LegacyMonitoringMigrationRequired(db *gorm.DB) (bool, LegacyMonitoringSummary, error) {
|
||||
done, err := appconfig.GetAs[bool](legacyMonitoringMigrationDoneKey, false)
|
||||
if err != nil {
|
||||
return false, LegacyMonitoringSummary{}, fmt.Errorf("read legacy monitoring migration marker: %w", err)
|
||||
}
|
||||
summary, err := InspectLegacyMonitoring(db)
|
||||
if err != nil {
|
||||
return false, LegacyMonitoringSummary{}, err
|
||||
}
|
||||
return !done && summary.MonitoringRows > 0, summary, nil
|
||||
}
|
||||
|
||||
func InspectLegacyMonitoring(db *gorm.DB) (LegacyMonitoringSummary, error) {
|
||||
var summary LegacyMonitoringSummary
|
||||
if db == nil {
|
||||
return summary, fmt.Errorf("migration database is nil")
|
||||
}
|
||||
if db.Migrator().HasTable(&models.Client{}) {
|
||||
if err := db.Model(&models.Client{}).Count(&summary.ServerCount).Error; err != nil {
|
||||
return summary, fmt.Errorf("count monitored servers: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
counts := map[string]*int64{
|
||||
"records": &summary.LoadRows,
|
||||
"records_long_term": &summary.LoadRows,
|
||||
"gpu_records": &summary.GPURows,
|
||||
"ping_records": &summary.LatencyRows,
|
||||
}
|
||||
var oldest, newest time.Time
|
||||
for _, table := range legacyMonitoringTables {
|
||||
if !db.Migrator().HasTable(table) {
|
||||
continue
|
||||
}
|
||||
var count int64
|
||||
if err := db.Table(table).Count(&count).Error; err != nil {
|
||||
return summary, fmt.Errorf("count legacy %s: %w", table, err)
|
||||
}
|
||||
*counts[table] += count
|
||||
if count == 0 {
|
||||
continue
|
||||
}
|
||||
var bounds legacyMonitoringBounds
|
||||
if err := db.Table(table).Select("MIN(time) AS oldest, MAX(time) AS newest").Scan(&bounds).Error; err != nil {
|
||||
return summary, fmt.Errorf("inspect legacy %s time range: %w", table, err)
|
||||
}
|
||||
tableOldest, err := parseLegacyTimestamp(bounds.Oldest, time.UTC)
|
||||
if err != nil {
|
||||
return summary, fmt.Errorf("parse legacy %s oldest time: %w", table, err)
|
||||
}
|
||||
tableNewest, err := parseLegacyTimestamp(bounds.Newest, time.UTC)
|
||||
if err != nil {
|
||||
return summary, fmt.Errorf("parse legacy %s newest time: %w", table, err)
|
||||
}
|
||||
if oldest.IsZero() || (!tableOldest.IsZero() && tableOldest.Before(oldest)) {
|
||||
oldest = tableOldest
|
||||
}
|
||||
if newest.IsZero() || tableNewest.After(newest) {
|
||||
newest = tableNewest
|
||||
}
|
||||
}
|
||||
|
||||
summary.MonitoringRows = summary.LoadRows + summary.GPURows + summary.LatencyRows
|
||||
summary.EstimatedPoints = summary.LoadRows*19 + summary.GPURows*4 + summary.LatencyRows*2
|
||||
if !oldest.IsZero() {
|
||||
oldestCopy := oldest
|
||||
summary.OldestAt = &oldestCopy
|
||||
}
|
||||
if !newest.IsZero() {
|
||||
newestCopy := newest
|
||||
summary.NewestAt = &newestCopy
|
||||
}
|
||||
if !oldest.IsZero() && !newest.IsZero() {
|
||||
retentionEnd := time.Now().UTC()
|
||||
if newest.After(retentionEnd) {
|
||||
retentionEnd = newest
|
||||
}
|
||||
summary.RetentionDays = int(math.Ceil(retentionEnd.Sub(oldest).Hours() / 24))
|
||||
if summary.RetentionDays == 0 && summary.MonitoringRows > 0 {
|
||||
summary.RetentionDays = 1
|
||||
}
|
||||
}
|
||||
return summary, nil
|
||||
}
|
||||
|
||||
// DeleteLegacyMonitoringBefore removes only monitoring history older than the
|
||||
// cutoff. Clients, users, ping tasks and other business data are untouched.
|
||||
func DeleteLegacyMonitoringBefore(db *gorm.DB, cutoff time.Time) (LegacyMonitoringDeleted, error) {
|
||||
var deleted LegacyMonitoringDeleted
|
||||
if db == nil {
|
||||
return deleted, fmt.Errorf("migration database is nil")
|
||||
}
|
||||
if cutoff.IsZero() {
|
||||
return deleted, fmt.Errorf("cleanup cutoff is required")
|
||||
}
|
||||
|
||||
err := db.Transaction(func(tx *gorm.DB) error {
|
||||
for _, table := range legacyMonitoringTables {
|
||||
if !tx.Migrator().HasTable(table) {
|
||||
continue
|
||||
}
|
||||
result := tx.Exec("DELETE FROM "+table+" WHERE time < ?", cutoff.UTC())
|
||||
if result.Error != nil {
|
||||
return fmt.Errorf("delete legacy %s before cutoff: %w", table, result.Error)
|
||||
}
|
||||
switch table {
|
||||
case "records", "records_long_term":
|
||||
deleted.LoadRows += result.RowsAffected
|
||||
case "gpu_records":
|
||||
deleted.GPURows += result.RowsAffected
|
||||
case "ping_records":
|
||||
deleted.LatencyRows += result.RowsAffected
|
||||
}
|
||||
}
|
||||
return nil
|
||||
})
|
||||
return deleted, err
|
||||
}
|
||||
|
||||
func MigrateLegacyMonitoring(ctx context.Context, db *gorm.DB, s *metric.Store, progress func(LegacyMonitoringProgress)) (LegacyMonitoringStats, error) {
|
||||
return migrateLegacyMonitoringTables(ctx, db, s, progress)
|
||||
}
|
||||
|
||||
// CompleteLegacyMonitoringMigration removes the legacy tables, runs any
|
||||
// caller-owned database maintenance, and only then persists the completion
|
||||
// marker. Keeping the marker last prevents a failed finalization from being
|
||||
// reported as a completed migration.
|
||||
func CompleteLegacyMonitoringMigration(db *gorm.DB, finalize func() error) error {
|
||||
if err := dropLegacyMonitoringTables(db); err != nil {
|
||||
return err
|
||||
}
|
||||
if finalize != nil {
|
||||
if err := finalize(); err != nil {
|
||||
return fmt.Errorf("finalize legacy monitoring migration: %w", err)
|
||||
}
|
||||
}
|
||||
if err := appconfig.Set(legacyMonitoringMigrationDoneKey, true); err != nil {
|
||||
return fmt.Errorf("mark legacy monitoring migration done: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func runLegacyMonitoringMigration(ctx context.Context, db *gorm.DB, s *metric.Store, done bool, markDone func() error) (LegacyMonitoringStats, error) {
|
||||
var stats LegacyMonitoringStats
|
||||
if done {
|
||||
return stats, nil
|
||||
}
|
||||
if db == nil {
|
||||
return stats, fmt.Errorf("migration database is nil")
|
||||
}
|
||||
if s == nil {
|
||||
return stats, fmt.Errorf("metric store is nil")
|
||||
}
|
||||
if markDone == nil {
|
||||
return stats, fmt.Errorf("migration marker writer is nil")
|
||||
}
|
||||
|
||||
stats, err := migrateLegacyMonitoringTables(ctx, db, s, nil)
|
||||
if err != nil {
|
||||
return stats, err
|
||||
}
|
||||
if err := dropLegacyMonitoringTables(db); err != nil {
|
||||
return stats, err
|
||||
}
|
||||
if err := markDone(); err != nil {
|
||||
return stats, fmt.Errorf("mark legacy monitoring migration done: %w", err)
|
||||
}
|
||||
return stats, nil
|
||||
}
|
||||
|
||||
func migrateLegacyMonitoringTables(ctx context.Context, db *gorm.DB, s *metric.Store, progress func(LegacyMonitoringProgress)) (LegacyMonitoringStats, error) {
|
||||
var stats LegacyMonitoringStats
|
||||
summary, err := InspectLegacyMonitoring(db)
|
||||
if err != nil {
|
||||
return stats, err
|
||||
}
|
||||
var rowsDone, writtenPoints int64
|
||||
emit := func(table string, rows, points int64) {
|
||||
rowsDone += rows
|
||||
writtenPoints += points
|
||||
if progress != nil {
|
||||
progress(LegacyMonitoringProgress{
|
||||
Phase: "migrating",
|
||||
Table: table,
|
||||
SourceRowsDone: rowsDone,
|
||||
SourceRowsTotal: summary.MonitoringRows,
|
||||
WrittenPoints: writtenPoints,
|
||||
})
|
||||
}
|
||||
}
|
||||
if progress != nil {
|
||||
emit("", 0, 0)
|
||||
}
|
||||
for _, table := range []string{"records", "records_long_term"} {
|
||||
n, err := migrateLegacyRecordTable(ctx, s, db, table, emit)
|
||||
if err != nil {
|
||||
return stats, err
|
||||
}
|
||||
stats.Records += n
|
||||
}
|
||||
|
||||
n, err := migrateLegacyGPURecordTable(ctx, s, db, "gpu_records", emit)
|
||||
if err != nil {
|
||||
return stats, err
|
||||
}
|
||||
stats.GPU += n
|
||||
|
||||
n, err = migrateLegacyPingRecordTable(ctx, s, db, "ping_records", emit)
|
||||
if err != nil {
|
||||
return stats, err
|
||||
}
|
||||
stats.Ping += n
|
||||
|
||||
return stats, nil
|
||||
}
|
||||
|
||||
func migrateLegacyRecordTable(ctx context.Context, s *metric.Store, db *gorm.DB, table string, progress legacyBatchProgress) (int64, error) {
|
||||
if !db.Migrator().HasTable(table) {
|
||||
return 0, nil
|
||||
}
|
||||
|
||||
var total int64
|
||||
if err := db.Table(table).Count(&total).Error; err != nil {
|
||||
return 0, fmt.Errorf("count legacy %s: %w", table, err)
|
||||
}
|
||||
if total == 0 {
|
||||
return 0, nil
|
||||
}
|
||||
|
||||
log.Printf("[legacy-migration] importing %d rows from %s", total, table)
|
||||
var migrated int64
|
||||
var lastRowID int64
|
||||
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return migrated, ctx.Err()
|
||||
default:
|
||||
}
|
||||
|
||||
var rows []legacyRecordRow
|
||||
q := db.Table(table).Select("rowid AS row_id, *").Where("rowid > ?", lastRowID).Order("rowid ASC").Limit(legacyMonitoringBatchSize)
|
||||
if err := q.Find(&rows).Error; err != nil {
|
||||
return migrated, fmt.Errorf("read legacy %s: %w", table, err)
|
||||
}
|
||||
if len(rows) == 0 {
|
||||
break
|
||||
}
|
||||
|
||||
points := make([]metric.Point, 0, len(rows)*19)
|
||||
for i := range rows {
|
||||
points = append(points, recordToPoints(rows[i].Record)...)
|
||||
}
|
||||
if err := s.WriteBatch(ctx, points); err != nil {
|
||||
return migrated, fmt.Errorf("write legacy %s to metric store: %w", table, err)
|
||||
}
|
||||
|
||||
last := rows[len(rows)-1]
|
||||
lastRowID = last.RowID
|
||||
migrated += int64(len(rows))
|
||||
if progress != nil {
|
||||
progress(table, int64(len(rows)), int64(len(points)))
|
||||
}
|
||||
}
|
||||
|
||||
log.Printf("[legacy-migration] imported %d rows from %s", migrated, table)
|
||||
return migrated, nil
|
||||
}
|
||||
|
||||
func migrateLegacyGPURecordTable(ctx context.Context, s *metric.Store, db *gorm.DB, table string, progress legacyBatchProgress) (int64, error) {
|
||||
if !db.Migrator().HasTable(table) {
|
||||
return 0, nil
|
||||
}
|
||||
|
||||
var total int64
|
||||
if err := db.Table(table).Count(&total).Error; err != nil {
|
||||
return 0, fmt.Errorf("count legacy %s: %w", table, err)
|
||||
}
|
||||
if total == 0 {
|
||||
return 0, nil
|
||||
}
|
||||
|
||||
log.Printf("[legacy-migration] importing %d rows from %s", total, table)
|
||||
var migrated int64
|
||||
var lastRowID int64
|
||||
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return migrated, ctx.Err()
|
||||
default:
|
||||
}
|
||||
|
||||
var rows []legacyGPURecordRow
|
||||
q := db.Table(table).Select("rowid AS row_id, *").Where("rowid > ?", lastRowID).Order("rowid ASC").Limit(legacyMonitoringBatchSize)
|
||||
if err := q.Find(&rows).Error; err != nil {
|
||||
return migrated, fmt.Errorf("read legacy %s: %w", table, err)
|
||||
}
|
||||
if len(rows) == 0 {
|
||||
break
|
||||
}
|
||||
|
||||
points := make([]metric.Point, 0, len(rows)*4)
|
||||
for i := range rows {
|
||||
points = append(points, gpuRecordToPoints(rows[i].GPURecord)...)
|
||||
}
|
||||
if err := s.WriteBatch(ctx, points); err != nil {
|
||||
return migrated, fmt.Errorf("write legacy %s to metric store: %w", table, err)
|
||||
}
|
||||
|
||||
last := rows[len(rows)-1]
|
||||
lastRowID = last.RowID
|
||||
migrated += int64(len(rows))
|
||||
if progress != nil {
|
||||
progress(table, int64(len(rows)), int64(len(points)))
|
||||
}
|
||||
}
|
||||
|
||||
log.Printf("[legacy-migration] imported %d rows from %s", migrated, table)
|
||||
return migrated, nil
|
||||
}
|
||||
|
||||
func migrateLegacyPingRecordTable(ctx context.Context, s *metric.Store, db *gorm.DB, table string, progress legacyBatchProgress) (int64, error) {
|
||||
if !db.Migrator().HasTable(table) {
|
||||
return 0, nil
|
||||
}
|
||||
|
||||
var total int64
|
||||
if err := db.Table(table).Count(&total).Error; err != nil {
|
||||
return 0, fmt.Errorf("count legacy %s: %w", table, err)
|
||||
}
|
||||
if total == 0 {
|
||||
return 0, nil
|
||||
}
|
||||
|
||||
log.Printf("[legacy-migration] importing %d rows from %s", total, table)
|
||||
var migrated int64
|
||||
var lastRowID int64
|
||||
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return migrated, ctx.Err()
|
||||
default:
|
||||
}
|
||||
|
||||
var rows []legacyPingRecordRow
|
||||
q := db.Table(table).Select("rowid AS row_id, *").Where("rowid > ?", lastRowID).Order("rowid ASC").Limit(legacyMonitoringBatchSize)
|
||||
if err := q.Find(&rows).Error; err != nil {
|
||||
return migrated, fmt.Errorf("read legacy %s: %w", table, err)
|
||||
}
|
||||
if len(rows) == 0 {
|
||||
break
|
||||
}
|
||||
|
||||
points := make([]metric.Point, 0, len(rows)*2)
|
||||
for i := range rows {
|
||||
points = append(points, pingRecordToPoints(rows[i].PingRecord)...)
|
||||
}
|
||||
if err := s.WriteBatch(ctx, points); err != nil {
|
||||
return migrated, fmt.Errorf("write legacy %s to metric store: %w", table, err)
|
||||
}
|
||||
|
||||
last := rows[len(rows)-1]
|
||||
lastRowID = last.RowID
|
||||
migrated += int64(len(rows))
|
||||
if progress != nil {
|
||||
progress(table, int64(len(rows)), int64(len(points)))
|
||||
}
|
||||
}
|
||||
|
||||
log.Printf("[legacy-migration] imported %d rows from %s", migrated, table)
|
||||
return migrated, nil
|
||||
}
|
||||
|
||||
func recordToPoints(rec models.Record) []metric.Point {
|
||||
ts := rec.Time
|
||||
entityID := rec.Client
|
||||
return []metric.Point{
|
||||
{MetricName: metricstore.MetricCPU, EntityID: entityID, Timestamp: ts, Value: float64(rec.Cpu)},
|
||||
{MetricName: metricstore.MetricGPU, EntityID: entityID, Timestamp: ts, Value: float64(rec.Gpu)},
|
||||
{MetricName: metricstore.MetricRAM, EntityID: entityID, Timestamp: ts, Value: float64(rec.Ram)},
|
||||
{MetricName: metricstore.MetricRAMTotal, EntityID: entityID, Timestamp: ts, Value: float64(rec.RamTotal)},
|
||||
{MetricName: metricstore.MetricSwap, EntityID: entityID, Timestamp: ts, Value: float64(rec.Swap)},
|
||||
{MetricName: metricstore.MetricSwapTotal, EntityID: entityID, Timestamp: ts, Value: float64(rec.SwapTotal)},
|
||||
{MetricName: metricstore.MetricLoad, EntityID: entityID, Timestamp: ts, Value: float64(rec.Load)},
|
||||
{MetricName: metricstore.MetricTemp, EntityID: entityID, Timestamp: ts, Value: float64(rec.Temp)},
|
||||
{MetricName: metricstore.MetricDisk, EntityID: entityID, Timestamp: ts, Value: float64(rec.Disk)},
|
||||
{MetricName: metricstore.MetricDiskTotal, EntityID: entityID, Timestamp: ts, Value: float64(rec.DiskTotal)},
|
||||
{MetricName: metricstore.MetricNetIn, EntityID: entityID, Timestamp: ts, Value: float64(rec.NetIn)},
|
||||
{MetricName: metricstore.MetricNetOut, EntityID: entityID, Timestamp: ts, Value: float64(rec.NetOut)},
|
||||
{MetricName: metricstore.MetricNetTotalUp, EntityID: entityID, Timestamp: ts, Value: float64(rec.NetTotalUp)},
|
||||
{MetricName: metricstore.MetricNetTotalDown, EntityID: entityID, Timestamp: ts, Value: float64(rec.NetTotalDown)},
|
||||
{MetricName: metricstore.MetricTrafficUp, EntityID: entityID, Timestamp: ts, Value: float64(rec.TrafficUp)},
|
||||
{MetricName: metricstore.MetricTrafficDown, EntityID: entityID, Timestamp: ts, Value: float64(rec.TrafficDown)},
|
||||
{MetricName: metricstore.MetricProcess, EntityID: entityID, Timestamp: ts, Value: float64(rec.Process)},
|
||||
{MetricName: metricstore.MetricConnections, EntityID: entityID, Timestamp: ts, Value: float64(rec.Connections)},
|
||||
{MetricName: metricstore.MetricConnectionsUDP, EntityID: entityID, Timestamp: ts, Value: float64(rec.ConnectionsUdp)},
|
||||
}
|
||||
}
|
||||
|
||||
func gpuRecordToPoints(rec models.GPURecord) []metric.Point {
|
||||
ts := rec.Time
|
||||
tags := map[string]string{
|
||||
"device_index": fmt.Sprintf("%d", rec.DeviceIndex),
|
||||
"device_name": rec.DeviceName,
|
||||
}
|
||||
return []metric.Point{
|
||||
{MetricName: metricstore.MetricGPUMem, EntityID: rec.Client, Timestamp: ts, Value: float64(rec.MemUsed), Tags: tags},
|
||||
{MetricName: metricstore.MetricGPUMemTotal, EntityID: rec.Client, Timestamp: ts, Value: float64(rec.MemTotal), Tags: tags},
|
||||
{MetricName: metricstore.MetricGPUDeviceUsage, EntityID: rec.Client, Timestamp: ts, Value: float64(rec.Utilization), Tags: tags},
|
||||
{MetricName: metricstore.MetricGPUTemp, EntityID: rec.Client, Timestamp: ts, Value: float64(rec.Temperature), Tags: tags},
|
||||
}
|
||||
}
|
||||
|
||||
func pingRecordToPoints(rec models.PingRecord) []metric.Point {
|
||||
ts := rec.Time
|
||||
tags := map[string]string{"task_id": fmt.Sprintf("%d", rec.TaskId)}
|
||||
loss := 0.0
|
||||
if rec.Value < 0 {
|
||||
loss = 1
|
||||
}
|
||||
return []metric.Point{
|
||||
{
|
||||
MetricName: metricstore.MetricPingLatency,
|
||||
EntityID: rec.Client,
|
||||
Timestamp: ts,
|
||||
Value: float64(rec.Value),
|
||||
Tags: tags,
|
||||
},
|
||||
{
|
||||
MetricName: metricstore.MetricPingLoss,
|
||||
EntityID: rec.Client,
|
||||
Timestamp: ts,
|
||||
Value: loss,
|
||||
Tags: tags,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func dropLegacyMonitoringTables(db *gorm.DB) error {
|
||||
for _, table := range legacyMonitoringTables {
|
||||
if !db.Migrator().HasTable(table) {
|
||||
continue
|
||||
}
|
||||
log.Printf("[legacy-migration] dropping legacy table %s", table)
|
||||
if err := db.Migrator().DropTable(table); err != nil {
|
||||
return fmt.Errorf("drop legacy table %s: %w", table, err)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,277 @@
|
||||
package migrations
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"errors"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/komari-monitor/komari/database/metricstore"
|
||||
"github.com/komari-monitor/komari/database/models"
|
||||
appconfig "github.com/komari-monitor/komari/pkg/config"
|
||||
"github.com/komari-monitor/komari/pkg/metric"
|
||||
"gorm.io/driver/sqlite"
|
||||
"gorm.io/gorm"
|
||||
)
|
||||
|
||||
func TestLegacyMonitoringTablesMigratedByOneShotMigration(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
mainDB, err := gorm.Open(sqlite.Open("file:"+filepath.ToSlash(filepath.Join(t.TempDir(), "komari.db"))+"?mode=rwc"), &gorm.Config{})
|
||||
if err != nil {
|
||||
t.Fatalf("open legacy db: %v", err)
|
||||
}
|
||||
if sqlDB, err := mainDB.DB(); err == nil {
|
||||
t.Cleanup(func() { _ = sqlDB.Close() })
|
||||
} else {
|
||||
t.Fatalf("legacy sql db: %v", err)
|
||||
}
|
||||
if err := mainDB.AutoMigrate(&models.Record{}, &models.GPURecord{}, &models.PingRecord{}); err != nil {
|
||||
t.Fatalf("migrate legacy tables: %v", err)
|
||||
}
|
||||
if err := mainDB.Table("records_long_term").AutoMigrate(&models.Record{}); err != nil {
|
||||
t.Fatalf("migrate legacy long-term table: %v", err)
|
||||
}
|
||||
|
||||
base := time.Date(2026, 7, 8, 23, 42, 0, 0, time.UTC)
|
||||
if err := mainDB.Create(&models.Record{Client: "client-a", Time: base, Cpu: 12.5, Ram: 2048}).Error; err != nil {
|
||||
t.Fatalf("seed records: %v", err)
|
||||
}
|
||||
if err := mainDB.Table("records_long_term").Create(&models.Record{Client: "client-a", Time: base.Add(time.Minute), Cpu: 22.5, Ram: 4096}).Error; err != nil {
|
||||
t.Fatalf("seed records_long_term: %v", err)
|
||||
}
|
||||
if err := mainDB.Create(&models.GPURecord{Client: "client-a", Time: base, DeviceIndex: 0, DeviceName: "GPU 0", MemUsed: 1024, MemTotal: 2048, Utilization: 67, Temperature: 55}).Error; err != nil {
|
||||
t.Fatalf("seed gpu_records: %v", err)
|
||||
}
|
||||
if err := mainDB.Create(&models.PingRecord{Client: "client-a", TaskId: 7, Time: base, Value: 36}).Error; err != nil {
|
||||
t.Fatalf("seed ping_records: %v", err)
|
||||
}
|
||||
if err := mainDB.Create(&models.PingRecord{Client: "client-a", TaskId: 7, Time: base.Add(30 * time.Second), Value: -1}).Error; err != nil {
|
||||
t.Fatalf("seed loss ping_records: %v", err)
|
||||
}
|
||||
summary, err := InspectLegacyMonitoring(mainDB)
|
||||
if err != nil {
|
||||
t.Fatalf("inspect legacy monitoring summary: %v", err)
|
||||
}
|
||||
if summary.LoadRows != 2 || summary.GPURows != 1 || summary.LatencyRows != 2 || summary.MonitoringRows != 5 {
|
||||
t.Fatalf("unexpected legacy monitoring summary: %#v", summary)
|
||||
}
|
||||
if summary.EstimatedPoints != 46 || summary.RetentionDays < 1 {
|
||||
t.Fatalf("unexpected legacy point estimate or retention: %#v", summary)
|
||||
}
|
||||
|
||||
metricDB, err := sql.Open("sqlite3", "file:"+filepath.ToSlash(filepath.Join(t.TempDir(), "metrics.db"))+"?mode=rwc")
|
||||
if err != nil {
|
||||
t.Fatalf("open metric db: %v", err)
|
||||
}
|
||||
metricStore, err := metric.Open(ctx, metric.SQLite("", metric.WithDB(metricDB)))
|
||||
if err != nil {
|
||||
t.Fatalf("open metric store: %v", err)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
_ = metricStore.Close()
|
||||
_ = metricDB.Close()
|
||||
})
|
||||
|
||||
markDoneCalls := 0
|
||||
var lastProgress LegacyMonitoringProgress
|
||||
if _, err := MigrateLegacyMonitoring(ctx, mainDB, metricStore, func(progress LegacyMonitoringProgress) {
|
||||
lastProgress = progress
|
||||
}); err != nil {
|
||||
t.Fatalf("migrate legacy monitoring with progress: %v", err)
|
||||
}
|
||||
if lastProgress.SourceRowsDone != 5 || lastProgress.SourceRowsTotal != 5 || lastProgress.WrittenPoints != 46 {
|
||||
t.Fatalf("unexpected final migration progress: %#v", lastProgress)
|
||||
}
|
||||
|
||||
stats, err := runLegacyMonitoringMigration(ctx, mainDB, metricStore, false, func() error {
|
||||
markDoneCalls++
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("run legacy monitoring migration: %v", err)
|
||||
}
|
||||
if markDoneCalls != 1 {
|
||||
t.Fatalf("expected migration marker to be written once, got %d", markDoneCalls)
|
||||
}
|
||||
if stats.Records != 2 || stats.GPU != 1 || stats.Ping != 2 {
|
||||
t.Fatalf("unexpected stats: %#v", stats)
|
||||
}
|
||||
|
||||
cpuPoints, err := metricStore.Query(ctx, metric.Query{MetricName: metricstore.MetricCPU, EntityID: "client-a", Start: base.Add(-time.Second), End: base.Add(2 * time.Minute)})
|
||||
if err != nil {
|
||||
t.Fatalf("query cpu points: %v", err)
|
||||
}
|
||||
if len(cpuPoints) != 2 || cpuPoints[0].Value != 12.5 || cpuPoints[1].Value != 22.5 {
|
||||
t.Fatalf("unexpected cpu points: %#v", cpuPoints)
|
||||
}
|
||||
|
||||
gpuPoints, err := metricStore.Query(ctx, metric.Query{MetricName: metricstore.MetricGPUDeviceUsage, EntityID: "client-a", Start: base.Add(-time.Second), End: base.Add(time.Second), Tags: map[string]string{"device_index": "0"}})
|
||||
if err != nil {
|
||||
t.Fatalf("query gpu points: %v", err)
|
||||
}
|
||||
if len(gpuPoints) != 1 || gpuPoints[0].Value != 67 {
|
||||
t.Fatalf("unexpected gpu points: %#v", gpuPoints)
|
||||
}
|
||||
|
||||
pingPoints, err := metricStore.Query(ctx, metric.Query{MetricName: metricstore.MetricPingLatency, EntityID: "client-a", Start: base.Add(-time.Second), End: base.Add(time.Minute), Tags: map[string]string{"task_id": "7"}})
|
||||
if err != nil {
|
||||
t.Fatalf("query ping points: %v", err)
|
||||
}
|
||||
if len(pingPoints) != 2 || pingPoints[0].Value != 36 || pingPoints[1].Value != -1 {
|
||||
t.Fatalf("unexpected ping points: %#v", pingPoints)
|
||||
}
|
||||
|
||||
pingLossPoints, err := metricStore.Query(ctx, metric.Query{MetricName: metricstore.MetricPingLoss, EntityID: "client-a", Start: base.Add(-time.Second), End: base.Add(time.Minute), Tags: map[string]string{"task_id": "7"}})
|
||||
if err != nil {
|
||||
t.Fatalf("query ping loss points: %v", err)
|
||||
}
|
||||
if len(pingLossPoints) != 2 || pingLossPoints[0].Value != 0 || pingLossPoints[1].Value != 1 {
|
||||
t.Fatalf("unexpected ping loss points: %#v", pingLossPoints)
|
||||
}
|
||||
|
||||
for _, table := range legacyMonitoringTables {
|
||||
if mainDB.Migrator().HasTable(table) {
|
||||
t.Fatalf("legacy table %s still exists", table)
|
||||
}
|
||||
}
|
||||
|
||||
stats, err = runLegacyMonitoringMigration(ctx, mainDB, metricStore, true, func() error {
|
||||
markDoneCalls++
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("rerun completed legacy monitoring migration: %v", err)
|
||||
}
|
||||
if stats != (LegacyMonitoringStats{}) {
|
||||
t.Fatalf("completed migration should not scan legacy tables, got %#v", stats)
|
||||
}
|
||||
if markDoneCalls != 1 {
|
||||
t.Fatalf("completed migration rewrote marker, calls=%d", markDoneCalls)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDeleteLegacyMonitoringBeforeOnlyRemovesOldHistory(t *testing.T) {
|
||||
db, err := gorm.Open(sqlite.Open("file:"+filepath.ToSlash(filepath.Join(t.TempDir(), "cleanup.db"))+"?mode=rwc"), &gorm.Config{})
|
||||
if err != nil {
|
||||
t.Fatalf("open cleanup db: %v", err)
|
||||
}
|
||||
if sqlDB, err := db.DB(); err == nil {
|
||||
t.Cleanup(func() { _ = sqlDB.Close() })
|
||||
} else {
|
||||
t.Fatalf("cleanup sql db: %v", err)
|
||||
}
|
||||
if err := db.AutoMigrate(&models.Client{}, &models.Record{}, &models.GPURecord{}, &models.PingRecord{}); err != nil {
|
||||
t.Fatalf("migrate cleanup tables: %v", err)
|
||||
}
|
||||
if err := db.Table("records_long_term").AutoMigrate(&models.Record{}); err != nil {
|
||||
t.Fatalf("migrate cleanup long-term table: %v", err)
|
||||
}
|
||||
if err := db.Create(&models.Client{UUID: "client-a", Token: "token-a", Name: "A"}).Error; err != nil {
|
||||
t.Fatalf("seed client: %v", err)
|
||||
}
|
||||
|
||||
cutoff := time.Date(2026, 7, 10, 0, 0, 0, 0, time.UTC)
|
||||
oldTime := cutoff.Add(-time.Hour)
|
||||
newTime := cutoff.Add(time.Hour)
|
||||
for _, record := range []models.Record{{Client: "client-a", Time: oldTime}, {Client: "client-a", Time: newTime}} {
|
||||
if err := db.Create(&record).Error; err != nil {
|
||||
t.Fatalf("seed load record: %v", err)
|
||||
}
|
||||
}
|
||||
for _, record := range []models.GPURecord{{Client: "client-a", Time: oldTime}, {Client: "client-a", Time: newTime}} {
|
||||
if err := db.Create(&record).Error; err != nil {
|
||||
t.Fatalf("seed GPU record: %v", err)
|
||||
}
|
||||
}
|
||||
for _, record := range []models.PingRecord{{Client: "client-a", TaskId: 1, Time: oldTime}, {Client: "client-a", TaskId: 1, Time: newTime}} {
|
||||
if err := db.Create(&record).Error; err != nil {
|
||||
t.Fatalf("seed ping record: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
deleted, err := DeleteLegacyMonitoringBefore(db, cutoff)
|
||||
if err != nil {
|
||||
t.Fatalf("delete legacy monitoring before cutoff: %v", err)
|
||||
}
|
||||
if deleted.LoadRows != 1 || deleted.GPURows != 1 || deleted.LatencyRows != 1 {
|
||||
t.Fatalf("unexpected deleted rows: %#v", deleted)
|
||||
}
|
||||
summary, err := InspectLegacyMonitoring(db)
|
||||
if err != nil {
|
||||
t.Fatalf("inspect remaining legacy monitoring: %v", err)
|
||||
}
|
||||
if summary.LoadRows != 1 || summary.GPURows != 1 || summary.LatencyRows != 1 || summary.ServerCount != 1 {
|
||||
t.Fatalf("unexpected remaining legacy rows: %#v", summary)
|
||||
}
|
||||
var clients int64
|
||||
if err := db.Model(&models.Client{}).Count(&clients).Error; err != nil || clients != 1 {
|
||||
t.Fatalf("cleanup changed clients: count=%d err=%v", clients, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCompleteLegacyMonitoringMigrationFinalizesBeforeMarkingDone(t *testing.T) {
|
||||
db, err := gorm.Open(sqlite.Open("file:"+filepath.ToSlash(filepath.Join(t.TempDir(), "finalize.db"))+"?mode=rwc"), &gorm.Config{})
|
||||
if err != nil {
|
||||
t.Fatalf("open finalization database: %v", err)
|
||||
}
|
||||
if sqlDB, err := db.DB(); err == nil {
|
||||
t.Cleanup(func() { _ = sqlDB.Close() })
|
||||
} else {
|
||||
t.Fatalf("finalization sql db: %v", err)
|
||||
}
|
||||
appconfig.SetDb(db)
|
||||
if err := db.AutoMigrate(&models.Record{}); err != nil {
|
||||
t.Fatalf("migrate legacy table: %v", err)
|
||||
}
|
||||
if err := appconfig.Set(legacyMonitoringMigrationDoneKey, false); err != nil {
|
||||
t.Fatalf("reset completion marker: %v", err)
|
||||
}
|
||||
|
||||
finalized := false
|
||||
if err := CompleteLegacyMonitoringMigration(db, func() error {
|
||||
finalized = true
|
||||
if db.Migrator().HasTable(&models.Record{}) {
|
||||
t.Fatal("legacy table still exists during finalization")
|
||||
}
|
||||
done, err := appconfig.GetAs[bool](legacyMonitoringMigrationDoneKey, false)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if done {
|
||||
t.Fatal("completion marker was written before finalization")
|
||||
}
|
||||
return nil
|
||||
}); err != nil {
|
||||
t.Fatalf("complete legacy migration: %v", err)
|
||||
}
|
||||
if !finalized {
|
||||
t.Fatal("finalizer was not called")
|
||||
}
|
||||
done, err := appconfig.GetAs[bool](legacyMonitoringMigrationDoneKey, false)
|
||||
if err != nil {
|
||||
t.Fatalf("read completion marker: %v", err)
|
||||
}
|
||||
if !done {
|
||||
t.Fatal("completion marker was not written after finalization")
|
||||
}
|
||||
|
||||
if err := db.AutoMigrate(&models.Record{}); err != nil {
|
||||
t.Fatalf("restore legacy table: %v", err)
|
||||
}
|
||||
if err := appconfig.Set(legacyMonitoringMigrationDoneKey, false); err != nil {
|
||||
t.Fatalf("reset completion marker after success: %v", err)
|
||||
}
|
||||
finalizeErr := errors.New("vacuum failed")
|
||||
if err := CompleteLegacyMonitoringMigration(db, func() error { return finalizeErr }); !errors.Is(err, finalizeErr) {
|
||||
t.Fatalf("finalization error = %v, want %v", err, finalizeErr)
|
||||
}
|
||||
done, err = appconfig.GetAs[bool](legacyMonitoringMigrationDoneKey, false)
|
||||
if err != nil {
|
||||
t.Fatalf("read completion marker after failure: %v", err)
|
||||
}
|
||||
if done {
|
||||
t.Fatal("failed finalization wrote the completion marker")
|
||||
}
|
||||
}
|
||||
@@ -27,8 +27,6 @@ type legacyModelConfig struct {
|
||||
EulaAccepted bool `json:"eula_accepted" gorm:"default:false"`
|
||||
GeoIpEnabled bool `json:"geo_ip_enabled" gorm:"default:true"`
|
||||
GeoIpProvider string `json:"geo_ip_provider" gorm:"type:varchar(20);default:'ip-api'"`
|
||||
NezhaCompatEnabled bool `json:"nezha_compat_enabled" gorm:"default:false"`
|
||||
NezhaCompatListen string `json:"nezha_compat_listen" gorm:"type:varchar(100);default:''"`
|
||||
OAuthEnabled bool `json:"o_auth_enabled" gorm:"default:false"`
|
||||
OAuthProvider string `json:"o_auth_provider" gorm:"type:varchar(50);default:'github'"`
|
||||
DisablePasswordLogin bool `json:"disable_password_login" gorm:"default:false"`
|
||||
@@ -41,11 +39,8 @@ type legacyModelConfig struct {
|
||||
ExpireNotificationLeadDays int `json:"expire_notification_lead_days" gorm:"default:7"`
|
||||
LoginNotification bool `json:"login_notification" gorm:"default:false"`
|
||||
TrafficLimitPercentage float64 `json:"traffic_limit_percentage" gorm:"default:80.00"`
|
||||
RecordEnabled bool `json:"record_enabled" gorm:"default:true"`
|
||||
RecordPreserveTime int `json:"record_preserve_time" gorm:"default:720"`
|
||||
PingRecordPreserveTime int `json:"ping_record_preserve_time" gorm:"default:24"`
|
||||
CreatedAt models.LocalTime
|
||||
UpdatedAt models.LocalTime
|
||||
CreatedAt time.Time
|
||||
UpdatedAt time.Time
|
||||
}
|
||||
|
||||
func (legacyModelConfig) TableName() string {
|
||||
@@ -66,12 +61,9 @@ type legacyConfig struct {
|
||||
BaseScriptsURLKey string `json:"base_scripts_url"`
|
||||
GeoIpEnabled bool `json:"geo_ip_enabled"`
|
||||
GeoIpProvider string `json:"geo_ip_provider"`
|
||||
NezhaCompatEnabled bool `json:"nezha_compat_enabled"`
|
||||
NezhaCompatListen string `json:"nezha_compat_listen"`
|
||||
OAuthEnabled bool `json:"o_auth_enabled"`
|
||||
OAuthProvider string `json:"o_auth_provider"`
|
||||
DisablePasswordLogin bool `json:"disable_password_login"`
|
||||
CloudflareTunnelToken string `json:"cloudflare_tunnel_token"`
|
||||
CustomHead string `json:"custom_head"`
|
||||
CustomBody string `json:"custom_body"`
|
||||
NotificationEnabled bool `json:"notification_enabled"`
|
||||
@@ -81,9 +73,6 @@ type legacyConfig struct {
|
||||
ExpireNotificationLeadDays int `json:"expire_notification_lead_days"`
|
||||
LoginNotification bool `json:"login_notification"`
|
||||
TrafficLimitPercentage float64 `json:"traffic_limit_percentage"`
|
||||
RecordEnabled bool `json:"record_enabled"`
|
||||
RecordPreserveTime int `json:"record_preserve_time"`
|
||||
PingRecordPreserveTime int `json:"ping_record_preserve_time"`
|
||||
UpdatedAt time.Time `json:"updated_at"`
|
||||
}
|
||||
|
||||
@@ -108,6 +97,9 @@ func Run(ctx Context) error {
|
||||
}
|
||||
|
||||
legacyConfigTable := hasLegacyConfigTable(db)
|
||||
if err := migrateLegacyTimestampColumns(db); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if legacyConfigTable {
|
||||
if err := migrateLegacyOidcConfig(db); err != nil {
|
||||
@@ -117,7 +109,6 @@ func Run(ctx Context) error {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
if err := migrateLegacyClientInfo(db); err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -132,10 +123,36 @@ func Run(ctx Context) error {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if err := migrateDeprecatedMetricRetentionConfig(db); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := migrateRemovedCompatibilityConfig(db); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := markTimestampMigrationDone(db); err != nil {
|
||||
return fmt.Errorf("mark UTC timestamp migration done: %w", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func migrateDeprecatedMetricRetentionConfig(db *gorm.DB) error {
|
||||
if !db.Migrator().HasTable(&appconfig.ConfigItem{}) {
|
||||
return nil
|
||||
}
|
||||
return db.Delete(&appconfig.ConfigItem{}, "key = ?", "metric_retention_days").Error
|
||||
}
|
||||
|
||||
func migrateRemovedCompatibilityConfig(db *gorm.DB) error {
|
||||
if !db.Migrator().HasTable(&appconfig.ConfigItem{}) {
|
||||
return nil
|
||||
}
|
||||
return db.Delete(&appconfig.ConfigItem{}, "key IN ?", []string{
|
||||
"nezha_compat_enabled",
|
||||
"nezha_compat_listen",
|
||||
}).Error
|
||||
}
|
||||
|
||||
func hasLegacyConfigTable(db *gorm.DB) bool {
|
||||
if !db.Migrator().HasTable("configs") {
|
||||
return false
|
||||
@@ -390,7 +407,7 @@ func migrateLegacyClientInfo(db *gorm.DB) error {
|
||||
client.Weight = info.Weight
|
||||
client.Price = info.Price
|
||||
client.BillingCycle = info.BillingCycle
|
||||
client.ExpiredAt = models.FromTime(info.ExpiredAt)
|
||||
client.ExpiredAt = info.ExpiredAt
|
||||
if err := db.Save(&client).Error; err != nil {
|
||||
return fmt.Errorf("update Client record %s: %w", info.UUID, err)
|
||||
}
|
||||
|
||||
@@ -79,6 +79,59 @@ func TestRunSkipsLegacyConfigMigrationForCurrentConfigItemTable(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunRemovesDeprecatedMetricRetentionConfig(t *testing.T) {
|
||||
db := openTestDB(t, "migrations_remove_metric_retention")
|
||||
if err := db.AutoMigrate(&appconfig.ConfigItem{}); err != nil {
|
||||
t.Fatalf("migrate config item table: %v", err)
|
||||
}
|
||||
if err := db.Create(&appconfig.ConfigItem{Key: "metric_retention_days", Value: "90"}).Error; err != nil {
|
||||
t.Fatalf("seed deprecated config: %v", err)
|
||||
}
|
||||
if err := Run(Context{DB: db}); err != nil {
|
||||
t.Fatalf("run migrations: %v", err)
|
||||
}
|
||||
var count int64
|
||||
if err := db.Model(&appconfig.ConfigItem{}).Where("key = ?", "metric_retention_days").Count(&count).Error; err != nil {
|
||||
t.Fatalf("count deprecated config: %v", err)
|
||||
}
|
||||
if count != 0 {
|
||||
t.Fatalf("deprecated metric retention config was not removed: %d", count)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunRemovesCompatibilityConfig(t *testing.T) {
|
||||
db := openTestDB(t, "migrations_remove_compatibility_config")
|
||||
if err := db.AutoMigrate(&appconfig.ConfigItem{}); err != nil {
|
||||
t.Fatalf("migrate config item table: %v", err)
|
||||
}
|
||||
for _, key := range []string{"nezha_compat_enabled", "nezha_compat_listen"} {
|
||||
if err := db.Create(&appconfig.ConfigItem{Key: key, Value: "true"}).Error; err != nil {
|
||||
t.Fatalf("seed removed config %q: %v", key, err)
|
||||
}
|
||||
}
|
||||
if err := db.Create(&appconfig.ConfigItem{Key: "sitename", Value: `"Komari"`}).Error; err != nil {
|
||||
t.Fatalf("seed retained config: %v", err)
|
||||
}
|
||||
|
||||
if err := Run(Context{DB: db}); err != nil {
|
||||
t.Fatalf("run migrations: %v", err)
|
||||
}
|
||||
|
||||
var count int64
|
||||
if err := db.Model(&appconfig.ConfigItem{}).Where("key IN ?", []string{
|
||||
"nezha_compat_enabled",
|
||||
"nezha_compat_listen",
|
||||
}).Count(&count).Error; err != nil {
|
||||
t.Fatalf("count removed config: %v", err)
|
||||
}
|
||||
if count != 0 {
|
||||
t.Fatalf("removed compatibility config remains: %d", count)
|
||||
}
|
||||
if err := db.First(&appconfig.ConfigItem{}, "key = ?", "sitename").Error; err != nil {
|
||||
t.Fatalf("unrelated config was removed: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunPreservesVersion120RuntimeShape(t *testing.T) {
|
||||
db := openTestDB(t, "migrations_v120_runtime_shape")
|
||||
if err := db.AutoMigrate(
|
||||
@@ -90,7 +143,7 @@ func TestRunPreservesVersion120RuntimeShape(t *testing.T) {
|
||||
); err != nil {
|
||||
t.Fatalf("migrate 1.2.0 runtime shape: %v", err)
|
||||
}
|
||||
now := models.FromTime(time.Now())
|
||||
now := time.Now().UTC()
|
||||
if err := db.Create(&models.Client{UUID: "client-a", Token: "token-a", CreatedAt: now, UpdatedAt: now}).Error; err != nil {
|
||||
t.Fatalf("seed client: %v", err)
|
||||
}
|
||||
@@ -169,9 +222,6 @@ func TestRunMigratesLegacyConfigTableToConfigItems(t *testing.T) {
|
||||
OAuthProvider: "github",
|
||||
NotificationMethod: "none",
|
||||
TrafficLimitPercentage: 66.5,
|
||||
RecordEnabled: true,
|
||||
RecordPreserveTime: 48,
|
||||
PingRecordPreserveTime: 12,
|
||||
ExpireNotificationLeadDays: 3,
|
||||
}
|
||||
if err := db.Create(&legacy).Error; err != nil {
|
||||
@@ -218,7 +268,7 @@ func TestRunExpandsLegacyPingAllClientsTasks(t *testing.T) {
|
||||
`).Error; err != nil {
|
||||
t.Fatalf("create legacy ping_tasks: %v", err)
|
||||
}
|
||||
now := models.FromTime(time.Now())
|
||||
now := time.Now().UTC()
|
||||
clients := []models.Client{
|
||||
{UUID: "client-a", Token: "token-a", CreatedAt: now, UpdatedAt: now},
|
||||
{UUID: "client-b", Token: "token-b", CreatedAt: now, UpdatedAt: now},
|
||||
|
||||
@@ -0,0 +1,242 @@
|
||||
package migrations
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"fmt"
|
||||
"log"
|
||||
"math"
|
||||
"os"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
appconfig "github.com/komari-monitor/komari/pkg/config"
|
||||
"gorm.io/gorm"
|
||||
"gorm.io/gorm/clause"
|
||||
)
|
||||
|
||||
const timestampUTCMigrationKey = "internal_timestamp_utc_migrated"
|
||||
|
||||
type timestampColumn struct {
|
||||
table string
|
||||
column string
|
||||
}
|
||||
|
||||
var legacyTimestampColumns = [...]timestampColumn{
|
||||
{table: "clients", column: "expired_at"},
|
||||
{table: "clients", column: "created_at"},
|
||||
{table: "clients", column: "updated_at"},
|
||||
{table: "client_infos", column: "expired_at"},
|
||||
{table: "client_infos", column: "created_at"},
|
||||
{table: "client_infos", column: "updated_at"},
|
||||
{table: "users", column: "created_at"},
|
||||
{table: "users", column: "updated_at"},
|
||||
{table: "sessions", column: "latest_online"},
|
||||
{table: "sessions", column: "expires"},
|
||||
{table: "sessions", column: "created_at"},
|
||||
{table: "logs", column: "time"},
|
||||
{table: "clipboards", column: "created_at"},
|
||||
{table: "clipboards", column: "updated_at"},
|
||||
{table: "offline_notifications", column: "last_notified"},
|
||||
{table: "load_notifications", column: "last_notified"},
|
||||
{table: "task_results", column: "finished_at"},
|
||||
{table: "task_results", column: "created_at"},
|
||||
{table: "records", column: "time"},
|
||||
{table: "records_long_term", column: "time"},
|
||||
{table: "gpu_records", column: "time"},
|
||||
{table: "ping_records", column: "time"},
|
||||
{table: "configs", column: "created_at"},
|
||||
{table: "configs", column: "updated_at"},
|
||||
}
|
||||
|
||||
var legacyTimestampLayouts = [...]string{
|
||||
"2006-01-02 15:04:05.999999999-07:00",
|
||||
"2006-01-02 15:04:05-07:00",
|
||||
"2006-01-02 15:04:05.999999999",
|
||||
"2006-01-02 15:04:05",
|
||||
"2006-01-02T15:04:05.999999999",
|
||||
"2006-01-02T15:04:05",
|
||||
"2006-01-02",
|
||||
}
|
||||
|
||||
type timestampRow struct {
|
||||
rowID int64
|
||||
value sql.NullString
|
||||
}
|
||||
|
||||
// migrateLegacyTimestampColumns makes old offset-free SQLite values
|
||||
// unambiguous before any current model scans them as time.Time.
|
||||
func migrateLegacyTimestampColumns(db *gorm.DB) error {
|
||||
if timestampMigrationDone(db) {
|
||||
return nil
|
||||
}
|
||||
|
||||
location := legacyTimestampLocation()
|
||||
var converted int64
|
||||
err := db.Transaction(func(tx *gorm.DB) error {
|
||||
for _, target := range legacyTimestampColumns {
|
||||
if !tx.Migrator().HasTable(target.table) || !tx.Migrator().HasColumn(target.table, target.column) {
|
||||
continue
|
||||
}
|
||||
count, err := migrateTimestampColumn(tx, target, location)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
converted += count
|
||||
}
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if converted > 0 {
|
||||
log.Printf("Converted %d legacy timestamp values to explicit UTC", converted)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func migrateTimestampColumn(db *gorm.DB, target timestampColumn, location *time.Location) (int64, error) {
|
||||
table := quoteSQLiteIdentifier(target.table)
|
||||
column := quoteSQLiteIdentifier(target.column)
|
||||
var converted int64
|
||||
var lastRowID int64
|
||||
|
||||
for {
|
||||
query := fmt.Sprintf(
|
||||
"SELECT rowid, CAST(%s AS TEXT) FROM %s WHERE rowid > ? AND %s IS NOT NULL ORDER BY rowid LIMIT 1000",
|
||||
column, table, column,
|
||||
)
|
||||
rows, err := db.Raw(query, lastRowID).Rows()
|
||||
if err != nil {
|
||||
return converted, fmt.Errorf("read legacy timestamp %s.%s: %w", target.table, target.column, err)
|
||||
}
|
||||
|
||||
batch := make([]timestampRow, 0, 1000)
|
||||
for rows.Next() {
|
||||
var row timestampRow
|
||||
if err := rows.Scan(&row.rowID, &row.value); err != nil {
|
||||
rows.Close()
|
||||
return converted, fmt.Errorf("scan legacy timestamp %s.%s: %w", target.table, target.column, err)
|
||||
}
|
||||
batch = append(batch, row)
|
||||
lastRowID = row.rowID
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
rows.Close()
|
||||
return converted, fmt.Errorf("iterate legacy timestamp %s.%s: %w", target.table, target.column, err)
|
||||
}
|
||||
rows.Close()
|
||||
if len(batch) == 0 {
|
||||
break
|
||||
}
|
||||
|
||||
for _, row := range batch {
|
||||
raw := strings.TrimSpace(row.value.String)
|
||||
if !row.value.Valid || raw == "" {
|
||||
continue
|
||||
}
|
||||
stamp, err := parseLegacyTimestamp(raw, location)
|
||||
if err != nil {
|
||||
return converted, fmt.Errorf("convert legacy timestamp %s.%s rowid=%d: %w", target.table, target.column, row.rowID, err)
|
||||
}
|
||||
update := fmt.Sprintf("UPDATE %s SET %s = ? WHERE rowid = ?", table, column)
|
||||
if err := db.Exec(update, stamp.UTC(), row.rowID).Error; err != nil {
|
||||
return converted, fmt.Errorf("write UTC timestamp %s.%s rowid=%d: %w", target.table, target.column, row.rowID, err)
|
||||
}
|
||||
converted++
|
||||
}
|
||||
if len(batch) < 1000 {
|
||||
break
|
||||
}
|
||||
}
|
||||
return converted, nil
|
||||
}
|
||||
|
||||
func parseLegacyTimestamp(value string, location *time.Location) (time.Time, error) {
|
||||
raw := strings.TrimSpace(value)
|
||||
if raw == "" {
|
||||
return time.Time{}, fmt.Errorf("timestamp is empty")
|
||||
}
|
||||
if location == nil {
|
||||
location = time.UTC
|
||||
}
|
||||
if stamp, err := time.Parse(time.RFC3339Nano, raw); err == nil {
|
||||
return stamp.UTC(), nil
|
||||
}
|
||||
for _, layout := range legacyTimestampLayouts {
|
||||
if stamp, err := time.ParseInLocation(layout, raw, location); err == nil {
|
||||
return stamp.UTC(), nil
|
||||
}
|
||||
}
|
||||
epoch, err := strconv.ParseInt(raw, 10, 64)
|
||||
if err != nil {
|
||||
return time.Time{}, fmt.Errorf("unsupported legacy timestamp %q", value)
|
||||
}
|
||||
return legacyEpochTime(epoch), nil
|
||||
}
|
||||
|
||||
func legacyEpochTime(value int64) time.Time {
|
||||
abs := value
|
||||
if abs < 0 {
|
||||
if abs == math.MinInt64 {
|
||||
return time.Unix(0, value).UTC()
|
||||
}
|
||||
abs = -abs
|
||||
}
|
||||
switch {
|
||||
case abs >= 1e17:
|
||||
return time.Unix(0, value).UTC()
|
||||
case abs >= 1e14:
|
||||
return time.Unix(0, value*int64(time.Microsecond)).UTC()
|
||||
case abs >= 1e11:
|
||||
return time.UnixMilli(value).UTC()
|
||||
default:
|
||||
return time.Unix(value, 0).UTC()
|
||||
}
|
||||
}
|
||||
|
||||
// legacyTimestampLocation is intentionally migration-only. The old custom
|
||||
// type used TZ and defaulted to UTC; current runtime code uses system time.Local.
|
||||
func legacyTimestampLocation() *time.Location {
|
||||
name := strings.TrimSpace(os.Getenv("TZ"))
|
||||
if name == "" {
|
||||
return time.UTC
|
||||
}
|
||||
location, err := time.LoadLocation(name)
|
||||
if err != nil {
|
||||
log.Printf("Legacy timezone %q cannot be loaded; interpreting old timestamps as UTC: %v", name, err)
|
||||
return time.UTC
|
||||
}
|
||||
return location
|
||||
}
|
||||
|
||||
func timestampMigrationDone(db *gorm.DB) bool {
|
||||
if !db.Migrator().HasTable(&appconfig.ConfigItem{}) || hasLegacyConfigTable(db) {
|
||||
return false
|
||||
}
|
||||
var item appconfig.ConfigItem
|
||||
if err := db.Where("key = ?", timestampUTCMigrationKey).First(&item).Error; err != nil {
|
||||
return false
|
||||
}
|
||||
return item.Value == "true"
|
||||
}
|
||||
|
||||
func markTimestampMigrationDone(db *gorm.DB) error {
|
||||
if hasLegacyConfigTable(db) {
|
||||
return fmt.Errorf("new config item table is unavailable")
|
||||
}
|
||||
if !db.Migrator().HasTable(&appconfig.ConfigItem{}) {
|
||||
if err := db.AutoMigrate(&appconfig.ConfigItem{}); err != nil {
|
||||
return fmt.Errorf("create config item table: %w", err)
|
||||
}
|
||||
}
|
||||
item := appconfig.ConfigItem{Key: timestampUTCMigrationKey, Value: "true"}
|
||||
return db.Clauses(clause.OnConflict{
|
||||
Columns: []clause.Column{{Name: "key"}},
|
||||
DoUpdates: clause.AssignmentColumns([]string{"value"}),
|
||||
}).Create(&item).Error
|
||||
}
|
||||
|
||||
func quoteSQLiteIdentifier(value string) string {
|
||||
return `"` + strings.ReplaceAll(value, `"`, `""`) + `"`
|
||||
}
|
||||
Loaded 100 of 227 files, more files were not shown because too many files have changed in this diff.
Show more
Reference in new issue
Block a user