mirror of
https://github.com/komari-monitor/komari.git
synced 2026-11-11 08:42:39 +00:00
Compare commits
104
Commits
1.2.5-fix1
...
1.3.1
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
d69effee7a | ||
|
|
ebe5278183 | ||
|
|
16d6f2670d | ||
|
|
abc53b3cc0 | ||
|
|
a1ae82169c | ||
|
|
8c849aa423 | ||
|
|
0d699951b7 | ||
|
|
1ffe413eb7 | ||
|
|
c3c3ed8ecf | ||
|
|
0103b99651 | ||
|
|
b8c100823f | ||
|
|
c7819cd96e | ||
|
|
17aa37c64f | ||
|
|
045f24ef85 | ||
|
|
6ab9c6724e | ||
|
|
6efc72ce31 | ||
|
|
1f6bd1391a | ||
|
|
5b93181d94 | ||
|
|
f07bd749c4 | ||
|
|
887381ad4f | ||
|
|
11617b46b3 | ||
|
|
5a8488926c | ||
|
|
db2ed8e424 | ||
|
|
312834f921 | ||
|
|
9d149ab660 | ||
|
|
9d7e24ab78 | ||
|
|
99c684e75e | ||
|
|
a402cc6189 | ||
|
|
0bac110d8a | ||
|
|
7a955467fd | ||
|
|
eb7ecc6fe4 | ||
|
|
f59e0e813c | ||
|
|
b4d3977871 | ||
|
|
b55436842f | ||
|
|
cdfd5bd13f | ||
|
|
5a66f28e81 | ||
|
|
be3daa42fc | ||
|
|
42b290f169 | ||
|
|
99fda6b023 | ||
|
|
7f128509e9 | ||
|
|
fee8c860d7 | ||
|
|
3c278230f4 | ||
|
|
d853a83ec6 | ||
|
|
a78ef878eb | ||
|
|
312e320c8f | ||
|
|
73c3c436a4 | ||
|
|
f42cfe7ddc | ||
|
|
0f6f2f4294 | ||
|
|
fc3febc3be | ||
|
|
5b90804a13 | ||
|
|
43547b947f | ||
|
|
e2a8284c03 | ||
|
|
c1178ebfe1 | ||
|
|
7baf2edf26 | ||
|
|
66b7691e01 | ||
|
|
7f368817e0 | ||
|
|
5fa59ab7de | ||
|
|
0c80f0f23a | ||
|
|
1eb3549ed9 | ||
|
|
5c73eba475 | ||
|
|
76794305c8 | ||
|
|
724dd3b666 | ||
|
|
c828653d20 | ||
|
|
ff358edd8b | ||
|
|
3b48339209 | ||
|
|
42ef993cd6 | ||
|
|
39c5f7bb69 | ||
|
|
9831df0af8 | ||
|
|
29eecf55f7 | ||
|
|
0660ee74d4 | ||
|
|
5ade710f32 | ||
|
|
855ff82f97 | ||
|
|
8910c2e9c7 | ||
|
|
b2fe9ab068 | ||
|
|
8a529b4eb0 | ||
|
|
46b4e37178 | ||
|
|
5ad791fe50 | ||
|
|
8911c08a5b | ||
|
|
7d95240bc1 | ||
|
|
7e0d542694 | ||
|
|
84130dfff5 | ||
|
|
e0094447a6 | ||
|
|
ab31fc69fe | ||
|
|
73071ff1e8 | ||
|
|
4ad190a7f4 | ||
|
|
6ddb143365 | ||
|
|
eb4ca83a13 | ||
|
|
43c80d238b | ||
|
|
416ea871be | ||
|
|
5c2bd4ec74 | ||
|
|
be6af33d85 | ||
|
|
d02d9347dc | ||
|
|
64998b527c | ||
|
|
9e5b6df196 | ||
|
|
332053d8ec | ||
|
|
b6fcdd396d | ||
|
|
92ec774249 | ||
|
|
404d97d846 | ||
|
|
0132d1cbd8 | ||
|
|
ff6daa6ce3 | ||
|
|
6a5f3ef04d | ||
|
|
da047cf612 | ||
|
|
f0877d6d32 | ||
|
|
f2c46dbfb1 |
No files matched your search
@@ -0,0 +1,62 @@
|
||||
name: Build frontend theme
|
||||
description: Clone komari-web, build the default theme, and optionally upload it as an artifact.
|
||||
|
||||
inputs:
|
||||
node-version:
|
||||
description: Node.js version used to build komari-web.
|
||||
default: "23"
|
||||
repository:
|
||||
description: komari-web repository URL.
|
||||
default: https://github.com/komari-monitor/komari-web
|
||||
ref:
|
||||
description: Optional komari-web ref to build. Uses the repository default branch when empty.
|
||||
default: ""
|
||||
artifact-name:
|
||||
description: Optional artifact name for the packed frontend theme.
|
||||
default: ""
|
||||
|
||||
runs:
|
||||
using: composite
|
||||
steps:
|
||||
- name: Setup Node.js
|
||||
uses: actions/setup-node@v4
|
||||
with:
|
||||
node-version: ${{ inputs.node-version }}
|
||||
|
||||
- name: Clone and build frontend
|
||||
shell: bash
|
||||
run: |
|
||||
set -euxo pipefail
|
||||
rm -rf komari-web
|
||||
if [ -n "${{ inputs.ref }}" ]; then
|
||||
git clone --depth=1 --branch "${{ inputs.ref }}" "${{ inputs.repository }}" komari-web
|
||||
else
|
||||
git clone --depth=1 "${{ inputs.repository }}" komari-web
|
||||
fi
|
||||
cd komari-web
|
||||
npm install
|
||||
npm run build
|
||||
cd ..
|
||||
|
||||
mkdir -p web/public/defaultTheme/dist
|
||||
rm -rf web/public/defaultTheme/dist/*
|
||||
cp -r komari-web/dist/* web/public/defaultTheme/dist/
|
||||
cp -f komari-web/komari-theme.json web/public/defaultTheme/
|
||||
if [ -f komari-web/preview.png ]; then cp -f komari-web/preview.png web/public/defaultTheme/; fi
|
||||
if [ -f komari-web/perview.png ]; then cp -f komari-web/perview.png web/public/defaultTheme/; fi
|
||||
if [ -f web/public/defaultTheme/preview.png ] && [ ! -f web/public/defaultTheme/perview.png ]; then cp -f web/public/defaultTheme/preview.png web/public/defaultTheme/perview.png; fi
|
||||
if [ -f web/public/defaultTheme/perview.png ] && [ ! -f web/public/defaultTheme/preview.png ]; then cp -f web/public/defaultTheme/perview.png web/public/defaultTheme/preview.png; fi
|
||||
|
||||
- name: Pack frontend artifact
|
||||
if: ${{ inputs.artifact-name != '' }}
|
||||
shell: bash
|
||||
run: |
|
||||
set -euxo pipefail
|
||||
tar -czf "$RUNNER_TEMP/${{ inputs.artifact-name }}.tar.gz" -C web/public/defaultTheme .
|
||||
|
||||
- name: Upload frontend artifact
|
||||
if: ${{ inputs.artifact-name != '' }}
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: ${{ inputs.artifact-name }}
|
||||
path: ${{ runner.temp }}/${{ inputs.artifact-name }}.tar.gz
|
||||
@@ -0,0 +1,24 @@
|
||||
name: Restore frontend theme
|
||||
description: Download and unpack a previously built frontend theme artifact.
|
||||
|
||||
inputs:
|
||||
artifact-name:
|
||||
description: Frontend theme artifact name.
|
||||
default: komari-frontend
|
||||
|
||||
runs:
|
||||
using: composite
|
||||
steps:
|
||||
- name: Download frontend artifact
|
||||
uses: actions/download-artifact@v4
|
||||
with:
|
||||
name: ${{ inputs.artifact-name }}
|
||||
path: ${{ runner.temp }}/frontend-artifact
|
||||
|
||||
- name: Extract frontend artifact
|
||||
shell: bash
|
||||
run: |
|
||||
set -euxo pipefail
|
||||
mkdir -p web/public/defaultTheme
|
||||
rm -rf web/public/defaultTheme/*
|
||||
tar -xzf "$RUNNER_TEMP/frontend-artifact/${{ inputs.artifact-name }}.tar.gz" -C web/public/defaultTheme
|
||||
@@ -0,0 +1,31 @@
|
||||
name: Setup Zig
|
||||
description: Restore or install the Zig toolchain used for CGO cross-compilation.
|
||||
|
||||
inputs:
|
||||
zig-version:
|
||||
description: Zig version to install.
|
||||
default: 0.14.1
|
||||
|
||||
runs:
|
||||
using: composite
|
||||
steps:
|
||||
- name: Restore Zig cache
|
||||
id: zig-cache
|
||||
uses: actions/cache@v4
|
||||
with:
|
||||
path: ~/.cache/komari-zig/zig-x86_64-linux-${{ inputs.zig-version }}
|
||||
key: zig-${{ runner.os }}-x86_64-${{ inputs.zig-version }}
|
||||
|
||||
- name: Download Zig
|
||||
if: ${{ steps.zig-cache.outputs.cache-hit != 'true' }}
|
||||
shell: bash
|
||||
run: |
|
||||
set -euxo pipefail
|
||||
mkdir -p "$HOME/.cache/komari-zig"
|
||||
curl -fsSL "https://ziglang.org/download/${{ inputs.zig-version }}/zig-x86_64-linux-${{ inputs.zig-version }}.tar.xz" -o "$RUNNER_TEMP/zig.tar.xz"
|
||||
tar -C "$HOME/.cache/komari-zig" -Jxf "$RUNNER_TEMP/zig.tar.xz"
|
||||
|
||||
- name: Add Zig to PATH
|
||||
shell: bash
|
||||
run: |
|
||||
echo "$HOME/.cache/komari-zig/zig-x86_64-linux-${{ inputs.zig-version }}" >> "$GITHUB_PATH"
|
||||
+20
-27
@@ -7,7 +7,20 @@ on:
|
||||
workflow_dispatch:
|
||||
|
||||
jobs:
|
||||
build-frontend:
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Build frontend artifact
|
||||
uses: ./.github/actions/build-frontend
|
||||
with:
|
||||
artifact-name: komari-frontend
|
||||
|
||||
build-binaries:
|
||||
needs: build-frontend
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
strategy:
|
||||
@@ -46,6 +59,10 @@ jobs:
|
||||
goarch: riscv64
|
||||
zig_target_triple: riscv64-linux-musl
|
||||
binary_extension: ""
|
||||
- goos: linux
|
||||
goarch: loong64
|
||||
zig_target_triple: loongarch64-linux-musl
|
||||
binary_extension: ""
|
||||
|
||||
exclude:
|
||||
- goos: windows
|
||||
@@ -55,27 +72,8 @@ jobs:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Setup Node.js
|
||||
uses: actions/setup-node@v4
|
||||
with:
|
||||
node-version: "23"
|
||||
|
||||
- name: Clone and build frontend
|
||||
run: |
|
||||
git clone https://github.com/komari-monitor/komari-web komari-web
|
||||
cd komari-web
|
||||
npm install
|
||||
npm run build
|
||||
cd ..
|
||||
mkdir -p web/public/defaultTheme/dist
|
||||
rm -rf web/public/defaultTheme/dist/*
|
||||
cp -r komari-web/dist/* web/public/defaultTheme/dist/
|
||||
cp -f komari-web/komari-theme.json web/public/defaultTheme/
|
||||
if [ -f komari-web/preview.png ]; then cp -f komari-web/preview.png web/public/defaultTheme/; fi
|
||||
if [ -f komari-web/perview.png ]; then cp -f komari-web/perview.png web/public/defaultTheme/; fi
|
||||
# Compatibility: ensure both filenames exist if only one is present
|
||||
if [ -f web/public/defaultTheme/preview.png ] && [ ! -f web/public/defaultTheme/perview.png ]; then cp -f web/public/defaultTheme/preview.png web/public/defaultTheme/perview.png; fi
|
||||
if [ -f web/public/defaultTheme/perview.png ] && [ ! -f web/public/defaultTheme/preview.png ]; then cp -f web/public/defaultTheme/perview.png web/public/defaultTheme/preview.png; fi
|
||||
- name: Restore frontend artifact
|
||||
uses: ./.github/actions/restore-frontend
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v5
|
||||
@@ -83,12 +81,7 @@ jobs:
|
||||
go-version: "1.23"
|
||||
|
||||
- name: Install Zig cross-compiler toolchain
|
||||
run: |
|
||||
sudo apt-get update # Update apt package lists
|
||||
sudo apt-get install -y build-essential curl
|
||||
wget https://ziglang.org/download/0.14.1/zig-x86_64-linux-0.14.1.tar.xz
|
||||
sudo tar -C /usr/local -Jxf zig-x86_64-linux-0.14.1.tar.xz
|
||||
echo "PATH=$PATH:/usr/local/zig-x86_64-linux-0.14.1" >> $GITHUB_ENV
|
||||
uses: ./.github/actions/setup-zig
|
||||
|
||||
- name: Build binary
|
||||
env:
|
||||
|
||||
@@ -20,21 +20,7 @@ jobs:
|
||||
# echo "<html><body><h1>This is komari development server.</h1></body></html>" > web/public/defaultTheme/dist/index.html
|
||||
|
||||
- name: Clone and build frontend
|
||||
run: |
|
||||
git clone https://github.com/komari-monitor/komari-web komari-web
|
||||
cd komari-web
|
||||
npm install
|
||||
npm run build
|
||||
cd ..
|
||||
mkdir -p web/public/defaultTheme/dist
|
||||
rm -rf web/public/defaultTheme/dist/*
|
||||
cp -r komari-web/dist/* web/public/defaultTheme/dist/
|
||||
cp -f komari-web/komari-theme.json web/public/defaultTheme/
|
||||
if [ -f komari-web/preview.png ]; then cp -f komari-web/preview.png web/public/defaultTheme/; fi
|
||||
if [ -f komari-web/perview.png ]; then cp -f komari-web/perview.png web/public/defaultTheme/; fi
|
||||
# Compatibility: ensure both filenames exist if only one is present
|
||||
if [ -f web/public/defaultTheme/preview.png ] && [ ! -f web/public/defaultTheme/perview.png ]; then cp -f web/public/defaultTheme/preview.png web/public/defaultTheme/perview.png; fi
|
||||
if [ -f web/public/defaultTheme/perview.png ] && [ ! -f web/public/defaultTheme/preview.png ]; then cp -f web/public/defaultTheme/perview.png web/public/defaultTheme/preview.png; fi
|
||||
uses: ./.github/actions/build-frontend
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v5
|
||||
|
||||
@@ -11,37 +11,28 @@ env:
|
||||
IMAGE_NAME: ${{ github.repository }}
|
||||
|
||||
jobs:
|
||||
build-and-push-multi-arch:
|
||||
build-frontend:
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
contents: read
|
||||
packages: write
|
||||
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Setup Node.js
|
||||
uses: actions/setup-node@v4
|
||||
- name: Build frontend artifact
|
||||
uses: ./.github/actions/build-frontend
|
||||
with:
|
||||
node-version: "23"
|
||||
artifact-name: komari-frontend
|
||||
|
||||
- name: Clone and build frontend
|
||||
run: |
|
||||
git clone https://github.com/komari-monitor/komari-web komari-web
|
||||
cd komari-web
|
||||
npm install
|
||||
npm run build
|
||||
cd ..
|
||||
mkdir -p web/public/defaultTheme/dist
|
||||
rm -rf web/public/defaultTheme/dist/*
|
||||
cp -r komari-web/dist/* web/public/defaultTheme/dist/
|
||||
cp -f komari-web/komari-theme.json web/public/defaultTheme/
|
||||
if [ -f komari-web/preview.png ]; then cp -f komari-web/preview.png web/public/defaultTheme/; fi
|
||||
if [ -f komari-web/perview.png ]; then cp -f komari-web/perview.png web/public/defaultTheme/; fi
|
||||
# Compatibility: ensure both filenames exist if only one is present
|
||||
if [ -f web/public/defaultTheme/preview.png ] && [ ! -f web/public/defaultTheme/perview.png ]; then cp -f web/public/defaultTheme/preview.png web/public/defaultTheme/perview.png; fi
|
||||
if [ -f web/public/defaultTheme/perview.png ] && [ ! -f web/public/defaultTheme/preview.png ]; then cp -f web/public/defaultTheme/perview.png web/public/defaultTheme/preview.png; fi
|
||||
build-docker-binaries:
|
||||
needs: build-frontend
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Restore frontend artifact
|
||||
uses: ./.github/actions/restore-frontend
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v5
|
||||
@@ -49,10 +40,7 @@ jobs:
|
||||
go-version: "1.23"
|
||||
|
||||
- name: Install Zig
|
||||
run: |
|
||||
wget https://ziglang.org/download/0.14.1/zig-x86_64-linux-0.14.1.tar.xz
|
||||
sudo tar -C /usr/local -Jxf zig-x86_64-linux-0.14.1.tar.xz
|
||||
echo "PATH=$PATH:/usr/local/zig-x86_64-linux-0.14.1" >> $GITHUB_ENV
|
||||
uses: ./.github/actions/setup-zig
|
||||
|
||||
- name: Build static binaries for multi-arch
|
||||
env:
|
||||
@@ -70,6 +58,29 @@ jobs:
|
||||
echo "Building for linux/arm64..."
|
||||
GOARCH=arm64 CC="zig cc -target aarch64-linux-musl" go build -trimpath -ldflags="$LDFLAGS" -o komari-linux-arm64
|
||||
|
||||
- name: Upload Docker binaries
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: komari-docker-binaries
|
||||
path: komari-linux-*
|
||||
|
||||
build-and-push-multi-arch:
|
||||
needs: build-docker-binaries
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
contents: read
|
||||
packages: write
|
||||
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Download Docker binaries
|
||||
uses: actions/download-artifact@v4
|
||||
with:
|
||||
name: komari-docker-binaries
|
||||
path: .
|
||||
|
||||
- name: Set up QEMU
|
||||
uses: docker/setup-qemu-action@v3
|
||||
|
||||
|
||||
@@ -3,17 +3,16 @@ name: Generate Release Notes
|
||||
on:
|
||||
release:
|
||||
types: [published]
|
||||
repository_dispatch:
|
||||
types:
|
||||
- komari-web-release-published
|
||||
schedule:
|
||||
- cron: "41 * * * *"
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
tag:
|
||||
description: "Release tag to regenerate notes for. Leave empty to use the latest release."
|
||||
required: false
|
||||
type: string
|
||||
base_ref:
|
||||
description: "Base ref or SHA for the release comparison. Snapshot builds pass the previous main SHA."
|
||||
required: false
|
||||
type: string
|
||||
|
||||
permissions:
|
||||
contents: write
|
||||
@@ -21,22 +20,65 @@ permissions:
|
||||
jobs:
|
||||
update-release-notes:
|
||||
runs-on: ubuntu-latest
|
||||
if: ${{ github.event_name != 'release' || !startsWith(github.event.release.tag_name, 'Snapshot-') }}
|
||||
|
||||
steps:
|
||||
- name: Generate and apply release notes
|
||||
uses: actions/github-script@v9
|
||||
env:
|
||||
OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
|
||||
OPENAI_BASE_URL: ${{ secrets.OPENAI_BASE_URL }}
|
||||
OPENAI_MODEL: ${{ vars.OPENAI_MODEL }}
|
||||
with:
|
||||
script: |
|
||||
const { owner, repo } = context.repo;
|
||||
const webRepo = { owner: "komari-monitor", repo: "komari-web" };
|
||||
|
||||
const AI_CONTEXT_CHAR_LIMIT = 120000;
|
||||
const AI_MAX_COMMITS = 160;
|
||||
const AI_MAX_CHANGED_FILES = 120;
|
||||
const AI_MAX_PULL_REQUESTS = 80;
|
||||
const AI_MAX_PATCH_CHARS_PER_FILE = 2200;
|
||||
const OPENAI_TIMEOUT_MS = 120000;
|
||||
|
||||
const clean = (value) => (typeof value === "string" ? value.trim() : "");
|
||||
const isSnapshotTag = (tag) => /^Snapshot-/i.test(clean(tag));
|
||||
const releaseTimestamp = (release) => Date.parse(release.published_at || release.created_at || "");
|
||||
const isStableRelease = (release) => !release.draft && !release.prerelease && !isSnapshotTag(release.tag_name);
|
||||
|
||||
function limitText(value, maxLength) {
|
||||
const text = clean(value);
|
||||
if (text.length <= maxLength) {
|
||||
return text;
|
||||
}
|
||||
|
||||
return `${text.slice(0, maxLength)}\n...[truncated ${text.length - maxLength} chars]`;
|
||||
}
|
||||
|
||||
function appendWithinLimit(parts, value, maxLength) {
|
||||
const text = clean(value);
|
||||
if (!text) {
|
||||
return false;
|
||||
}
|
||||
|
||||
const currentLength = parts.reduce((total, part) => total + part.length + 2, 0);
|
||||
if (currentLength + text.length + 2 <= maxLength) {
|
||||
parts.push(text);
|
||||
return true;
|
||||
}
|
||||
|
||||
const remaining = maxLength - currentLength - 64;
|
||||
if (remaining > 500) {
|
||||
parts.push(`${text.slice(0, remaining)}\n...[context truncated]`);
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
async function resolveRelease() {
|
||||
const requestedTag = clean(context.payload.release?.tag_name)
|
||||
|| clean(context.payload.client_payload?.tag)
|
||||
|| clean(core.getInput("tag"));
|
||||
|| clean(context.payload.inputs?.tag);
|
||||
|
||||
if (requestedTag) {
|
||||
const release = context.payload.release?.tag_name === requestedTag
|
||||
@@ -57,6 +99,8 @@ jobs:
|
||||
});
|
||||
|
||||
const currentTime = releaseTimestamp(release);
|
||||
const currentIsSnapshot = isSnapshotTag(tag);
|
||||
const currentIsStable = isStableRelease(release);
|
||||
const candidates = releases
|
||||
.filter((item) => !item.draft && item.tag_name !== tag)
|
||||
.filter((item) => {
|
||||
@@ -67,11 +111,66 @@ jobs:
|
||||
const itemTime = releaseTimestamp(item);
|
||||
return Number.isFinite(itemTime) && itemTime < currentTime;
|
||||
})
|
||||
.filter((item) => {
|
||||
if (currentIsSnapshot || currentIsStable) {
|
||||
return isStableRelease(item);
|
||||
}
|
||||
|
||||
return item.prerelease && !isSnapshotTag(item.tag_name);
|
||||
})
|
||||
.sort((left, right) => releaseTimestamp(right) - releaseTimestamp(left));
|
||||
|
||||
return candidates[0]?.tag_name || "";
|
||||
}
|
||||
|
||||
async function resolveComparisonBase(tag, release) {
|
||||
const requestedBaseRef = clean(context.payload.client_payload?.base_ref)
|
||||
|| clean(context.payload.inputs?.base_ref);
|
||||
|
||||
if (requestedBaseRef) {
|
||||
return {
|
||||
ref: requestedBaseRef,
|
||||
previousTag: "",
|
||||
source: "requested base ref",
|
||||
};
|
||||
}
|
||||
|
||||
const previousTag = await getPreviousReleaseTag(tag, release);
|
||||
if (previousTag) {
|
||||
return {
|
||||
ref: previousTag,
|
||||
previousTag,
|
||||
source: isSnapshotTag(tag) ? "previous stable release" : "previous release in the same channel",
|
||||
};
|
||||
}
|
||||
|
||||
if (isSnapshotTag(tag)) {
|
||||
try {
|
||||
const commit = (await github.rest.repos.getCommit({
|
||||
owner,
|
||||
repo,
|
||||
ref: release.target_commitish || tag,
|
||||
})).data;
|
||||
const parentSha = commit.parents?.[0]?.sha || "";
|
||||
if (parentSha) {
|
||||
return {
|
||||
ref: parentSha,
|
||||
previousTag: "",
|
||||
source: "snapshot target parent commit",
|
||||
};
|
||||
}
|
||||
} catch (error) {
|
||||
core.warning(`Failed to resolve parent commit for ${tag}: ${error.message}`);
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
ref: "",
|
||||
previousTag: "",
|
||||
source: "none",
|
||||
};
|
||||
}
|
||||
|
||||
async function getGeneratedNotes(tag, release, previousTag) {
|
||||
const params = {
|
||||
owner,
|
||||
@@ -92,21 +191,29 @@ jobs:
|
||||
return clean(generated.data.body);
|
||||
}
|
||||
|
||||
async function getComparedCommits(previousTag, tag) {
|
||||
if (!previousTag) {
|
||||
return [];
|
||||
async function getComparison(baseRef, tag) {
|
||||
if (!baseRef) {
|
||||
return { commits: [], files: [], htmlUrl: "" };
|
||||
}
|
||||
|
||||
const commits = [];
|
||||
let files = [];
|
||||
let htmlUrl = "";
|
||||
|
||||
for (let page = 1; page <= 20; page += 1) {
|
||||
const response = await github.rest.repos.compareCommitsWithBasehead({
|
||||
owner,
|
||||
repo,
|
||||
basehead: `${previousTag}...${tag}`,
|
||||
basehead: `${baseRef}...${tag}`,
|
||||
per_page: 100,
|
||||
page,
|
||||
});
|
||||
|
||||
if (page === 1) {
|
||||
files = response.data.files || [];
|
||||
htmlUrl = response.data.html_url || "";
|
||||
}
|
||||
|
||||
commits.push(...response.data.commits);
|
||||
|
||||
if (!response.headers.link?.includes('rel="next"')) {
|
||||
@@ -114,7 +221,7 @@ jobs:
|
||||
}
|
||||
}
|
||||
|
||||
return commits;
|
||||
return { commits, files, htmlUrl };
|
||||
}
|
||||
|
||||
function renderCommitLine(commit) {
|
||||
@@ -127,17 +234,286 @@ jobs:
|
||||
return `- ${subject} ([${shortSha}](${commit.html_url}))${author}`;
|
||||
}
|
||||
|
||||
async function buildCommitSection(previousTag, tag) {
|
||||
if (!previousTag) {
|
||||
return "## Commits\n\nNo previous release tag was found; commit comparison is unavailable.";
|
||||
function buildWhatsChangedSection(baseRef, tag, comparison) {
|
||||
if (!baseRef) {
|
||||
return "## What's Changed\n\nNo comparison base was found for this release.";
|
||||
}
|
||||
|
||||
const commits = await getComparedCommits(previousTag, tag);
|
||||
if (commits.length === 0) {
|
||||
return `## Commits\n\nNo commits found between \`${previousTag}\` and \`${tag}\`.`;
|
||||
if (comparison.commits.length === 0) {
|
||||
return `## What's Changed\n\nNo commits found between \`${baseRef}\` and \`${tag}\`.`;
|
||||
}
|
||||
|
||||
return ["## Commits", "", ...commits.map(renderCommitLine)].join("\n");
|
||||
const lines = ["## What's Changed", "", ...comparison.commits.map(renderCommitLine)];
|
||||
if (comparison.htmlUrl) {
|
||||
lines.push("", `**Full Changelog**: ${comparison.htmlUrl}`);
|
||||
}
|
||||
|
||||
return lines.join("\n");
|
||||
}
|
||||
|
||||
function appendCommitsToGeneratedNotes(generatedNotes, baseRef, tag, comparison) {
|
||||
const notes = clean(generatedNotes);
|
||||
if (!notes || comparison.commits.length === 0) {
|
||||
return notes || buildWhatsChangedSection(baseRef, tag, comparison);
|
||||
}
|
||||
|
||||
const lines = notes.split("\n");
|
||||
const fullChangelogIndex = lines.findIndex((line) => /^\*\*Full Changelog\*\*:/i.test(line.trim()));
|
||||
const nextTopLevelSectionIndex = lines.findIndex((line, index) => (
|
||||
index > 0
|
||||
&& /^##\s+/.test(line)
|
||||
&& !/^##\s+What's Changed\b/i.test(line)
|
||||
));
|
||||
const insertionIndex = nextTopLevelSectionIndex >= 0
|
||||
? nextTopLevelSectionIndex
|
||||
: (fullChangelogIndex >= 0 ? fullChangelogIndex : lines.length);
|
||||
const commitSection = [
|
||||
"### Commits",
|
||||
"",
|
||||
...comparison.commits.map(renderCommitLine),
|
||||
"",
|
||||
];
|
||||
|
||||
lines.splice(insertionIndex, 0, ...commitSection);
|
||||
return clean(lines.join("\n"));
|
||||
}
|
||||
|
||||
async function getPullRequestContext(commits) {
|
||||
const pullRequests = new Map();
|
||||
|
||||
for (const commit of commits.slice(0, AI_MAX_COMMITS)) {
|
||||
try {
|
||||
const response = await github.rest.repos.listPullRequestsAssociatedWithCommit({
|
||||
owner,
|
||||
repo,
|
||||
commit_sha: commit.sha,
|
||||
});
|
||||
|
||||
for (const pullRequest of response.data || []) {
|
||||
pullRequests.set(pullRequest.html_url, pullRequest);
|
||||
}
|
||||
} catch (error) {
|
||||
core.warning(`Failed to load pull requests for ${commit.sha.slice(0, 7)}: ${error.message}`);
|
||||
}
|
||||
|
||||
if (pullRequests.size >= AI_MAX_PULL_REQUESTS) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
return [...pullRequests.values()].slice(0, AI_MAX_PULL_REQUESTS);
|
||||
}
|
||||
|
||||
function buildChangedFilesContext(files) {
|
||||
if (!files.length) {
|
||||
return "No changed file metadata was returned by GitHub.";
|
||||
}
|
||||
|
||||
return files.slice(0, AI_MAX_CHANGED_FILES).map((file) => {
|
||||
const lines = [
|
||||
`File: ${file.filename}`,
|
||||
`Status: ${file.status}; additions: ${file.additions}; deletions: ${file.deletions}; changes: ${file.changes}`,
|
||||
];
|
||||
|
||||
if (file.patch) {
|
||||
lines.push("Patch excerpt:");
|
||||
lines.push(limitText(file.patch, AI_MAX_PATCH_CHARS_PER_FILE));
|
||||
}
|
||||
|
||||
return lines.join("\n");
|
||||
}).join("\n\n");
|
||||
}
|
||||
|
||||
function buildCommitContext(commits) {
|
||||
if (!commits.length) {
|
||||
return "No commits were returned by GitHub.";
|
||||
}
|
||||
|
||||
return commits.slice(0, AI_MAX_COMMITS).map((commit) => {
|
||||
const shortSha = commit.sha.slice(0, 7);
|
||||
const author = commit.author?.login || commit.commit.author?.name || "unknown";
|
||||
const message = limitText(commit.commit.message, 1800);
|
||||
return [
|
||||
`Commit: ${shortSha}`,
|
||||
`Author: ${author}`,
|
||||
`URL: ${commit.html_url}`,
|
||||
"Message:",
|
||||
message,
|
||||
].join("\n");
|
||||
}).join("\n\n");
|
||||
}
|
||||
|
||||
function buildPullRequestContext(pullRequests) {
|
||||
if (!pullRequests.length) {
|
||||
return "No associated pull requests were found for the compared commits.";
|
||||
}
|
||||
|
||||
return pullRequests.map((pullRequest) => {
|
||||
const labels = (pullRequest.labels || []).map((label) => label.name).filter(Boolean).join(", ") || "none";
|
||||
return [
|
||||
`PR: #${pullRequest.number} ${pullRequest.title}`,
|
||||
`URL: ${pullRequest.html_url}`,
|
||||
`Author: ${pullRequest.user?.login || "unknown"}`,
|
||||
`Labels: ${labels}`,
|
||||
"Body:",
|
||||
limitText(pullRequest.body, 1800) || "(empty)",
|
||||
].join("\n");
|
||||
}).join("\n\n");
|
||||
}
|
||||
|
||||
function buildAiContext({ tag, release, comparisonBase, whatsChangedSection, comparison, pullRequests, webReleaseSection }) {
|
||||
const sections = [];
|
||||
|
||||
appendWithinLimit(sections, [
|
||||
`Repository: ${owner}/${repo}`,
|
||||
`Release tag: ${tag}`,
|
||||
`Release title: ${release.name || tag}`,
|
||||
`Prerelease: ${release.prerelease ? "yes" : "no"}`,
|
||||
`Snapshot tag: ${isSnapshotTag(tag) ? "yes" : "no"}`,
|
||||
`Target commitish: ${release.target_commitish || "(unknown)"}`,
|
||||
`Comparison base: ${comparisonBase.ref || "(none)"}`,
|
||||
`Comparison base source: ${comparisonBase.source}`,
|
||||
`Compare URL: ${comparison.htmlUrl || "(unavailable)"}`,
|
||||
].join("\n"), AI_CONTEXT_CHAR_LIMIT);
|
||||
|
||||
appendWithinLimit(sections, `What's Changed section:\n${whatsChangedSection}`, AI_CONTEXT_CHAR_LIMIT);
|
||||
appendWithinLimit(sections, `Changed files and patch excerpts:\n${buildChangedFilesContext(comparison.files)}`, AI_CONTEXT_CHAR_LIMIT);
|
||||
appendWithinLimit(sections, `Associated pull requests:\n${buildPullRequestContext(pullRequests)}`, AI_CONTEXT_CHAR_LIMIT);
|
||||
appendWithinLimit(sections, `Commit details:\n${buildCommitContext(comparison.commits)}`, AI_CONTEXT_CHAR_LIMIT);
|
||||
|
||||
if (webReleaseSection) {
|
||||
appendWithinLimit(sections, `Komari Web release notes for the same tag:\n${webReleaseSection}`, AI_CONTEXT_CHAR_LIMIT);
|
||||
}
|
||||
|
||||
return sections.join("\n\n---\n\n");
|
||||
}
|
||||
|
||||
function buildChatCompletionsUrl(baseUrl) {
|
||||
const trimmed = clean(baseUrl).replace(/\/+$/, "");
|
||||
if (!trimmed) {
|
||||
return "";
|
||||
}
|
||||
|
||||
if (trimmed.endsWith("/chat/completions")) {
|
||||
return trimmed;
|
||||
}
|
||||
|
||||
if (trimmed.endsWith("/v1")) {
|
||||
return `${trimmed}/chat/completions`;
|
||||
}
|
||||
|
||||
return `${trimmed}/v1/chat/completions`;
|
||||
}
|
||||
|
||||
function normalizeReleaseSummary(markdown) {
|
||||
const summary = clean(markdown);
|
||||
if (!summary) {
|
||||
return "";
|
||||
}
|
||||
|
||||
const withoutAiHeading = clean(summary
|
||||
.replace(/^#{1,6}\s+AI Summary\b\s*/im, "")
|
||||
.replace(/^\*\*AI Summary\*\*\s*/im, ""));
|
||||
const whatsChangedIndex = withoutAiHeading.search(/^##\s+What's Changed\b/im);
|
||||
if (whatsChangedIndex >= 0) {
|
||||
return clean(withoutAiHeading.slice(0, whatsChangedIndex));
|
||||
}
|
||||
|
||||
return withoutAiHeading;
|
||||
}
|
||||
|
||||
async function callOpenAiChatCompletion(url, apiKey, model, messages) {
|
||||
const controller = new AbortController();
|
||||
const timeout = setTimeout(() => controller.abort(), OPENAI_TIMEOUT_MS);
|
||||
|
||||
try {
|
||||
const response = await fetch(url, {
|
||||
method: "POST",
|
||||
headers: {
|
||||
Authorization: `Bearer ${apiKey}`,
|
||||
"Content-Type": "application/json",
|
||||
},
|
||||
body: JSON.stringify({ model, messages, temperature: 0 }),
|
||||
signal: controller.signal,
|
||||
});
|
||||
|
||||
const responseText = await response.text();
|
||||
let responseJson = {};
|
||||
try {
|
||||
responseJson = responseText ? JSON.parse(responseText) : {};
|
||||
} catch {
|
||||
responseJson = {};
|
||||
}
|
||||
|
||||
if (!response.ok) {
|
||||
const message = clean(responseJson.error?.message) || limitText(responseText, 500);
|
||||
throw new Error(`OpenAI request failed with ${response.status}: ${message}`);
|
||||
}
|
||||
|
||||
return clean(responseJson.choices?.[0]?.message?.content);
|
||||
} finally {
|
||||
clearTimeout(timeout);
|
||||
}
|
||||
}
|
||||
|
||||
async function buildReleaseSummarySection({ tag, release, comparisonBase, whatsChangedSection, comparison, webReleaseSection }) {
|
||||
const apiKey = clean(process.env.OPENAI_API_KEY);
|
||||
const baseUrl = clean(process.env.OPENAI_BASE_URL);
|
||||
const model = clean(process.env.OPENAI_MODEL);
|
||||
|
||||
if (!apiKey || !baseUrl || !model) {
|
||||
core.warning("OPENAI_API_KEY, OPENAI_BASE_URL, or OPENAI_MODEL is missing. Skipping generated release summary.");
|
||||
return "";
|
||||
}
|
||||
|
||||
const pullRequests = await getPullRequestContext(comparison.commits);
|
||||
const aiContext = buildAiContext({
|
||||
tag,
|
||||
release,
|
||||
comparisonBase,
|
||||
whatsChangedSection,
|
||||
comparison,
|
||||
pullRequests,
|
||||
webReleaseSection,
|
||||
});
|
||||
|
||||
const messages = [
|
||||
{
|
||||
role: "system",
|
||||
content: [
|
||||
"You write accurate release notes for the Komari server project.",
|
||||
"Use only the supplied repository context.",
|
||||
"Include both Komari server changes and Komari Web frontend changes when frontend release notes are supplied.",
|
||||
"Prioritize user-visible behavior, compatibility or migration notes, operations/deployment changes, security fixes, and important bug fixes.",
|
||||
"Do not invent features or implementation details that are not supported by the context.",
|
||||
"Write concise Markdown in Simplified Chinese.",
|
||||
].join(" "),
|
||||
},
|
||||
{
|
||||
role: "user",
|
||||
content: [
|
||||
`Create the release summary section for release ${tag}.`,
|
||||
comparisonBase.ref
|
||||
? `Summarize only changes in the comparison ${comparisonBase.ref}...${tag}.`
|
||||
: "No comparison base was found, so summarize only the supplied release context.",
|
||||
"Return only Markdown for the summary section that appears before '## What's Changed'.",
|
||||
"Do not use the heading 'AI Summary'. Do not add a 'What's Changed' section.",
|
||||
"Use these top-level headings, in this order: '## 主要更新内容', '## Bug修复', and '## 其他'.",
|
||||
"Omit '## Bug修复' when there are no concrete bug fixes. Keep bullets short and specific.",
|
||||
"",
|
||||
aiContext,
|
||||
].join("\n"),
|
||||
},
|
||||
];
|
||||
|
||||
try {
|
||||
const url = buildChatCompletionsUrl(baseUrl);
|
||||
const summary = await callOpenAiChatCompletion(url, apiKey, model, messages);
|
||||
return normalizeReleaseSummary(summary);
|
||||
} catch (error) {
|
||||
core.warning(`Generated release summary failed: ${error.message}`);
|
||||
return "";
|
||||
}
|
||||
}
|
||||
|
||||
async function buildWebReleaseSection(tag) {
|
||||
@@ -178,17 +554,41 @@ jobs:
|
||||
return;
|
||||
}
|
||||
|
||||
const previousTag = await getPreviousReleaseTag(tag, release);
|
||||
const generatedBody = await getGeneratedNotes(tag, release, previousTag);
|
||||
if (!generatedBody) {
|
||||
core.setFailed(`GitHub returned empty release notes for ${tag}.`);
|
||||
const comparisonBase = await resolveComparisonBase(tag, release);
|
||||
core.info(comparisonBase.ref
|
||||
? `Comparison base for ${tag}: ${comparisonBase.ref} (${comparisonBase.source}).`
|
||||
: `No comparison base found for ${tag}.`);
|
||||
|
||||
const comparison = await getComparison(comparisonBase.ref, tag);
|
||||
const whatsChangedSection = comparisonBase.previousTag
|
||||
? appendCommitsToGeneratedNotes(
|
||||
await getGeneratedNotes(tag, release, comparisonBase.previousTag),
|
||||
comparisonBase.ref,
|
||||
tag,
|
||||
comparison,
|
||||
)
|
||||
: (!comparisonBase.ref && !isSnapshotTag(tag)
|
||||
? await getGeneratedNotes(tag, release, "")
|
||||
: buildWhatsChangedSection(comparisonBase.ref, tag, comparison));
|
||||
|
||||
if (!whatsChangedSection) {
|
||||
core.setFailed(`Failed to build What's Changed section for ${tag}.`);
|
||||
return;
|
||||
}
|
||||
|
||||
const webReleaseSection = await buildWebReleaseSection(tag);
|
||||
const releaseSummarySection = await buildReleaseSummarySection({
|
||||
tag,
|
||||
release,
|
||||
comparisonBase,
|
||||
whatsChangedSection,
|
||||
comparison,
|
||||
webReleaseSection,
|
||||
});
|
||||
|
||||
const body = [
|
||||
generatedBody,
|
||||
await buildCommitSection(previousTag, tag),
|
||||
await buildWebReleaseSection(tag),
|
||||
releaseSummarySection,
|
||||
whatsChangedSection,
|
||||
].filter(Boolean).join("\n\n");
|
||||
|
||||
if (clean(release.body) === clean(body)) {
|
||||
|
||||
@@ -0,0 +1,159 @@
|
||||
name: Rebuild Existing Release
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
release_tag:
|
||||
description: Existing GitHub Release tag whose assets and version image will be replaced.
|
||||
required: true
|
||||
type: string
|
||||
server_ref:
|
||||
description: Server commit, branch, or tag to build.
|
||||
required: true
|
||||
type: string
|
||||
frontend_ref:
|
||||
description: komari-web commit, branch, or tag to build.
|
||||
required: true
|
||||
type: string
|
||||
|
||||
permissions:
|
||||
contents: write
|
||||
packages: write
|
||||
|
||||
env:
|
||||
REGISTRY: ghcr.io
|
||||
IMAGE_NAME: ${{ github.repository }}
|
||||
|
||||
jobs:
|
||||
build-frontend:
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
steps:
|
||||
- name: Checkout workflow source
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Build selected frontend artifact
|
||||
uses: ./.github/actions/build-frontend
|
||||
with:
|
||||
artifact-name: komari-frontend
|
||||
ref: ${{ inputs.frontend_ref }}
|
||||
|
||||
build-and-attach-binaries:
|
||||
needs: build-frontend
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
strategy:
|
||||
matrix:
|
||||
include:
|
||||
- goos: windows
|
||||
goarch: amd64
|
||||
zig_target_triple: x86_64-windows-gnu
|
||||
binary_extension: .exe
|
||||
- goos: windows
|
||||
goarch: arm64
|
||||
zig_target_triple: aarch64-windows-gnu
|
||||
binary_extension: .exe
|
||||
- goos: windows
|
||||
goarch: 386
|
||||
zig_target_triple: x86-windows-gnu
|
||||
binary_extension: .exe
|
||||
- goos: linux
|
||||
goarch: amd64
|
||||
zig_target_triple: x86_64-linux-musl
|
||||
binary_extension: ""
|
||||
- goos: linux
|
||||
goarch: arm64
|
||||
zig_target_triple: aarch64-linux-musl
|
||||
binary_extension: ""
|
||||
- goos: linux
|
||||
goarch: 386
|
||||
zig_target_triple: x86-linux-musl
|
||||
binary_extension: ""
|
||||
- goos: linux
|
||||
goarch: riscv64
|
||||
zig_target_triple: riscv64-linux-musl
|
||||
binary_extension: ""
|
||||
|
||||
steps:
|
||||
- name: Checkout server source
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
ref: ${{ inputs.server_ref }}
|
||||
|
||||
- name: Restore frontend artifact
|
||||
uses: komari-monitor/komari/.github/actions/restore-frontend@main
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v5
|
||||
with:
|
||||
go-version: "1.23"
|
||||
|
||||
- name: Install Zig cross-compiler toolchain
|
||||
uses: komari-monitor/komari/.github/actions/setup-zig@main
|
||||
|
||||
- name: Build binary
|
||||
env:
|
||||
GOOS: ${{ matrix.goos }}
|
||||
GOARCH: ${{ matrix.goarch }}
|
||||
CGO_ENABLED: 1
|
||||
CC: zig cc -target ${{ matrix.zig_target_triple }}
|
||||
GIN_MODE: release
|
||||
RELEASE_TAG: ${{ inputs.release_tag }}
|
||||
run: |
|
||||
BINARY_NAME=komari-${{ matrix.goos }}-${{ matrix.goarch }}${{ matrix.binary_extension }}
|
||||
VERSION_HASH="$(git rev-parse HEAD)"
|
||||
go build -trimpath -ldflags="-s -w -X github.com/komari-monitor/komari/utils.CurrentVersion=${RELEASE_TAG} -X github.com/komari-monitor/komari/utils.VersionHash=${VERSION_HASH}" -o "$BINARY_NAME"
|
||||
|
||||
- name: Upload binary to release
|
||||
env:
|
||||
GH_TOKEN: ${{ github.token }}
|
||||
RELEASE_TAG: ${{ inputs.release_tag }}
|
||||
run: |
|
||||
BINARY_NAME=komari-${{ matrix.goos }}-${{ matrix.goarch }}${{ matrix.binary_extension }}
|
||||
gh release upload "$RELEASE_TAG" "$BINARY_NAME" --clobber --repo "$GITHUB_REPOSITORY"
|
||||
|
||||
- name: Upload binary as artifact
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: komari-${{ matrix.goos }}-${{ matrix.goarch }}
|
||||
path: komari-${{ matrix.goos }}-${{ matrix.goarch }}${{ matrix.binary_extension }}
|
||||
|
||||
build-and-push-docker:
|
||||
needs: build-and-attach-binaries
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
steps:
|
||||
- name: Checkout server source
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
ref: ${{ inputs.server_ref }}
|
||||
|
||||
- name: Download Docker binaries
|
||||
uses: actions/download-artifact@v4
|
||||
with:
|
||||
pattern: komari-linux-*
|
||||
path: .
|
||||
merge-multiple: true
|
||||
|
||||
- name: Set up QEMU
|
||||
uses: docker/setup-qemu-action@v3
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@v3
|
||||
|
||||
- name: Log in to the Container registry
|
||||
uses: docker/login-action@v3
|
||||
with:
|
||||
registry: ${{ env.REGISTRY }}
|
||||
username: ${{ github.actor }}
|
||||
password: ${{ github.token }}
|
||||
|
||||
- name: Build and push version Docker image
|
||||
uses: docker/build-push-action@v5
|
||||
with:
|
||||
context: .
|
||||
push: true
|
||||
platforms: linux/amd64,linux/arm64
|
||||
tags: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ inputs.release_tag }}
|
||||
no-cache: true
|
||||
provenance: false
|
||||
@@ -1,9 +1,12 @@
|
||||
name: Publish Docker Image
|
||||
name: Publish Docker Image Manually
|
||||
|
||||
on:
|
||||
release:
|
||||
types: [published]
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
version:
|
||||
description: Docker image tag and embedded version.
|
||||
required: true
|
||||
type: string
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
@@ -14,7 +17,61 @@ env:
|
||||
IMAGE_NAME: ${{ github.repository }}
|
||||
|
||||
jobs:
|
||||
build-frontend:
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Build frontend artifact
|
||||
uses: ./.github/actions/build-frontend
|
||||
with:
|
||||
artifact-name: komari-frontend
|
||||
|
||||
build-docker-binaries:
|
||||
needs: build-frontend
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Restore frontend artifact
|
||||
uses: ./.github/actions/restore-frontend
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v5
|
||||
with:
|
||||
go-version: "1.23"
|
||||
|
||||
- name: Install Zig
|
||||
uses: ./.github/actions/setup-zig
|
||||
|
||||
- name: Build static binaries for multi-arch
|
||||
env:
|
||||
GOOS: linux
|
||||
CGO_ENABLED: 1
|
||||
GIN_MODE: release
|
||||
VERSION: ${{ github.event.inputs.version }}
|
||||
run: |
|
||||
VERSION_HASH="${{ github.sha }}"
|
||||
LDFLAGS="-s -w -X github.com/komari-monitor/komari/utils.CurrentVersion=${VERSION} -X github.com/komari-monitor/komari/utils.VersionHash=${VERSION_HASH}"
|
||||
|
||||
echo "Building for linux/amd64..."
|
||||
GOARCH=amd64 CC="zig cc -target x86_64-linux-musl" go build -trimpath -ldflags="$LDFLAGS" -o komari-linux-amd64
|
||||
|
||||
echo "Building for linux/arm64..."
|
||||
GOARCH=arm64 CC="zig cc -target aarch64-linux-musl" go build -trimpath -ldflags="$LDFLAGS" -o komari-linux-arm64
|
||||
|
||||
- name: Upload Docker binaries
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: komari-docker-binaries
|
||||
path: komari-linux-*
|
||||
|
||||
build-and-push-multi-arch:
|
||||
needs: build-docker-binaries
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
contents: read
|
||||
@@ -24,54 +81,11 @@ jobs:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Setup Node.js
|
||||
uses: actions/setup-node@v4
|
||||
- name: Download Docker binaries
|
||||
uses: actions/download-artifact@v4
|
||||
with:
|
||||
node-version: "23"
|
||||
|
||||
- name: Clone and build frontend
|
||||
run: |
|
||||
git clone https://github.com/komari-monitor/komari-web komari-web
|
||||
cd komari-web
|
||||
npm install
|
||||
npm run build
|
||||
cd ..
|
||||
mkdir -p web/public/defaultTheme/dist
|
||||
rm -rf web/public/defaultTheme/dist/*
|
||||
cp -r komari-web/dist/* web/public/defaultTheme/dist/
|
||||
cp -f komari-web/komari-theme.json web/public/defaultTheme/
|
||||
if [ -f komari-web/preview.png ]; then cp -f komari-web/preview.png web/public/defaultTheme/; fi
|
||||
if [ -f komari-web/perview.png ]; then cp -f komari-web/perview.png web/public/defaultTheme/; fi
|
||||
# Compatibility: ensure both filenames exist if only one is present
|
||||
if [ -f web/public/defaultTheme/preview.png ] && [ ! -f web/public/defaultTheme/perview.png ]; then cp -f web/public/defaultTheme/preview.png web/public/defaultTheme/perview.png; fi
|
||||
if [ -f web/public/defaultTheme/perview.png ] && [ ! -f web/public/defaultTheme/preview.png ]; then cp -f web/public/defaultTheme/perview.png web/public/defaultTheme/preview.png; fi
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v5
|
||||
with:
|
||||
go-version: "1.23"
|
||||
|
||||
- name: Install Zig
|
||||
run: |
|
||||
wget https://ziglang.org/download/0.14.1/zig-x86_64-linux-0.14.1.tar.xz
|
||||
sudo tar -C /usr/local -Jxf zig-x86_64-linux-0.14.1.tar.xz
|
||||
echo "PATH=$PATH:/usr/local/zig-x86_64-linux-0.14.1" >> $GITHUB_ENV
|
||||
|
||||
- name: Build static binaries for multi-arch
|
||||
env:
|
||||
GOOS: linux
|
||||
CGO_ENABLED: 1
|
||||
GIN_MODE: release
|
||||
run: |
|
||||
VERSION="${{ github.event.release.tag_name }}"
|
||||
VERSION_HASH="${{ github.sha }}"
|
||||
LDFLAGS="-s -w -X github.com/komari-monitor/komari/utils.CurrentVersion=${VERSION} -X github.com/komari-monitor/komari/utils.VersionHash=${VERSION_HASH}"
|
||||
|
||||
echo "Building for linux/amd64..."
|
||||
GOARCH=amd64 CC="zig cc -target x86_64-linux-musl" go build -trimpath -ldflags="$LDFLAGS" -o komari-linux-amd64
|
||||
|
||||
echo "Building for linux/arm64..."
|
||||
GOARCH=arm64 CC="zig cc -target aarch64-linux-musl" go build -trimpath -ldflags="$LDFLAGS" -o komari-linux-arm64
|
||||
name: komari-docker-binaries
|
||||
path: .
|
||||
|
||||
- name: Set up QEMU
|
||||
uses: docker/setup-qemu-action@v3
|
||||
@@ -92,8 +106,7 @@ jobs:
|
||||
with:
|
||||
images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
|
||||
tags: |
|
||||
type=raw,value=${{ github.event.release.tag_name }}
|
||||
type=raw,value=latest
|
||||
type=raw,value=${{ github.event.inputs.version }}
|
||||
|
||||
- name: Build and push Docker image
|
||||
uses: docker/build-push-action@v5
|
||||
|
||||
+114
-27
@@ -13,7 +13,38 @@ env:
|
||||
IMAGE_NAME: ${{ github.repository }}
|
||||
|
||||
jobs:
|
||||
resolve-release:
|
||||
runs-on: ubuntu-latest
|
||||
outputs:
|
||||
frontend_ref: ${{ steps.release.outputs.frontend_ref }}
|
||||
|
||||
steps:
|
||||
- id: release
|
||||
env:
|
||||
TAG: ${{ github.event.release.tag_name }}
|
||||
run: |
|
||||
frontend_ref=""
|
||||
if [[ "$TAG" =~ ^(.+)-fix[0-9]+$ ]]; then
|
||||
frontend_ref="${BASH_REMATCH[1]}"
|
||||
fi
|
||||
echo "frontend_ref=$frontend_ref" >> "$GITHUB_OUTPUT"
|
||||
|
||||
build-frontend:
|
||||
needs: resolve-release
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Build frontend artifact
|
||||
uses: ./.github/actions/build-frontend
|
||||
with:
|
||||
artifact-name: komari-frontend
|
||||
ref: ${{ needs.resolve-release.outputs.frontend_ref }}
|
||||
|
||||
build-and-attach-binaries:
|
||||
needs: build-frontend
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
strategy:
|
||||
@@ -52,6 +83,10 @@ jobs:
|
||||
goarch: riscv64
|
||||
zig_target_triple: riscv64-linux-musl
|
||||
binary_extension: ""
|
||||
- goos: linux
|
||||
goarch: loong64
|
||||
zig_target_triple: loongarch64-linux-musl
|
||||
binary_extension: ""
|
||||
|
||||
exclude:
|
||||
- goos: windows
|
||||
@@ -61,27 +96,8 @@ jobs:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Setup Node.js
|
||||
uses: actions/setup-node@v4
|
||||
with:
|
||||
node-version: "23"
|
||||
|
||||
- name: Clone and build frontend
|
||||
run: |
|
||||
git clone https://github.com/komari-monitor/komari-web komari-web
|
||||
cd komari-web
|
||||
npm install
|
||||
npm run build
|
||||
cd ..
|
||||
mkdir -p web/public/defaultTheme/dist
|
||||
rm -rf web/public/defaultTheme/dist/*
|
||||
cp -r komari-web/dist/* web/public/defaultTheme/dist/
|
||||
cp -f komari-web/komari-theme.json web/public/defaultTheme/
|
||||
if [ -f komari-web/preview.png ]; then cp -f komari-web/preview.png web/public/defaultTheme/; fi
|
||||
if [ -f komari-web/perview.png ]; then cp -f komari-web/perview.png web/public/defaultTheme/; fi
|
||||
# Compatibility: ensure both filenames exist if only one is present
|
||||
if [ -f web/public/defaultTheme/preview.png ] && [ ! -f web/public/defaultTheme/perview.png ]; then cp -f web/public/defaultTheme/preview.png web/public/defaultTheme/perview.png; fi
|
||||
if [ -f web/public/defaultTheme/perview.png ] && [ ! -f web/public/defaultTheme/preview.png ]; then cp -f web/public/defaultTheme/perview.png web/public/defaultTheme/preview.png; fi
|
||||
- name: Restore frontend artifact
|
||||
uses: ./.github/actions/restore-frontend
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v5
|
||||
@@ -89,12 +105,7 @@ jobs:
|
||||
go-version: "1.23"
|
||||
|
||||
- name: Install Zig cross-compiler toolchain
|
||||
run: |
|
||||
sudo apt-get update # Update apt package lists
|
||||
sudo apt-get install -y build-essential curl
|
||||
wget https://ziglang.org/download/0.14.1/zig-x86_64-linux-0.14.1.tar.xz
|
||||
sudo tar -C /usr/local -Jxf zig-x86_64-linux-0.14.1.tar.xz
|
||||
echo "PATH=$PATH:/usr/local/zig-x86_64-linux-0.14.1" >> $GITHUB_ENV
|
||||
uses: ./.github/actions/setup-zig
|
||||
|
||||
- name: Build binary
|
||||
env:
|
||||
@@ -112,6 +123,12 @@ jobs:
|
||||
VERSION_HASH="${{ github.sha }}" # Use commit SHA for hash
|
||||
go build -trimpath -ldflags="-s -w -X github.com/komari-monitor/komari/utils.CurrentVersion=${VERSION} -X github.com/komari-monitor/komari/utils.VersionHash=${VERSION_HASH}" -o $BINARY_NAME
|
||||
|
||||
- name: Upload binary as artifact
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: komari-${{ matrix.goos }}-${{ matrix.goarch }}
|
||||
path: komari-${{ matrix.goos }}-${{ matrix.goarch }}*
|
||||
|
||||
- name: Upload binary to release
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
@@ -124,3 +141,73 @@ jobs:
|
||||
# Upload the binary as a release asset using GitHub CLI
|
||||
gh release upload ${{ github.event.release.tag_name }} $BINARY_NAME --repo ${{ github.repository }}
|
||||
shell: bash
|
||||
|
||||
build-and-push-docker:
|
||||
needs: build-and-attach-binaries
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
contents: read
|
||||
packages: write
|
||||
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Download Docker binaries
|
||||
uses: actions/download-artifact@v4
|
||||
with:
|
||||
pattern: komari-linux-*
|
||||
path: .
|
||||
merge-multiple: true
|
||||
|
||||
- name: Set up QEMU
|
||||
uses: docker/setup-qemu-action@v3
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@v3
|
||||
|
||||
- name: Log in to the Container registry
|
||||
uses: docker/login-action@v3
|
||||
with:
|
||||
registry: ${{ env.REGISTRY }}
|
||||
username: ${{ github.actor }}
|
||||
password: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
- name: Check whether this is the latest GitHub release
|
||||
id: release
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
TAG: ${{ github.event.release.tag_name }}
|
||||
run: |
|
||||
is_latest="$(gh api graphql \
|
||||
-f 'query=query($owner: String!, $name: String!, $tag: String!) { repository(owner: $owner, name: $name) { release(tagName: $tag) { isLatest } } }' \
|
||||
-F owner="${{ github.repository_owner }}" \
|
||||
-F name="${{ github.event.repository.name }}" \
|
||||
-F tag="$TAG" \
|
||||
--jq '.data.repository.release.isLatest')"
|
||||
echo "is_latest=$is_latest" >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Extract metadata
|
||||
id: meta
|
||||
uses: docker/metadata-action@v5
|
||||
with:
|
||||
images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
|
||||
tags: |
|
||||
type=raw,value=${{ github.event.release.tag_name }}
|
||||
|
||||
- name: Build and push Docker image
|
||||
uses: docker/build-push-action@v5
|
||||
with:
|
||||
context: .
|
||||
push: true
|
||||
platforms: linux/amd64,linux/arm64
|
||||
tags: ${{ steps.meta.outputs.tags }}
|
||||
labels: ${{ steps.meta.outputs.labels }}
|
||||
no-cache: true
|
||||
provenance: false
|
||||
|
||||
- name: Promote Docker latest tag
|
||||
if: steps.release.outputs.is_latest == 'true'
|
||||
env:
|
||||
TAG: ${{ github.event.release.tag_name }}
|
||||
run: docker buildx imagetools create --tag "${REGISTRY}/${IMAGE_NAME}:latest" "${REGISTRY}/${IMAGE_NAME}:${TAG}"
|
||||
+103
-79
@@ -14,8 +14,38 @@ env:
|
||||
REGISTRY: ghcr.io
|
||||
IMAGE_NAME: ${{ github.repository }}
|
||||
|
||||
concurrency:
|
||||
group: snapshot-${{ github.ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
jobs:
|
||||
validate-ref:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
fetch-depth: 1
|
||||
|
||||
- name: Ensure snapshot target is main
|
||||
run: |
|
||||
if [ "${GITHUB_REF}" != "refs/heads/main" ]; then
|
||||
echo "Snapshot builds must run from refs/heads/main. Current ref: ${GITHUB_REF}"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
git fetch --no-tags --depth=1 origin main
|
||||
MAIN_SHA="$(git rev-parse FETCH_HEAD)"
|
||||
|
||||
if [ "${GITHUB_SHA}" != "${MAIN_SHA}" ]; then
|
||||
echo "Snapshot builds must use the current main commit."
|
||||
echo "Workflow SHA: ${GITHUB_SHA}"
|
||||
echo "Current main: ${MAIN_SHA}"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
prepare:
|
||||
needs: validate-ref
|
||||
runs-on: ubuntu-latest
|
||||
outputs:
|
||||
version: ${{ steps.version.outputs.version }}
|
||||
@@ -23,13 +53,26 @@ jobs:
|
||||
- name: Generate snapshot version
|
||||
id: version
|
||||
run: |
|
||||
# Snapshot 版本号格式: Snapshot-yymmddhhMM (UTC)
|
||||
VERSION="Snapshot-$(date -u +%y%m%d%H%M)"
|
||||
# Snapshot 版本号格式: Snapshot-yymmddhhMM (UTC+8)
|
||||
VERSION="Snapshot-$(date -u -d '+8 hours' +%y%m%d%H%M)"
|
||||
echo "version=${VERSION}" >> "$GITHUB_OUTPUT"
|
||||
echo "Snapshot version: ${VERSION}"
|
||||
|
||||
build-frontend:
|
||||
needs: validate-ref
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Build frontend artifact
|
||||
uses: ./.github/actions/build-frontend
|
||||
with:
|
||||
artifact-name: komari-frontend
|
||||
|
||||
build-and-attach-binaries:
|
||||
needs: prepare
|
||||
needs: [prepare, build-frontend]
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
strategy:
|
||||
@@ -69,6 +112,10 @@ jobs:
|
||||
goarch: riscv64
|
||||
zig_target_triple: riscv64-linux-musl
|
||||
binary_extension: ""
|
||||
- goos: linux
|
||||
goarch: loong64
|
||||
zig_target_triple: loongarch64-linux-musl
|
||||
binary_extension: ""
|
||||
|
||||
exclude:
|
||||
- goos: windows
|
||||
@@ -78,27 +125,8 @@ jobs:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Setup Node.js
|
||||
uses: actions/setup-node@v4
|
||||
with:
|
||||
node-version: "23"
|
||||
|
||||
- name: Clone and build frontend
|
||||
run: |
|
||||
git clone https://github.com/komari-monitor/komari-web komari-web
|
||||
cd komari-web
|
||||
npm install
|
||||
npm run build
|
||||
cd ..
|
||||
mkdir -p web/public/defaultTheme/dist
|
||||
rm -rf web/public/defaultTheme/dist/*
|
||||
cp -r komari-web/dist/* web/public/defaultTheme/dist/
|
||||
cp -f komari-web/komari-theme.json web/public/defaultTheme/
|
||||
if [ -f komari-web/preview.png ]; then cp -f komari-web/preview.png web/public/defaultTheme/; fi
|
||||
if [ -f komari-web/perview.png ]; then cp -f komari-web/perview.png web/public/defaultTheme/; fi
|
||||
# Compatibility: ensure both filenames exist if only one is present
|
||||
if [ -f web/public/defaultTheme/preview.png ] && [ ! -f web/public/defaultTheme/perview.png ]; then cp -f web/public/defaultTheme/preview.png web/public/defaultTheme/perview.png; fi
|
||||
if [ -f web/public/defaultTheme/perview.png ] && [ ! -f web/public/defaultTheme/preview.png ]; then cp -f web/public/defaultTheme/perview.png web/public/defaultTheme/preview.png; fi
|
||||
- name: Restore frontend artifact
|
||||
uses: ./.github/actions/restore-frontend
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v5
|
||||
@@ -106,12 +134,7 @@ jobs:
|
||||
go-version: "1.23"
|
||||
|
||||
- name: Install Zig cross-compiler toolchain
|
||||
run: |
|
||||
sudo apt-get update # Update apt package lists
|
||||
sudo apt-get install -y build-essential curl
|
||||
wget https://ziglang.org/download/0.14.1/zig-x86_64-linux-0.14.1.tar.xz
|
||||
sudo tar -C /usr/local -Jxf zig-x86_64-linux-0.14.1.tar.xz
|
||||
echo "PATH=$PATH:/usr/local/zig-x86_64-linux-0.14.1" >> $GITHUB_ENV
|
||||
uses: ./.github/actions/setup-zig
|
||||
|
||||
- name: Build binary
|
||||
env:
|
||||
@@ -139,11 +162,24 @@ jobs:
|
||||
needs: [prepare, build-and-attach-binaries]
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
actions: write
|
||||
contents: write
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Ensure commit is still current main
|
||||
run: |
|
||||
git fetch --no-tags --depth=1 origin main
|
||||
MAIN_SHA="$(git rev-parse FETCH_HEAD)"
|
||||
|
||||
if [ "${GITHUB_SHA}" != "${MAIN_SHA}" ]; then
|
||||
echo "Snapshot publish skipped because a newer main commit exists."
|
||||
echo "Workflow SHA: ${GITHUB_SHA}"
|
||||
echo "Current main: ${MAIN_SHA}"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
- name: Download all binaries
|
||||
uses: actions/download-artifact@v4
|
||||
with:
|
||||
@@ -173,10 +209,29 @@ jobs:
|
||||
--notes "Automated snapshot build (${VERSION}) from commit ${{ github.sha }}." \
|
||||
--prerelease \
|
||||
artifacts/komari-*
|
||||
|
||||
gh workflow run generate-release-notes.yml \
|
||||
--repo "${{ github.repository }}" \
|
||||
--ref main \
|
||||
-f tag="${VERSION}"
|
||||
|
||||
gh release list \
|
||||
--repo "${{ github.repository }}" \
|
||||
--limit 1000 \
|
||||
--json tagName,isPrerelease \
|
||||
--jq '.[] | select(.isPrerelease and (.tagName | startswith("Snapshot-"))) | .tagName' |
|
||||
while IFS= read -r tag; do
|
||||
if [ -z "${tag}" ] || [ "${tag}" = "${VERSION}" ]; then
|
||||
continue
|
||||
fi
|
||||
|
||||
echo "Deleting old snapshot release ${tag}..."
|
||||
gh release delete "${tag}" --repo "${{ github.repository }}" --cleanup-tag --yes
|
||||
done
|
||||
shell: bash
|
||||
|
||||
build-and-push-docker:
|
||||
needs: prepare
|
||||
needs: [prepare, build-and-attach-binaries, publish-snapshot-release]
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
contents: read
|
||||
@@ -186,54 +241,24 @@ jobs:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Setup Node.js
|
||||
uses: actions/setup-node@v4
|
||||
- name: Ensure commit is still current main
|
||||
run: |
|
||||
git fetch --no-tags --depth=1 origin main
|
||||
MAIN_SHA="$(git rev-parse FETCH_HEAD)"
|
||||
|
||||
if [ "${GITHUB_SHA}" != "${MAIN_SHA}" ]; then
|
||||
echo "Snapshot Docker publish skipped because a newer main commit exists."
|
||||
echo "Workflow SHA: ${GITHUB_SHA}"
|
||||
echo "Current main: ${MAIN_SHA}"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
- name: Download Docker binaries
|
||||
uses: actions/download-artifact@v4
|
||||
with:
|
||||
node-version: "23"
|
||||
|
||||
- name: Clone and build frontend
|
||||
run: |
|
||||
git clone https://github.com/komari-monitor/komari-web komari-web
|
||||
cd komari-web
|
||||
npm install
|
||||
npm run build
|
||||
cd ..
|
||||
mkdir -p web/public/defaultTheme/dist
|
||||
rm -rf web/public/defaultTheme/dist/*
|
||||
cp -r komari-web/dist/* web/public/defaultTheme/dist/
|
||||
cp -f komari-web/komari-theme.json web/public/defaultTheme/
|
||||
if [ -f komari-web/preview.png ]; then cp -f komari-web/preview.png web/public/defaultTheme/; fi
|
||||
if [ -f komari-web/perview.png ]; then cp -f komari-web/perview.png web/public/defaultTheme/; fi
|
||||
# Compatibility: ensure both filenames exist if only one is present
|
||||
if [ -f web/public/defaultTheme/preview.png ] && [ ! -f web/public/defaultTheme/perview.png ]; then cp -f web/public/defaultTheme/preview.png web/public/defaultTheme/perview.png; fi
|
||||
if [ -f web/public/defaultTheme/perview.png ] && [ ! -f web/public/defaultTheme/preview.png ]; then cp -f web/public/defaultTheme/perview.png web/public/defaultTheme/preview.png; fi
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v5
|
||||
with:
|
||||
go-version: "1.23"
|
||||
|
||||
- name: Install Zig
|
||||
run: |
|
||||
wget https://ziglang.org/download/0.14.1/zig-x86_64-linux-0.14.1.tar.xz
|
||||
sudo tar -C /usr/local -Jxf zig-x86_64-linux-0.14.1.tar.xz
|
||||
echo "PATH=$PATH:/usr/local/zig-x86_64-linux-0.14.1" >> $GITHUB_ENV
|
||||
|
||||
- name: Build static binaries for multi-arch
|
||||
env:
|
||||
GOOS: linux
|
||||
CGO_ENABLED: 1
|
||||
GIN_MODE: release
|
||||
run: |
|
||||
VERSION="${{ needs.prepare.outputs.version }}"
|
||||
VERSION_HASH="${{ github.sha }}"
|
||||
LDFLAGS="-s -w -X github.com/komari-monitor/komari/utils.CurrentVersion=${VERSION} -X github.com/komari-monitor/komari/utils.VersionHash=${VERSION_HASH}"
|
||||
|
||||
echo "Building for linux/amd64..."
|
||||
GOARCH=amd64 CC="zig cc -target x86_64-linux-musl" go build -trimpath -ldflags="$LDFLAGS" -o komari-linux-amd64
|
||||
|
||||
echo "Building for linux/arm64..."
|
||||
GOARCH=arm64 CC="zig cc -target aarch64-linux-musl" go build -trimpath -ldflags="$LDFLAGS" -o komari-linux-arm64
|
||||
pattern: komari-linux-*
|
||||
path: .
|
||||
merge-multiple: true
|
||||
|
||||
- name: Set up QEMU
|
||||
uses: docker/setup-qemu-action@v3
|
||||
@@ -255,7 +280,6 @@ jobs:
|
||||
images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
|
||||
tags: |
|
||||
type=raw,value=snapshot
|
||||
type=raw,value=${{ needs.prepare.outputs.version }}
|
||||
|
||||
- name: Build and push Docker image
|
||||
uses: docker/build-push-action@v5
|
||||
|
||||
+1
-18
@@ -8,29 +8,12 @@ ARG TARGETARCH
|
||||
|
||||
RUN apk add --no-cache ca-certificates curl tzdata
|
||||
|
||||
RUN set -eux; \
|
||||
case "${TARGETARCH}" in \
|
||||
amd64) cloudflared_arch="amd64" ;; \
|
||||
386) cloudflared_arch="386" ;; \
|
||||
arm64) cloudflared_arch="arm64" ;; \
|
||||
arm) cloudflared_arch="arm" ;; \
|
||||
*) echo "Unsupported TARGETARCH: ${TARGETARCH}" >&2; exit 1 ;; \
|
||||
esac; \
|
||||
curl -fsSL "https://github.com/cloudflare/cloudflared/releases/latest/download/cloudflared-linux-${cloudflared_arch}" -o /usr/local/bin/cloudflared; \
|
||||
chmod +x /usr/local/bin/cloudflared
|
||||
|
||||
COPY komari-${TARGETOS}-${TARGETARCH} /app/komari
|
||||
|
||||
|
||||
RUN chmod +x /app/komari
|
||||
|
||||
ENV GIN_MODE=release
|
||||
ENV KOMARI_DB_TYPE=sqlite
|
||||
ENV KOMARI_DB_FILE=/app/data/komari.db
|
||||
ENV KOMARI_DB_HOST=localhost
|
||||
ENV KOMARI_DB_PORT=3306
|
||||
ENV KOMARI_DB_USER=root
|
||||
ENV KOMARI_DB_PASS=
|
||||
ENV KOMARI_DB_NAME=komari
|
||||
ENV KOMARI_LISTEN=0.0.0.0:25774
|
||||
|
||||
EXPOSE 25774
|
||||
|
||||
@@ -5,7 +5,7 @@
|
||||
|
||||

|
||||
|
||||
[简体中文](./docs/README_zh.md) | [繁體中文](./docs/README_zh-TW.md) | [日本語](./docs/README_ja.md)
|
||||
[简体中文](./README_zh-cn.md)
|
||||
|
||||
Komari is a lightweight, self-hosted server monitoring tool designed to provide a simple and efficient solution for monitoring server performance. It supports viewing server status through a web interface and collects data through a lightweight agent.
|
||||
|
||||
@@ -14,7 +14,7 @@ Komari is a lightweight, self-hosted server monitoring tool designed to provide
|
||||
> Users are solely responsible for how they deploy and operate Komari. The developers do not accept responsibility for unauthorized or abusive use, or for any resulting consequences.
|
||||
> On Windows, when remote control is enabled, the client displays a Windows notification at each user login to remind the user that Komari is remote control software.
|
||||
|
||||
[Documentation](https://komari-document.pages.dev/) | [文档(镜像站 By Geekertao)](https://www.komari.wiki) | [Telegram Group](https://t.me/komari_monitor)
|
||||
[文档/Documentation](https://komari-document.pages.dev/) | [文档(镜像站 By Geekertao)](https://www.komari.wiki) | [Telegram Group](https://t.me/komari_monitor)
|
||||
|
||||
## Features
|
||||
|
||||
@@ -22,9 +22,7 @@ Komari is a lightweight, self-hosted server monitoring tool designed to provide
|
||||
- **Self-hosted**: Complete control over data privacy, easy to deploy.
|
||||
- **Web Interface**: Intuitive monitoring dashboard, easy to use.
|
||||
|
||||
## Quick Start
|
||||
|
||||
### 0. One-click Deployment with Cloud Hosting
|
||||
## One-click Deployment
|
||||
|
||||
- Rainyun - CNY 4.5/month
|
||||
|
||||
@@ -34,87 +32,7 @@ Komari is a lightweight, self-hosted server monitoring tool designed to provide
|
||||
|
||||
Available on 1Panel App Store. Install via **App Store > Utilities > Komari**.
|
||||
|
||||
### 1. Use the One-click Install Script
|
||||
|
||||
Suitable for distributions using systemd (Ubuntu, Debian...).
|
||||
|
||||
```bash
|
||||
curl -fsSL https://raw.githubusercontent.com/komari-monitor/komari/main/install-komari.sh -o install-komari.sh
|
||||
chmod +x install-komari.sh
|
||||
sudo ./install-komari.sh
|
||||
```
|
||||
|
||||
### 2. Docker Deployment
|
||||
|
||||
1. Create a data directory:
|
||||
```bash
|
||||
mkdir -p ./data
|
||||
```
|
||||
2. Run the Docker container:
|
||||
```bash
|
||||
docker run -d \
|
||||
-p 25774:25774 \
|
||||
-v $(pwd)/data:/app/data \
|
||||
--name komari \
|
||||
ghcr.io/komari-monitor/komari:latest
|
||||
```
|
||||
3. View the default username and password:
|
||||
```bash
|
||||
docker logs komari
|
||||
```
|
||||
4. Access `http://<your_server_ip>:25774` in your browser.
|
||||
|
||||
> [!NOTE]
|
||||
> You can also customize the initial username and password through the environment variables `ADMIN_USERNAME` and `ADMIN_PASSWORD`.
|
||||
|
||||
### 3. Binary File Deployment
|
||||
|
||||
1. Visit Komari's [GitHub Release page](https://github.com/komari-monitor/komari/releases) to download the latest binary for your operating system.
|
||||
2. Run Komari:
|
||||
```bash
|
||||
./komari server -l 0.0.0.0:25774
|
||||
```
|
||||
3. Access `http://<your_server_ip>:25774` in your browser. The default port is `25774`.
|
||||
4. The default username and password can be found in the startup logs or set via the environment variables `ADMIN_USERNAME` and `ADMIN_PASSWORD`.
|
||||
|
||||
> [!NOTE]
|
||||
> Ensure the binary has execute permissions (`chmod +x komari`). Data will be saved in the `data` folder in the running directory.
|
||||
|
||||
### Manual Build
|
||||
|
||||
#### Dependencies
|
||||
|
||||
- Go 1.18+ and Node.js 20+ (for manual build)
|
||||
|
||||
1. Build the frontend static files:
|
||||
```bash
|
||||
git clone https://github.com/komari-monitor/komari-web
|
||||
cd komari-web
|
||||
npm install
|
||||
npm run build
|
||||
```
|
||||
2. Build the backend:
|
||||
```bash
|
||||
git clone https://github.com/komari-monitor/komari
|
||||
cd komari
|
||||
```
|
||||
Copy the static files generated in step 1 to the `/web/public/defaultTheme/dist` folder in the root of the `komari` project, and copy `komari-theme.json` + `preview.png`/`perview.png` to `/web/public/defaultTheme`.
|
||||
```bash
|
||||
go build -o komari
|
||||
```
|
||||
3. Run:
|
||||
```bash
|
||||
./komari server -l 0.0.0.0:25774
|
||||
```
|
||||
The default listening port is `25774`. Access `http://localhost:25774`.
|
||||
|
||||
## Frontend Development Guide
|
||||
|
||||
[Komari Theme Development Guide | Komari](https://komari-document.pages.dev/dev/theme.html)
|
||||
|
||||
## Client Agent Development Guide
|
||||
|
||||
[Komari Agent Information Reporting and Event Handling Documentation](https://komari-document.pages.dev/dev/agent.html)
|
||||
For Docker, binary, source-build, and update instructions, see the [installation guide](https://komari-document.pages.dev/en/install/quick-start).
|
||||
|
||||
## Contributing
|
||||
|
||||
@@ -122,26 +40,8 @@ Issues and Pull Requests are welcome!
|
||||
|
||||
## Acknowledgements
|
||||
|
||||
### 破碎工坊云
|
||||
|
||||
[破碎工坊云 - 专业云计算服务平台,提供高效、稳定、安全的高防服务器与CDN解决方案](https://www.crash.work/)
|
||||
|
||||
### DreamCloud
|
||||
|
||||
[DreamCloud - 极高性价比解锁直连亚太高防](https://as211392.com/)
|
||||
|
||||
### 🚀 Sponsored by SharonNetworks
|
||||
|
||||
[](https://sharon.io)
|
||||
|
||||
SharonNetworks 为您的业务起飞保驾护航!
|
||||
|
||||
亚太数据中心提供顶级的中国优化网络接入 · 低延时&高带宽&提供Tbps级本地清洗高防服务, 为您的业务保驾护航, 为您的客户提供极致体验. 加入社区 [Telegram群组](https://t.me/SharonNetwork) 可参与公益募捐或群内抽奖免费使用
|
||||
|
||||
### The open source software community
|
||||
|
||||
All the developers who submitted PRs and created themes
|
||||
|
||||
## Star History
|
||||
|
||||
[](https://www.star-history.com/#komari-monitor/komari&Date)
|
||||
| Acknowledgement | Details |
|
||||
| ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
|
||||
| <a href="https://whmcs.as211392.com/aff.php?aff=110"><img src="https://raw.githubusercontent.com/komari-monitor/public/refs/heads/main/images/dreamcloud.png" alt="Dream Cloud" width="180"></a> | 极高性价比解锁直连亚太高防,真高防,不虚标,打死退款 |
|
||||
| <a href="https://sharon.io"><img src="https://raw.githubusercontent.com/komari-monitor/public/refs/heads/main/images/sharon-networks.webp" alt="Sharon Networks" width="180"></a> | 亚太数据中心提供顶级的中国优化网络接入 · 低延时&高带宽&提供Tbps级本地清洗高防服务, 为您的业务保驾护航, 为您的客户提供极致体验. 加入社区 [Telegram群组](https://t.me/SharonNetwork) 可参与公益募捐或群内抽奖免费使用 |
|
||||
| Open-source community | Everyone who has submitted pull requests or created themes. |
|
||||
@@ -0,0 +1,49 @@
|
||||
# Komari
|
||||
|
||||

|
||||
[](https://deepwiki.com/komari-monitor/komari)
|
||||
|
||||

|
||||
|
||||
[简体中文](./README_zh-cn.md)
|
||||
|
||||
Komari 是一款轻量级的自托管服务器监控工具,旨在提供简单、高效的服务器性能监控解决方案。它支持通过 Web 界面查看服务器状态,并通过轻量级 Agent 收集数据。
|
||||
|
||||
> [!WARNING]
|
||||
> Komari 是一款自托管的监控/控制程序,仅应部署在你拥有或已获得授权管理的系统上。请勿将 Komari 武器化,或在未获授权的情况下部署、访问、持久化、执行命令及从事其他滥用行为。关于现实中的滥用风险,可参考 Huntress 的分析:[Komari C2 agent abuse](https://www.huntress.com/blog/komari-c2-agent-abuse)。
|
||||
> 用户需要自行承担部署和使用 Komari 的责任。开发者不对未经授权或滥用行为及其后果承担责任。
|
||||
> 在 Windows 端开启远程控制后,客户端会在每次用户登录时通过 Windows 通知提醒用户 Komari 是一款远程控制软件。
|
||||
|
||||
[文档/Documentation](https://www.komari.wiki/) | [Telegram 群](https://t.me/komari_monitor)
|
||||
|
||||
## 特性
|
||||
|
||||
- **轻量高效**:低资源占用,适合各种规模的服务器。
|
||||
- **自托管**:完全掌控数据隐私,部署简单。
|
||||
- **Web 界面**:直观的监控仪表盘,易于使用。
|
||||
|
||||
## 快速开始
|
||||
|
||||
### 0. 容器云一键部署
|
||||
|
||||
- 雨云云应用 - CNY 4.5/月
|
||||
|
||||
[](https://app.rainyun.com/apps/rca/store/6780/NzYxNzAz_)
|
||||
|
||||
- 1Panel 应用商店
|
||||
|
||||
已上架1Panel应用商店,应用商店-实用工具-Komari 即可安装
|
||||
|
||||
Docker、二进制文件、源码构建和更新说明,请参阅 [安装指南](https://www.komari.wiki/install/quick-start).
|
||||
|
||||
## 贡献
|
||||
|
||||
欢迎提交 Issue 或 Pull Request!
|
||||
|
||||
## 鸣谢
|
||||
|
||||
| 赞助商 | 描述 |
|
||||
| ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
|
||||
| <a href="https://whmcs.as211392.com/aff.php?aff=110"><img src="https://raw.githubusercontent.com/komari-monitor/public/refs/heads/main/images/dreamcloud.png" alt="Dream Cloud" width="180"></a> | 极高性价比解锁直连亚太高防,真高防,不虚标,打死退款 |
|
||||
| <a href="https://sharon.io"><img src="https://raw.githubusercontent.com/komari-monitor/public/refs/heads/main/images/sharon-networks.webp" alt="Sharon Networks" width="180"></a> | 亚太数据中心提供顶级的中国优化网络接入 · 低延时&高带宽&提供Tbps级本地清洗高防服务, 为您的业务保驾护航, 为您的客户提供极致体验. 加入社区 [Telegram群组](https://t.me/SharonNetwork) 可参与公益募捐或群内抽奖免费使用 |
|
||||
| 开源社区 | 提交 PR、制作主题的各位开发者 |
|
||||
@@ -10,11 +10,6 @@ var (
|
||||
// 数据库配置
|
||||
DatabaseType string // 数据库类型:sqlite
|
||||
DatabaseFile string // SQLite数据库文件路径
|
||||
DatabaseHost string // 保留的兼容参数,当前未使用
|
||||
DatabasePort string // 保留的兼容参数,当前未使用
|
||||
DatabaseUser string // 保留的兼容参数,当前未使用
|
||||
DatabasePass string // 保留的兼容参数,当前未使用
|
||||
DatabaseName string // 保留的兼容参数,当前未使用
|
||||
|
||||
Listen string
|
||||
)
|
||||
|
||||
@@ -4,7 +4,7 @@ import (
|
||||
"os"
|
||||
|
||||
"github.com/komari-monitor/komari/database/dbcore"
|
||||
"github.com/komari-monitor/komari/pkg/config"
|
||||
"github.com/komari-monitor/komari/internal/config"
|
||||
"github.com/spf13/cobra"
|
||||
)
|
||||
|
||||
|
||||
+2
-22
@@ -17,17 +17,6 @@ func GetEnv(key, defaultValue string) string {
|
||||
return value
|
||||
}
|
||||
|
||||
// 从环境变量获取默认值
|
||||
var (
|
||||
dbTypeEnv = GetEnv("KOMARI_DB_TYPE", "sqlite")
|
||||
dbFileEnv = GetEnv("KOMARI_DB_FILE", "./data/komari.db")
|
||||
dbHostEnv = GetEnv("KOMARI_DB_HOST", "localhost")
|
||||
dbPortEnv = GetEnv("KOMARI_DB_PORT", "3306")
|
||||
dbUserEnv = GetEnv("KOMARI_DB_USER", "root")
|
||||
dbPassEnv = GetEnv("KOMARI_DB_PASS", "")
|
||||
dbNameEnv = GetEnv("KOMARI_DB_NAME", "komari")
|
||||
)
|
||||
|
||||
var RootCmd = &cobra.Command{
|
||||
Use: "Komari",
|
||||
Short: "Komari is a simple server monitoring tool",
|
||||
@@ -47,15 +36,6 @@ func Execute() {
|
||||
}
|
||||
|
||||
func init() {
|
||||
// 设置命令行参数,提供环境变量作为默认值
|
||||
RootCmd.PersistentFlags().StringVarP(&flags.DatabaseType, "db-type", "t", dbTypeEnv, "Database type (sqlite) [env: KOMARI_DB_TYPE]")
|
||||
RootCmd.PersistentFlags().StringVarP(&flags.DatabaseFile, "database", "d", dbFileEnv, "SQLite database file path [env: KOMARI_DB_FILE]")
|
||||
RootCmd.PersistentFlags().StringVar(&flags.DatabaseHost, "db-host", dbHostEnv, "Reserved database host parameter [env: KOMARI_DB_HOST]")
|
||||
RootCmd.PersistentFlags().StringVar(&flags.DatabasePort, "db-port", dbPortEnv, "Reserved database port parameter [env: KOMARI_DB_PORT]")
|
||||
RootCmd.PersistentFlags().StringVar(&flags.DatabaseUser, "db-user", dbUserEnv, "Reserved database username parameter [env: KOMARI_DB_USER]")
|
||||
RootCmd.PersistentFlags().StringVar(&flags.DatabasePass, "db-pass", dbPassEnv, "Reserved database password parameter [env: KOMARI_DB_PASS]")
|
||||
RootCmd.PersistentFlags().StringVar(&flags.DatabaseName, "db-name", dbNameEnv, "Reserved database name parameter [env: KOMARI_DB_NAME]")
|
||||
for _, name := range []string{"db-host", "db-port", "db-user", "db-pass", "db-name"} {
|
||||
_ = RootCmd.PersistentFlags().MarkHidden(name)
|
||||
}
|
||||
RootCmd.PersistentFlags().StringVarP(&flags.DatabaseType, "db-type", "t", "sqlite", "Database type (sqlite)")
|
||||
RootCmd.PersistentFlags().StringVarP(&flags.DatabaseFile, "database", "d", "./data/komari.db", "SQLite database file path")
|
||||
}
|
||||
+89
-204
@@ -1,40 +1,13 @@
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"log"
|
||||
"net/http"
|
||||
"errors"
|
||||
"os"
|
||||
"os/signal"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
"github.com/gin-gonic/gin"
|
||||
appserver "github.com/komari-monitor/komari/internal/server"
|
||||
logger "github.com/komari-monitor/komari/utils/log"
|
||||
|
||||
"github.com/komari-monitor/komari/cmd/flags"
|
||||
"github.com/komari-monitor/komari/pkg/corn"
|
||||
"github.com/komari-monitor/komari/web/api"
|
||||
|
||||
"github.com/komari-monitor/komari/database"
|
||||
"github.com/komari-monitor/komari/database/accounts"
|
||||
"github.com/komari-monitor/komari/database/auditlog"
|
||||
"github.com/komari-monitor/komari/database/dbcore"
|
||||
"github.com/komari-monitor/komari/database/models"
|
||||
d_notification "github.com/komari-monitor/komari/database/notification"
|
||||
"github.com/komari-monitor/komari/database/records"
|
||||
"github.com/komari-monitor/komari/database/tasks"
|
||||
"github.com/komari-monitor/komari/pkg/config"
|
||||
"github.com/komari-monitor/komari/utils"
|
||||
"github.com/komari-monitor/komari/utils/cloudflared"
|
||||
"github.com/komari-monitor/komari/utils/geoip"
|
||||
logutil "github.com/komari-monitor/komari/utils/log"
|
||||
"github.com/komari-monitor/komari/utils/messageSender"
|
||||
"github.com/komari-monitor/komari/utils/notifier"
|
||||
"github.com/komari-monitor/komari/web/nezha"
|
||||
"github.com/komari-monitor/komari/web/oauth"
|
||||
report_cache "github.com/komari-monitor/komari/web/report"
|
||||
"github.com/komari-monitor/komari/web/router"
|
||||
"github.com/komari-monitor/komari/web/security"
|
||||
"github.com/spf13/cobra"
|
||||
)
|
||||
|
||||
@@ -54,184 +27,96 @@ func init() {
|
||||
RootCmd.AddCommand(ServerCmd)
|
||||
}
|
||||
|
||||
// RunServer 按显式的生命周期阶段启动服务端。
|
||||
//
|
||||
// 具体各阶段的职责与顺序见 internal/server.App。这里只负责串联:
|
||||
// 任一初始化阶段失败即中止启动,避免在半初始化状态下对外提供服务。
|
||||
func RunServer() {
|
||||
// #region 初始化
|
||||
if err := os.MkdirAll("./data/theme", os.ModePerm); err != nil {
|
||||
log.Fatalf("Failed to create theme directory: %v", err)
|
||||
app := appserver.New(appserver.Options{ListenAddr: flags.Listen})
|
||||
if err := app.Bootstrap(); err != nil {
|
||||
_ = app.Shutdown()
|
||||
logger.Fatalf("server", "server startup failed at %q: %v", "bootstrap", err)
|
||||
}
|
||||
InitDatabase()
|
||||
if utils.VersionHash != "unknown" {
|
||||
gin.SetMode(gin.ReleaseMode)
|
||||
}
|
||||
conf, err := config.GetManyAs[config.Settings]()
|
||||
|
||||
installRequired, err := app.InstallRequired()
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
_ = app.Shutdown()
|
||||
logger.Fatalf("server", "server startup failed at %q: %v", "detect-first-run-install", err)
|
||||
}
|
||||
go geoip.InitGeoIp()
|
||||
go DoScheduledWork()
|
||||
go messageSender.Initialize()
|
||||
// oidcInit
|
||||
go oauth.Initialize()
|
||||
|
||||
if conf.NezhaCompatEnabled {
|
||||
go func() {
|
||||
if err := nezha.StartNezhaCompat(conf.NezhaCompatListen); err != nil {
|
||||
log.Printf("Nezha compat server error: %v", err)
|
||||
auditlog.EventLog("error", fmt.Sprintf("Nezha compat server error: %v", err))
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
config.Subscribe(func(event config.ConfigEvent) {
|
||||
if ok, t := config.IsChangedT[string](event, config.OAuthProviderKey); ok {
|
||||
if t == "" || t == "none" {
|
||||
t = "github"
|
||||
}
|
||||
oidcProvider, err := database.GetOidcConfigByName(t)
|
||||
if err != nil {
|
||||
log.Printf("Failed to get OIDC provider config: %v", err)
|
||||
} else {
|
||||
log.Printf("Using %s as OIDC provider", oidcProvider.Name)
|
||||
}
|
||||
err = oauth.LoadProvider(oidcProvider.Name, oidcProvider.Addition)
|
||||
if err != nil {
|
||||
auditlog.EventLog("error", fmt.Sprintf("Failed to load OIDC provider: %v", err))
|
||||
}
|
||||
}
|
||||
|
||||
if ok, t := config.IsChangedT[bool](event, config.NezhaCompatEnabledKey); ok {
|
||||
if t {
|
||||
l, _ := config.GetAs[string](config.NezhaCompatListenKey)
|
||||
if err := nezha.StartNezhaCompat(l); err != nil {
|
||||
log.Printf("start Nezha compat server error: %v", err)
|
||||
auditlog.EventLog("error", fmt.Sprintf("start Nezha compat server error: %v", err))
|
||||
}
|
||||
} else {
|
||||
if err := nezha.StopNezhaCompat(); err != nil {
|
||||
log.Printf("stop Nezha compat server error: %v", err)
|
||||
auditlog.EventLog("error", fmt.Sprintf("stop Nezha compat server error: %v", err))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
})
|
||||
// 初始化 cloudflared
|
||||
if err := cloudflared.AutoStart(GetEnv("KOMARI_CLOUDFLARED_TOKEN", "")); err != nil {
|
||||
log.Printf("failed to auto start cloudflared: %v", err)
|
||||
}
|
||||
|
||||
r := gin.New()
|
||||
r.Use(logutil.GinLogger())
|
||||
r.Use(logutil.GinRecovery())
|
||||
|
||||
config.Subscribe(func(event config.ConfigEvent) {
|
||||
if event.IsChanged(config.GeoIpProviderKey) {
|
||||
go geoip.InitGeoIp()
|
||||
}
|
||||
|
||||
if event.IsChanged(config.NotificationMethodKey) {
|
||||
go messageSender.Initialize()
|
||||
}
|
||||
|
||||
})
|
||||
r.Use(security.CorsMiddleware(conf.CorsOriginCheckEnabled, conf.CorsAllowedOrigins))
|
||||
|
||||
r.Use(api.IdentityMiddleware())
|
||||
r.Use(api.PrivateSiteMiddleware())
|
||||
|
||||
r.Use(func(c *gin.Context) {
|
||||
if len(c.Request.URL.Path) >= 4 && c.Request.URL.Path[:4] == "/api" {
|
||||
c.Header("Cache-Control", "no-store")
|
||||
}
|
||||
c.Next()
|
||||
})
|
||||
|
||||
router.Register(r)
|
||||
|
||||
srv := &http.Server{
|
||||
Addr: flags.Listen,
|
||||
Handler: r,
|
||||
}
|
||||
log.Printf("Starting server on %s ...", flags.Listen)
|
||||
go func() {
|
||||
if err := srv.ListenAndServe(); err != nil && err != http.ErrServerClosed {
|
||||
OnFatal(err)
|
||||
log.Fatalf("listen: %s\n", err)
|
||||
}
|
||||
}()
|
||||
quit := make(chan os.Signal, 1)
|
||||
signal.Notify(quit, os.Interrupt, syscall.SIGTERM)
|
||||
<-quit
|
||||
OnShutdown()
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
defer cancel()
|
||||
if err := srv.Shutdown(ctx); err != nil {
|
||||
log.Fatalf("Server forced to shutdown: %v", err)
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
func InitDatabase() {
|
||||
var count int64 = 0
|
||||
if dbcore.GetDBInstance().Model(&models.User{}).Count(&count); count == 0 {
|
||||
user, passwd, err := accounts.CreateDefaultAdminAccount()
|
||||
if installRequired {
|
||||
completed, err := app.RunInstallGuide()
|
||||
if err != nil {
|
||||
panic(err)
|
||||
_ = app.Shutdown()
|
||||
logger.Fatalf("server", "server startup failed at %q: %v", "run-first-run-install", err)
|
||||
}
|
||||
log.Println("Default admin account created. Username:", user, ", Password:", passwd)
|
||||
if !completed {
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
for {
|
||||
requirement, err := app.DatabaseMigrationRequired()
|
||||
if err != nil {
|
||||
completed, recoveryErr := app.RunMetricStoreRecovery(err)
|
||||
if recoveryErr != nil {
|
||||
_ = app.Shutdown()
|
||||
logger.Fatalf("server", "server startup failed at %q: %v", "database-migration-detection-recovery", recoveryErr)
|
||||
}
|
||||
if !completed {
|
||||
return
|
||||
}
|
||||
continue
|
||||
}
|
||||
if !requirement.Required() {
|
||||
break
|
||||
}
|
||||
completed, err := app.RunDatabaseMigration(requirement)
|
||||
if err != nil {
|
||||
_ = app.Shutdown()
|
||||
logger.Fatalf("server", "server startup failed at %q: %v", "run-database-migration", err)
|
||||
}
|
||||
if !completed {
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
// Metric store 是唯一允许进入恢复向导的启动阶段:主库已经在
|
||||
// Bootstrap 中就绪,因此可以保留登录能力并让管理员修正 DSN。
|
||||
if err := app.ConnectMetricStoreWithRetry(); err != nil {
|
||||
completed, recoveryErr := app.RunMetricStoreRecovery(err)
|
||||
if recoveryErr != nil {
|
||||
_ = app.Shutdown()
|
||||
logger.Fatalf("server", "server startup failed at %q: %v", "metric-store-recovery", recoveryErr)
|
||||
}
|
||||
if !completed {
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
// 其余初始化阶段:任一步失败都不应继续对外服务。
|
||||
type stage struct {
|
||||
name string
|
||||
fn func() error
|
||||
}
|
||||
stages := []stage{
|
||||
{"init-stores", app.InitStores},
|
||||
{"init-providers", app.InitProviders},
|
||||
{"start-background", app.StartBackground},
|
||||
{"build-router", app.BuildRouter},
|
||||
}
|
||||
for _, s := range stages {
|
||||
if err := s.fn(); err != nil {
|
||||
// 已登记的资源尽力回收,再退出。
|
||||
_ = app.Shutdown()
|
||||
logger.Fatalf("server", "server startup failed at %q: %v", s.name, err)
|
||||
}
|
||||
}
|
||||
|
||||
if err := app.Run(); err != nil {
|
||||
if errors.Is(err, appserver.ErrRestartRequested) {
|
||||
logger.Infof("server", "Service stopped for restart: %v", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
logger.Fatalf("server", "server exited with error: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// #region 定时任务
|
||||
func DoScheduledWork() {
|
||||
if err := tasks.ReloadPingSchedule(); err != nil {
|
||||
log.Println("Failed to reload ping schedule:", err)
|
||||
}
|
||||
if err := d_notification.ReloadLoadNotificationSchedule(); err != nil {
|
||||
log.Println("Failed to reload load notification schedule:", err)
|
||||
}
|
||||
records.CompactRecord()
|
||||
|
||||
if err := corn.AddFunc("records:cleanup", "@every 30m", cleanupScheduledData); err != nil {
|
||||
log.Println("Failed to add cleanup scheduled task:", err)
|
||||
}
|
||||
if err := corn.AddFunc("records:minute", "@every 1m", minuteScheduledWork); err != nil {
|
||||
log.Println("Failed to add minute scheduled task:", err)
|
||||
}
|
||||
if err := corn.AddFunc("notifier:expire", "0 0 9 * * *", notifier.CheckExpireScheduledWork); err != nil {
|
||||
log.Println("Failed to add expire notification scheduled task:", err)
|
||||
}
|
||||
notifier.InitTrafficReportSchedule()
|
||||
}
|
||||
|
||||
func cleanupScheduledData() {
|
||||
cfg, _ := config.GetManyAs[config.Settings]()
|
||||
records.DeleteRecordBefore(time.Now().Add(-time.Hour * time.Duration(cfg.RecordPreserveTime)))
|
||||
records.CompactRecord()
|
||||
tasks.ClearTaskResultsByTimeBefore(time.Now().Add(-time.Hour * time.Duration(cfg.RecordPreserveTime)))
|
||||
tasks.DeletePingRecordsBefore(time.Now().Add(-time.Hour * time.Duration(cfg.PingRecordPreserveTime)))
|
||||
auditlog.RemoveOldLogs()
|
||||
accounts.RemoveExpiredSessions()
|
||||
}
|
||||
|
||||
func minuteScheduledWork() {
|
||||
cfg, _ := config.GetManyAs[config.Settings]()
|
||||
report_cache.SaveClientReportToDB()
|
||||
if !cfg.RecordEnabled {
|
||||
records.DeleteAll()
|
||||
tasks.DeleteAllPingRecords()
|
||||
}
|
||||
// 每分钟检查一次流量提醒
|
||||
notifier.CheckTraffic()
|
||||
}
|
||||
|
||||
func OnShutdown() {
|
||||
auditlog.Log("", "", "server is shutting down", "info")
|
||||
corn.StopAll()
|
||||
cloudflared.Shutdown()
|
||||
}
|
||||
|
||||
func OnFatal(err error) {
|
||||
auditlog.Log("", "", "server encountered a fatal error: "+err.Error(), "error")
|
||||
cloudflared.Shutdown()
|
||||
}
|
||||
@@ -4,14 +4,12 @@ import (
|
||||
"crypto/sha256"
|
||||
"encoding/base64"
|
||||
"fmt"
|
||||
"os"
|
||||
"time"
|
||||
|
||||
"github.com/google/uuid"
|
||||
"github.com/komari-monitor/komari/database/dbcore"
|
||||
"github.com/komari-monitor/komari/database/models"
|
||||
"github.com/komari-monitor/komari/utils"
|
||||
|
||||
"github.com/google/uuid"
|
||||
"gorm.io/gorm"
|
||||
)
|
||||
|
||||
const constantSalt = "06Wm4Jv1Hkxx"
|
||||
@@ -56,7 +54,10 @@ func hashPasswd(passwd string) string {
|
||||
}
|
||||
|
||||
func CreateAccount(username, passwd string) (user models.User, err error) {
|
||||
db := dbcore.GetDBInstance()
|
||||
return CreateAccountWithDB(dbcore.GetDBInstance(), username, passwd)
|
||||
}
|
||||
|
||||
func CreateAccountWithDB(db *gorm.DB, username, passwd string) (user models.User, err error) {
|
||||
hashedPassword := hashPasswd(passwd)
|
||||
user = models.User{
|
||||
UUID: uuid.New().String(),
|
||||
@@ -71,7 +72,10 @@ func CreateAccount(username, passwd string) (user models.User, err error) {
|
||||
}
|
||||
|
||||
func DeleteAccountByUsername(username string) (err error) {
|
||||
db := dbcore.GetDBInstance()
|
||||
return DeleteAccountByUsernameWithDB(dbcore.GetDBInstance(), username)
|
||||
}
|
||||
|
||||
func DeleteAccountByUsernameWithDB(db *gorm.DB, username string) (err error) {
|
||||
err = db.Where("username = ?", username).Delete(&models.User{}).Error
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -79,39 +83,6 @@ func DeleteAccountByUsername(username string) (err error) {
|
||||
return nil
|
||||
}
|
||||
|
||||
// 创建默认管理员账户,使用环境变量 ADMIN_USERNAME 作为用户名,环境变量 ADMIN_PASSWORD 作为密码
|
||||
func CreateDefaultAdminAccount() (username, passwd string, err error) {
|
||||
db := dbcore.GetDBInstance()
|
||||
|
||||
username = os.Getenv("ADMIN_USERNAME")
|
||||
if username == "" {
|
||||
username = "admin"
|
||||
}
|
||||
|
||||
passwd = os.Getenv("ADMIN_PASSWORD")
|
||||
if passwd == "" {
|
||||
passwd = utils.GeneratePassword()
|
||||
}
|
||||
|
||||
hashedPassword := hashPasswd(passwd)
|
||||
|
||||
user := models.User{
|
||||
UUID: uuid.New().String(),
|
||||
Username: username,
|
||||
Passwd: hashedPassword,
|
||||
SSOID: "",
|
||||
CreatedAt: models.FromTime(time.Now()),
|
||||
UpdatedAt: models.FromTime(time.Now()),
|
||||
}
|
||||
|
||||
err = db.Create(&user).Error
|
||||
if err != nil {
|
||||
return "", "", err
|
||||
}
|
||||
|
||||
return username, passwd, nil
|
||||
}
|
||||
|
||||
func GetUserByUUID(uuid string) (user models.User, err error) {
|
||||
db := dbcore.GetDBInstance()
|
||||
err = db.Where("uuid = ?", uuid).First(&user).Error
|
||||
@@ -171,7 +142,7 @@ func UpdateUser(uuid string, name, password, sso_type *string) error {
|
||||
if sso_type != nil {
|
||||
updates["sso_type"] = *sso_type
|
||||
}
|
||||
updates["updated_at"] = time.Now()
|
||||
updates["updated_at"] = time.Now().UTC()
|
||||
err := db.Model(&models.User{}).Where("uuid = ?", uuid).Updates(updates).Error
|
||||
if err != nil {
|
||||
return err
|
||||
|
||||
@@ -9,7 +9,7 @@ import (
|
||||
"github.com/komari-monitor/komari/database/dbcore"
|
||||
"github.com/komari-monitor/komari/database/models"
|
||||
messageevent "github.com/komari-monitor/komari/database/models/messageEvent"
|
||||
"github.com/komari-monitor/komari/pkg/config"
|
||||
"github.com/komari-monitor/komari/internal/config"
|
||||
"github.com/komari-monitor/komari/utils"
|
||||
"github.com/komari-monitor/komari/utils/geoip"
|
||||
"github.com/komari-monitor/komari/utils/messageSender"
|
||||
@@ -33,11 +33,11 @@ func CreateSession(uuid string, expires int, userAgent, ip, login_method string)
|
||||
sessionRecord := models.Session{
|
||||
UUID: uuid,
|
||||
Session: session,
|
||||
Expires: models.FromTime(time.Now().Add(time.Duration(expires) * time.Second)),
|
||||
Expires: time.Now().UTC().Add(time.Duration(expires) * time.Second),
|
||||
UserAgent: userAgent,
|
||||
Ip: ip,
|
||||
LoginMethod: login_method,
|
||||
LatestOnline: models.FromTime(time.Now()),
|
||||
LatestOnline: time.Now().UTC(),
|
||||
}
|
||||
go func() {
|
||||
LoginNotification, _ := config.GetAs[bool](config.LoginNotificationKey, false)
|
||||
@@ -50,7 +50,7 @@ func CreateSession(uuid string, expires int, userAgent, ip, login_method string)
|
||||
}
|
||||
messageSender.SendEvent(models.EventMessage{
|
||||
Event: messageevent.Login,
|
||||
Time: time.Now(),
|
||||
Time: time.Now().UTC(),
|
||||
Message: fmt.Sprintf("%s: %s (%s)\n%s", login_method, ip, loc, userAgent),
|
||||
Emoji: "🔑",
|
||||
})
|
||||
@@ -73,7 +73,7 @@ func GetSession(session string) (uuid string, err error) {
|
||||
return "", err
|
||||
}
|
||||
|
||||
if time.Now().After(sessionRecord.Expires.ToTime()) {
|
||||
if time.Now().UTC().After(sessionRecord.Expires) {
|
||||
// 会话已过期,删除它
|
||||
_ = DeleteSession(session)
|
||||
return "", errors.New("session expired")
|
||||
@@ -111,24 +111,10 @@ func DeleteAllSessions() error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func UpdateLatestOnline(session string) error {
|
||||
db := dbcore.GetDBInstance()
|
||||
return db.Model(&models.Session{}).Where("session = ?", session).Update("latest_online", time.Now()).Error
|
||||
}
|
||||
|
||||
func UpdateLatestUserAgent(session, userAgent string) error {
|
||||
db := dbcore.GetDBInstance()
|
||||
return db.Model(&models.Session{}).Where("session = ?", session).Update("latest_user_agent", userAgent).Error
|
||||
}
|
||||
func UpdateLatestIp(session, ip string) error {
|
||||
db := dbcore.GetDBInstance()
|
||||
return db.Model(&models.Session{}).Where("session = ?", session).Update("latest_ip", ip).Error
|
||||
}
|
||||
|
||||
func UpdateLatest(session, useragent, ip string) error {
|
||||
db := dbcore.GetDBInstance()
|
||||
return db.Model(&models.Session{}).Where("session = ?", session).Updates(map[string]interface{}{
|
||||
"latest_online": time.Now(),
|
||||
"latest_online": time.Now().UTC(),
|
||||
"latest_user_agent": useragent,
|
||||
"latest_ip": ip,
|
||||
}).Error
|
||||
@@ -136,7 +122,7 @@ func UpdateLatest(session, useragent, ip string) error {
|
||||
|
||||
func RemoveExpiredSessions() error {
|
||||
db := dbcore.GetDBInstance()
|
||||
result := db.Where("expires < ?", time.Now()).Delete(&models.Session{})
|
||||
result := db.Where("expires < ?", time.Now().UTC()).Delete(&models.Session{})
|
||||
if result.Error != nil {
|
||||
return result.Error
|
||||
}
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
package auditlog
|
||||
|
||||
import (
|
||||
"log"
|
||||
logger "github.com/komari-monitor/komari/utils/log"
|
||||
"time"
|
||||
|
||||
"github.com/komari-monitor/komari/database/dbcore"
|
||||
@@ -9,16 +9,17 @@ import (
|
||||
)
|
||||
|
||||
func Log(ip, uuid, message, msgType string) {
|
||||
now := time.Now()
|
||||
db := dbcore.GetDBInstance()
|
||||
logEntry := &models.Log{
|
||||
IP: ip,
|
||||
UUID: uuid,
|
||||
Message: message,
|
||||
MsgType: msgType,
|
||||
Time: models.FromTime(now),
|
||||
Time: time.Now().UTC(),
|
||||
}
|
||||
if err := db.Create(logEntry).Error; err != nil {
|
||||
logger.Error("audit", "failed to persist audit event", "error", err, "type", msgType)
|
||||
}
|
||||
db.Create(logEntry)
|
||||
}
|
||||
|
||||
func EventLog(eventType, message string) {
|
||||
@@ -28,8 +29,8 @@ func EventLog(eventType, message string) {
|
||||
// Delete logs older than 30 days
|
||||
func RemoveOldLogs() {
|
||||
db := dbcore.GetDBInstance()
|
||||
threshold := time.Now().AddDate(0, 0, -30)
|
||||
threshold := time.Now().UTC().AddDate(0, 0, -30)
|
||||
if err := db.Where("time < ?", threshold).Delete(&models.Log{}).Error; err != nil {
|
||||
log.Println("Failed to remove old logs:", err)
|
||||
logger.ErrorArgs("audit", "Failed to remove old logs:", err)
|
||||
}
|
||||
}
|
||||
+31
-41
@@ -3,7 +3,7 @@ package clients
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"log"
|
||||
logger "github.com/komari-monitor/komari/utils/log"
|
||||
"math"
|
||||
"time"
|
||||
|
||||
@@ -11,7 +11,6 @@ import (
|
||||
"github.com/komari-monitor/komari/database/models"
|
||||
"github.com/komari-monitor/komari/database/tasks"
|
||||
"github.com/komari-monitor/komari/utils"
|
||||
"gorm.io/gorm"
|
||||
|
||||
"github.com/google/uuid"
|
||||
)
|
||||
@@ -37,7 +36,7 @@ func SaveClientInfo(update map[string]interface{}) error {
|
||||
return fmt.Errorf("no fields to update")
|
||||
}
|
||||
|
||||
update["updated_at"] = time.Now()
|
||||
update["updated_at"] = time.Now().UTC()
|
||||
|
||||
toFloat64 := func(value interface{}) (float64, bool) {
|
||||
switch typed := value.(type) {
|
||||
@@ -122,24 +121,6 @@ func SaveClientInfo(update map[string]interface{}) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func EditClientName(clientUUID, clientName string) error {
|
||||
db := dbcore.GetDBInstance()
|
||||
err := db.Model(&models.Client{}).Where("uuid = ?", clientUUID).Update("name", clientName).Error
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func EditClientToken(clientUUID, token string) error {
|
||||
db := dbcore.GetDBInstance()
|
||||
err := db.Model(&models.Client{}).Where("uuid = ?", clientUUID).Update("token", token).Error
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// CreateClient 创建新客户端
|
||||
func CreateClient() (clientUUID, token string, err error) {
|
||||
db := dbcore.GetDBInstance()
|
||||
@@ -150,8 +131,8 @@ func CreateClient() (clientUUID, token string, err error) {
|
||||
UUID: clientUUID,
|
||||
Token: token,
|
||||
Name: "client_" + clientUUID[0:8],
|
||||
CreatedAt: models.FromTime(time.Now()),
|
||||
UpdatedAt: models.FromTime(time.Now()),
|
||||
CreatedAt: time.Now().UTC(),
|
||||
UpdatedAt: time.Now().UTC(),
|
||||
}
|
||||
|
||||
err = db.Create(&client).Error
|
||||
@@ -159,7 +140,7 @@ func CreateClient() (clientUUID, token string, err error) {
|
||||
return "", "", err
|
||||
}
|
||||
if err := tasks.AddDefaultOnClientUUID(clientUUID); err != nil {
|
||||
log.Println("Failed to apply default-on ping tasks to new client:", err)
|
||||
logger.ErrorArgs("clients", "Failed to apply default-on ping tasks to new client:", err)
|
||||
}
|
||||
return clientUUID, token, nil
|
||||
}
|
||||
@@ -175,8 +156,8 @@ func CreateClientWithName(name string) (clientUUID, token string, err error) {
|
||||
UUID: clientUUID,
|
||||
Token: token,
|
||||
Name: name,
|
||||
CreatedAt: models.FromTime(time.Now()),
|
||||
UpdatedAt: models.FromTime(time.Now()),
|
||||
CreatedAt: time.Now().UTC(),
|
||||
UpdatedAt: time.Now().UTC(),
|
||||
}
|
||||
|
||||
err = db.Create(&client).Error
|
||||
@@ -184,7 +165,7 @@ func CreateClientWithName(name string) (clientUUID, token string, err error) {
|
||||
return "", "", err
|
||||
}
|
||||
if err := tasks.AddDefaultOnClientUUID(clientUUID); err != nil {
|
||||
log.Println("Failed to apply default-on ping tasks to new client:", err)
|
||||
logger.ErrorArgs("clients", "Failed to apply default-on ping tasks to new client:", err)
|
||||
}
|
||||
return clientUUID, token, nil
|
||||
}
|
||||
@@ -210,19 +191,6 @@ func GetClientByUUID(uuid string) (client models.Client, err error) {
|
||||
return client, nil
|
||||
}
|
||||
|
||||
// GetClientBasicInfo 获取指定 UUID 的客户端基本信息
|
||||
func GetClientBasicInfo(uuid string) (client models.Client, err error) {
|
||||
db := dbcore.GetDBInstance()
|
||||
err = db.Where("uuid = ?", uuid).First(&client).Error
|
||||
if err != nil {
|
||||
if err == gorm.ErrRecordNotFound {
|
||||
return models.Client{}, fmt.Errorf("客户端不存在: %s", uuid)
|
||||
}
|
||||
return models.Client{}, err
|
||||
}
|
||||
return client, nil
|
||||
}
|
||||
|
||||
func GetClientTokenByUUID(uuid string) (token string, err error) {
|
||||
db := dbcore.GetDBInstance()
|
||||
var client models.Client
|
||||
@@ -261,8 +229,30 @@ func SaveClient(updates map[string]interface{}) error {
|
||||
}
|
||||
}
|
||||
}
|
||||
if value, exists := updates["expired_at"]; exists {
|
||||
switch typed := value.(type) {
|
||||
case nil:
|
||||
updates["expired_at"] = nil
|
||||
case time.Time:
|
||||
updates["expired_at"] = typed.UTC()
|
||||
case *time.Time:
|
||||
if typed == nil {
|
||||
updates["expired_at"] = nil
|
||||
} else {
|
||||
updates["expired_at"] = typed.UTC()
|
||||
}
|
||||
case string:
|
||||
stamp, err := time.Parse(time.RFC3339Nano, typed)
|
||||
if err != nil {
|
||||
return fmt.Errorf("expired_at must be an RFC3339 timestamp with a timezone: %w", err)
|
||||
}
|
||||
updates["expired_at"] = stamp.UTC()
|
||||
default:
|
||||
return fmt.Errorf("expired_at must be an RFC3339 timestamp with a timezone")
|
||||
}
|
||||
}
|
||||
|
||||
updates["updated_at"] = time.Now()
|
||||
updates["updated_at"] = time.Now().UTC()
|
||||
|
||||
err := db.Model(&models.Client{}).Where("uuid = ?", clientUUID).Updates(updates).Error
|
||||
if err != nil {
|
||||
|
||||
@@ -1,66 +1,14 @@
|
||||
package clients
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"math"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/komari-monitor/komari/database/dbcore"
|
||||
"github.com/komari-monitor/komari/database/models"
|
||||
v1 "github.com/komari-monitor/komari/protocol/v1"
|
||||
"github.com/komari-monitor/komari/utils"
|
||||
|
||||
"gorm.io/gorm"
|
||||
)
|
||||
|
||||
var reportSaveLocks sync.Map
|
||||
|
||||
func getReportSaveLock(clientUUID string) *sync.Mutex {
|
||||
lock, _ := reportSaveLocks.LoadOrStore(clientUUID, &sync.Mutex{})
|
||||
return lock.(*sync.Mutex)
|
||||
}
|
||||
|
||||
func getLatestTrafficRecord(tx *gorm.DB, clientUUID string) (*models.Record, error) {
|
||||
var record models.Record
|
||||
err := tx.Where("client = ?", clientUUID).Order("time DESC").First(&record).Error
|
||||
if err == nil {
|
||||
return &record, nil
|
||||
}
|
||||
if !errors.Is(err, gorm.ErrRecordNotFound) {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
err = tx.Table("records_long_term").Where("client = ?", clientUUID).Order("time DESC").First(&record).Error
|
||||
if err == nil {
|
||||
return &record, nil
|
||||
}
|
||||
if !errors.Is(err, gorm.ErrRecordNotFound) {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
// Report 表示客户端报告数据
|
||||
// SaveReport 保存报告数据
|
||||
func SaveReport(uuid string, data map[string]interface{}) (err error) {
|
||||
|
||||
report, err := ParseReport(data)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
err = SaveClientReport(uuid, report)
|
||||
if err != nil {
|
||||
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
|
||||
}
|
||||
|
||||
func GetClientUUIDByToken(token string) (clientUUID string, err error) {
|
||||
db := dbcore.GetDBInstance()
|
||||
var client models.Client
|
||||
@@ -71,18 +19,6 @@ func GetClientUUIDByToken(token string) (clientUUID string, err error) {
|
||||
return client.UUID, nil
|
||||
}
|
||||
|
||||
func ParseReport(data map[string]interface{}) (report v1.Report, err error) {
|
||||
jsonData, err := json.Marshal(data)
|
||||
if err != nil {
|
||||
return v1.Report{}, err
|
||||
}
|
||||
err = json.Unmarshal(jsonData, &report)
|
||||
if err != nil {
|
||||
return v1.Report{}, err
|
||||
}
|
||||
return report, nil
|
||||
}
|
||||
|
||||
// 检查数据防止异常数据导致数据库损坏
|
||||
func ReportVerify(report v1.Report) error {
|
||||
// 防止输入不合理范围
|
||||
@@ -166,132 +102,3 @@ func ReportVerify(report v1.Report) error {
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// SaveClientReport 保存客户端报告到 Record 表
|
||||
func SaveClientReport(clientUUID string, report v1.Report) (err error) {
|
||||
db := dbcore.GetDBInstance()
|
||||
lock := getReportSaveLock(clientUUID)
|
||||
lock.Lock()
|
||||
defer lock.Unlock()
|
||||
|
||||
if err := ReportVerify(report); err != nil {
|
||||
return fmt.Errorf("failed to save Record: %w", err)
|
||||
}
|
||||
|
||||
// 保存GPU详细记录到独立表
|
||||
currentTime := time.Now()
|
||||
if report.GPU != nil && len(report.GPU.DetailedInfo) > 0 {
|
||||
for idx, gpu := range report.GPU.DetailedInfo {
|
||||
gpuRecord := models.GPURecord{
|
||||
Client: clientUUID,
|
||||
Time: models.FromTime(currentTime),
|
||||
DeviceIndex: idx,
|
||||
DeviceName: gpu.Name,
|
||||
MemTotal: gpu.MemoryTotal,
|
||||
MemUsed: gpu.MemoryUsed,
|
||||
Utilization: float32(gpu.Utilization),
|
||||
Temperature: gpu.Temperature,
|
||||
}
|
||||
if err := db.Create(&gpuRecord).Error; err != nil {
|
||||
return fmt.Errorf("failed to save GPU record: %w", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 计算平均GPU使用率,用于向后兼容
|
||||
averageGPUUsage := float32(0)
|
||||
if report.GPU != nil && len(report.GPU.DetailedInfo) > 0 {
|
||||
averageGPUUsage = float32(report.GPU.AverageUsage)
|
||||
}
|
||||
|
||||
Record := models.Record{
|
||||
Client: clientUUID,
|
||||
Time: models.FromTime(currentTime),
|
||||
Cpu: float32(report.CPU.Usage),
|
||||
Gpu: averageGPUUsage, // 使用平均GPU使用率
|
||||
Ram: report.Ram.Used,
|
||||
RamTotal: report.Ram.Total,
|
||||
Swap: report.Swap.Used,
|
||||
SwapTotal: report.Swap.Total,
|
||||
Load: float32(report.Load.Load1), // 使用 Load1 作为主要负载指标
|
||||
Temp: 0, // Report 未提供 TEMP,设为 0(与原 nil 行为类似)
|
||||
Disk: report.Disk.Used,
|
||||
DiskTotal: report.Disk.Total,
|
||||
NetIn: report.Network.Down,
|
||||
NetOut: report.Network.Up,
|
||||
NetTotalUp: report.Network.TotalUp,
|
||||
NetTotalDown: report.Network.TotalDown,
|
||||
TrafficUp: 0,
|
||||
TrafficDown: 0,
|
||||
Process: report.Process,
|
||||
Connections: report.Connections.TCP,
|
||||
ConnectionsUdp: report.Connections.UDP,
|
||||
//Uptime: report.Uptime,
|
||||
}
|
||||
|
||||
// 使用事务确保 Record 和 ClientsInfo 一致性
|
||||
err = db.Transaction(func(tx *gorm.DB) error {
|
||||
previous, err := getLatestTrafficRecord(tx, clientUUID)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to load previous Record: %w", err)
|
||||
}
|
||||
if previous != nil {
|
||||
Record.TrafficUp = utils.ComputeTrafficDelta(report.Network.TotalUp, previous.NetTotalUp)
|
||||
Record.TrafficDown = utils.ComputeTrafficDelta(report.Network.TotalDown, previous.NetTotalDown)
|
||||
}
|
||||
|
||||
// 保存 Record
|
||||
if err := tx.Create(&Record).Error; err != nil {
|
||||
return fmt.Errorf("failed to save Record: %w", err)
|
||||
}
|
||||
return nil
|
||||
})
|
||||
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
/*
|
||||
// getString 从 map 中获取字符串
|
||||
func getString(data map[string]interface{}, key string) string {
|
||||
if val, ok := data[key]; ok {
|
||||
if str, ok := val.(string); ok {
|
||||
return str
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// getInt 从 map 中获取整数
|
||||
func getInt(data map[string]interface{}, key string) int {
|
||||
if val, ok := data[key]; ok {
|
||||
if num, ok := val.(float64); ok {
|
||||
return int(num)
|
||||
}
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
// getInt64 从 map 中获取 int64
|
||||
func getInt64(data map[string]interface{}, key string) int64 {
|
||||
if val, ok := data[key]; ok {
|
||||
if num, ok := val.(float64); ok {
|
||||
return int64(num)
|
||||
}
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
// getFloat 从 map 中获取 float64
|
||||
func getFloat(data map[string]interface{}, key string) float64 {
|
||||
if val, ok := data[key]; ok {
|
||||
if num, ok := val.(float64); ok {
|
||||
return num
|
||||
}
|
||||
}
|
||||
return 0.0
|
||||
}
|
||||
*/
|
||||
@@ -1,83 +0,0 @@
|
||||
package clients
|
||||
|
||||
import (
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/komari-monitor/komari/utils"
|
||||
"github.com/stretchr/testify/assert"
|
||||
)
|
||||
|
||||
func TestComputeTrafficDeltaHandlesZeroAndReset(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
current int64
|
||||
previous int64
|
||||
want int64
|
||||
}{
|
||||
{name: "previous zero counts current delta", current: 120, previous: 0, want: 120},
|
||||
{name: "monotonic counter uses difference", current: 250, previous: 200, want: 50},
|
||||
{name: "counter reset uses current", current: 15, previous: 250, want: 15},
|
||||
{name: "negative previous remains guarded", current: 15, previous: -1, want: 0},
|
||||
}
|
||||
|
||||
for _, test := range tests {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
assert.Equal(t, test.want, utils.ComputeTrafficDelta(test.current, test.previous))
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestReportSaveLockSerializesSameClient(t *testing.T) {
|
||||
clientUUID := "client-lock-same"
|
||||
lock := getReportSaveLock(clientUUID)
|
||||
lock.Lock()
|
||||
defer lock.Unlock()
|
||||
|
||||
acquired := make(chan struct{})
|
||||
go func() {
|
||||
secondLock := getReportSaveLock(clientUUID)
|
||||
secondLock.Lock()
|
||||
defer secondLock.Unlock()
|
||||
close(acquired)
|
||||
}()
|
||||
|
||||
select {
|
||||
case <-acquired:
|
||||
t.Fatal("same-client report save lock was acquired while already held")
|
||||
case <-time.After(20 * time.Millisecond):
|
||||
}
|
||||
|
||||
lock.Unlock()
|
||||
select {
|
||||
case <-acquired:
|
||||
case <-time.After(time.Second):
|
||||
t.Fatal("same-client report save lock was not released")
|
||||
}
|
||||
lock.Lock()
|
||||
}
|
||||
|
||||
func TestReportSaveLockAllowsDifferentClients(t *testing.T) {
|
||||
firstLock := getReportSaveLock("client-lock-a")
|
||||
firstLock.Lock()
|
||||
defer firstLock.Unlock()
|
||||
|
||||
var waitGroup sync.WaitGroup
|
||||
waitGroup.Add(1)
|
||||
acquired := make(chan struct{})
|
||||
go func() {
|
||||
defer waitGroup.Done()
|
||||
secondLock := getReportSaveLock("client-lock-b")
|
||||
secondLock.Lock()
|
||||
defer secondLock.Unlock()
|
||||
close(acquired)
|
||||
}()
|
||||
|
||||
select {
|
||||
case <-acquired:
|
||||
case <-time.After(time.Second):
|
||||
t.Fatal("different-client report save lock should not block")
|
||||
}
|
||||
waitGroup.Wait()
|
||||
}
|
||||
@@ -1,6 +1,8 @@
|
||||
package clipboard
|
||||
|
||||
import (
|
||||
"time"
|
||||
|
||||
"github.com/komari-monitor/komari/database/dbcore"
|
||||
"github.com/komari-monitor/komari/database/models"
|
||||
)
|
||||
@@ -8,6 +10,8 @@ import (
|
||||
// CreateClipboard 创建剪贴板记录
|
||||
func CreateClipboard(cb *models.Clipboard) error {
|
||||
db := dbcore.GetDBInstance()
|
||||
cb.CreatedAt = time.Now().UTC()
|
||||
cb.UpdatedAt = cb.CreatedAt
|
||||
return db.Create(cb).Error
|
||||
}
|
||||
|
||||
@@ -24,6 +28,8 @@ func GetClipboardByID(id int) (*models.Clipboard, error) {
|
||||
// UpdateClipboardFields 更新剪贴板记录
|
||||
func UpdateClipboardFields(id int, fields map[string]interface{}) error {
|
||||
db := dbcore.GetDBInstance()
|
||||
delete(fields, "created_at")
|
||||
fields["updated_at"] = time.Now().UTC()
|
||||
return db.Model(&models.Clipboard{}).Where("id = ?", id).Updates(fields).Error
|
||||
}
|
||||
|
||||
|
||||
+304
-135
@@ -4,7 +4,6 @@ import (
|
||||
"archive/zip"
|
||||
"fmt"
|
||||
"io"
|
||||
"log"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
@@ -13,9 +12,10 @@ import (
|
||||
|
||||
"github.com/komari-monitor/komari/cmd/flags"
|
||||
"github.com/komari-monitor/komari/database/models"
|
||||
"github.com/komari-monitor/komari/pkg/config"
|
||||
"github.com/komari-monitor/komari/pkg/migrations"
|
||||
logutil "github.com/komari-monitor/komari/utils/log"
|
||||
"github.com/komari-monitor/komari/internal/config"
|
||||
"github.com/komari-monitor/komari/internal/migrations"
|
||||
"github.com/komari-monitor/komari/internal/sqlitetune"
|
||||
logger "github.com/komari-monitor/komari/utils/log"
|
||||
"gorm.io/driver/sqlite"
|
||||
"gorm.io/gorm"
|
||||
)
|
||||
@@ -168,141 +168,310 @@ func unzipToDir(zipPath, dstDir string) error {
|
||||
var (
|
||||
instance *gorm.DB
|
||||
once sync.Once
|
||||
initErr error
|
||||
)
|
||||
|
||||
func GetDBInstance() *gorm.DB {
|
||||
// SystemVersionKey 是记录“上次启动所用版本标识”的配置键(存于 configs 表)。
|
||||
// 取代旧的 ./data/.komari-version 文件:版本标识随配置库一起备份/恢复,
|
||||
// 也避免额外的裸文件依赖。
|
||||
const SystemVersionKey = "system_version"
|
||||
|
||||
const (
|
||||
mainSQLiteBusyTimeout = 5 * time.Second
|
||||
mainSQLiteCacheSizeKB = 8 * 1024
|
||||
mainSQLiteWALAutoCheckpoint = 256
|
||||
mainSQLiteJournalSizeLimit = 1 << 20
|
||||
)
|
||||
|
||||
// versionID 是当前构建的版本标识,由 SetVersionID 在 Initialize 前注入。
|
||||
var versionID string
|
||||
|
||||
// dbFileExistedAtStartup 记录本次进程启动、打开数据库之前 komari.db 是否已存在,
|
||||
// 用于区分“全新安装”与“从旧版升级(无版本标记)”。在 doInitialize 打开数据库
|
||||
// 之前采集。
|
||||
var dbFileExistedAtStartup bool
|
||||
|
||||
// SetVersionID 设置当前构建的版本标识(通常为 CurrentVersion+"-"+VersionHash),
|
||||
// 用于版本升级检测与自动备份。应在 Initialize() 之前调用;为空则跳过升级备份。
|
||||
func SetVersionID(id string) {
|
||||
versionID = id
|
||||
}
|
||||
|
||||
// resolveDatabaseFile 返回当前使用的 SQLite 数据库文件路径。
|
||||
func resolveDatabaseFile() string {
|
||||
dbFile := flags.DatabaseFile
|
||||
if dbFile == "" {
|
||||
dbFile = "./data/komari.db"
|
||||
}
|
||||
return dbFile
|
||||
}
|
||||
|
||||
// backupOnVersionUpgrade 在检测到版本升级时,把当前 ./data 打包到
|
||||
// ./data/backup/upgrade-{time}.zip,便于升级(含 metrics 迁移)异常时回滚。
|
||||
//
|
||||
// 版本标识存放于配置库(configs 表,键 system_version),因此本函数必须在
|
||||
// config.SetDb 之后、一次性 metrics 迁移(InitStores)之前调用。
|
||||
//
|
||||
// 触发规则:
|
||||
// - versionID 为空:跳过(未注入版本,如部分测试场景)。
|
||||
// - 配置中无版本且启动前无数据库文件:全新安装,仅写版本,不备份。
|
||||
// - 配置中无版本但启动前已有数据库文件:从无版本标记的旧稳定版升级,备份。
|
||||
// - 配置中版本与当前不同:版本升级,备份。
|
||||
// - 配置中版本与当前一致:无需备份。
|
||||
//
|
||||
// 备份失败不阻止启动,但打印明确错误;备份成功(或无需备份)后写入/更新版本。
|
||||
func backupOnVersionUpgrade() {
|
||||
if versionID == "" {
|
||||
return
|
||||
}
|
||||
|
||||
prevVersion, readErr := config.GetAs[string](SystemVersionKey)
|
||||
prevVersion = strings.TrimSpace(prevVersion)
|
||||
versionRecorded := readErr == nil && prevVersion != ""
|
||||
|
||||
// 版本未变化,无需备份。
|
||||
if versionRecorded && prevVersion == versionID {
|
||||
return
|
||||
}
|
||||
|
||||
// 全新安装:配置中无版本且启动前无数据库文件,直接写版本不备份。
|
||||
if !versionRecorded && !dbFileExistedAtStartup {
|
||||
writeVersionMarker()
|
||||
return
|
||||
}
|
||||
|
||||
// 需要备份(升级或从旧稳定版首次带版本标记启动)。
|
||||
// 先做一次 WAL checkpoint,确保 komari.db 主文件包含最新数据,
|
||||
// 避免备份出的库缺少仍留在 -wal 中的写入。
|
||||
if instance != nil {
|
||||
instance.Exec("PRAGMA wal_checkpoint(TRUNCATE);")
|
||||
}
|
||||
|
||||
backupDir := filepath.Join(".", "data", "backup")
|
||||
if err := os.MkdirAll(backupDir, 0755); err != nil {
|
||||
logger.Errorf("dbcore", "[upgrade-backup] failed to create backup dir: %v", err)
|
||||
return
|
||||
}
|
||||
tsName := time.Now().UTC().Format("20060102-150405")
|
||||
bakPath := filepath.Join(backupDir, fmt.Sprintf("upgrade-%s.zip", tsName))
|
||||
backupZipPath := filepath.Join(".", "data", "backup.zip")
|
||||
if zipErr := zipDirectoryExcluding("./data", bakPath, map[string]struct{}{backupZipPath: {}, backupDir: {}}); zipErr != nil {
|
||||
logger.Errorf("dbcore", "[upgrade-backup] failed to backup ./data before upgrade (from %q to %q): %v", prevVersion, versionID, zipErr)
|
||||
return
|
||||
}
|
||||
logger.Infof("dbcore", "[upgrade-backup] ./data backed up to %s before upgrade (from %q to %q)", bakPath, prevVersion, versionID)
|
||||
|
||||
writeVersionMarker()
|
||||
}
|
||||
|
||||
// writeVersionMarker 将当前 versionID 写入配置库。
|
||||
func writeVersionMarker() {
|
||||
if err := config.Set(SystemVersionKey, versionID); err != nil {
|
||||
logger.Errorf("dbcore", "[upgrade-backup] failed to persist version marker: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func buildSQLiteDSN(databaseFile string) string {
|
||||
if databaseFile == "" {
|
||||
databaseFile = "./data/komari.db"
|
||||
}
|
||||
|
||||
params := fmt.Sprintf("_busy_timeout=%d&_txlock=immediate", mainSQLiteBusyTimeout.Milliseconds())
|
||||
separator := "?"
|
||||
if strings.Contains(databaseFile, "?") {
|
||||
separator = "&"
|
||||
}
|
||||
|
||||
if strings.HasPrefix(databaseFile, "file:") {
|
||||
return databaseFile + separator + params
|
||||
}
|
||||
|
||||
if databaseFile == ":memory:" {
|
||||
return "file::memory:?cache=shared&" + params
|
||||
}
|
||||
|
||||
return "file:" + filepath.ToSlash(databaseFile) + separator + params
|
||||
}
|
||||
|
||||
func mainSQLiteOptions() sqlitetune.Options {
|
||||
return sqlitetune.Options{
|
||||
BusyTimeout: mainSQLiteBusyTimeout,
|
||||
CacheSizeKB: mainSQLiteCacheSizeKB,
|
||||
MMapSizeBytes: 0,
|
||||
TempStoreMemory: false,
|
||||
CacheSpill: true,
|
||||
WALAutoCheckpoint: mainSQLiteWALAutoCheckpoint,
|
||||
JournalSizeLimitBytes: mainSQLiteJournalSizeLimit,
|
||||
Synchronous: sqlitetune.SynchronousNormal,
|
||||
}
|
||||
}
|
||||
|
||||
// Initialize 显式初始化数据库连接与表结构,仅执行一次。
|
||||
// 与 GetDBInstance 不同,Initialize 返回错误而非直接退出进程,
|
||||
// 便于启动生命周期统一处理错误、以及在测试/CLI 命令中做隔离。
|
||||
func Initialize() error {
|
||||
once.Do(func() {
|
||||
|
||||
var err error
|
||||
|
||||
// 在数据库初始化前执行:如果存在 ./data/backup.zip,则进行恢复逻辑
|
||||
func() {
|
||||
backupZipPath := filepath.Join(".", "data", "backup.zip")
|
||||
if _, statErr := os.Stat(backupZipPath); statErr == nil {
|
||||
// 4. 把除了 ./data/backup.zip 之外的所有文件压缩到 ./backup/{time}.zip
|
||||
if err := os.MkdirAll("./backup", 0755); err != nil {
|
||||
log.Printf("[restore] failed to create backup dir: %v", err)
|
||||
} else {
|
||||
tsName := time.Now().Format("20060102-150405")
|
||||
bakPath := filepath.Join("./backup", fmt.Sprintf("%s.zip", tsName))
|
||||
if zipErr := zipDirectoryExcluding("./data", bakPath, map[string]struct{}{backupZipPath: {}}); zipErr != nil {
|
||||
log.Printf("[restore] failed to zip current data: %v", zipErr)
|
||||
} else {
|
||||
log.Printf("[restore] current data zipped to %s", bakPath)
|
||||
}
|
||||
}
|
||||
|
||||
// 5. 删除除了 ./data/backup.zip 之外的所有文件
|
||||
if delErr := removeAllInDirExcept("./data", map[string]struct{}{backupZipPath: {}}); delErr != nil {
|
||||
log.Printf("[restore] failed to cleanup data dir: %v", delErr)
|
||||
}
|
||||
|
||||
// 6. 解压 ./data/backup.zip 到 ./data
|
||||
if unzipErr := unzipToDir(backupZipPath, "./data"); unzipErr != nil {
|
||||
log.Printf("[restore] failed to unzip backup into data: %v", unzipErr)
|
||||
} else {
|
||||
log.Printf("[restore] backup.zip extracted to ./data")
|
||||
}
|
||||
|
||||
// 7. 删除 ./data/backup.zip
|
||||
if rmErr := os.Remove(backupZipPath); rmErr != nil {
|
||||
log.Printf("[restore] failed to remove backup.zip: %v", rmErr)
|
||||
} else {
|
||||
log.Printf("[restore] backup.zip removed")
|
||||
}
|
||||
// 8. 删除标记
|
||||
if rmErr := os.Remove("./data/komari-backup-markup"); rmErr != nil {
|
||||
log.Printf("[restore] failed to remove komari-backup-markup: %v", rmErr)
|
||||
} else {
|
||||
log.Printf("[restore] komari-backup-markup removed")
|
||||
}
|
||||
}
|
||||
}()
|
||||
|
||||
logConfig := &gorm.Config{
|
||||
Logger: logutil.NewGormLogger(),
|
||||
}
|
||||
|
||||
// 根据数据库类型选择不同的连接方式
|
||||
switch flags.ApplyDatabaseTypeNormalization() {
|
||||
case flags.DatabaseTypeSQLite:
|
||||
// SQLite 连接
|
||||
instance, err = gorm.Open(sqlite.Open(flags.DatabaseFile), logConfig)
|
||||
if err != nil {
|
||||
log.Fatalf("Failed to connect to SQLite3 database: %v", err)
|
||||
}
|
||||
log.Printf("Using SQLite database file: %s", flags.DatabaseFile)
|
||||
instance.Exec("PRAGMA wal = ON;")
|
||||
if err := instance.Exec("PRAGMA journal_mode = WAL;").Error; err != nil {
|
||||
log.Printf("Failed to enable WAL mode for SQLite: %v", err)
|
||||
}
|
||||
instance.Exec("PRAGMA synchronous = NORMAL;")
|
||||
instance.Exec("PRAGMA cache_size = -65536;")
|
||||
instance.Exec("PRAGMA temp_store = MEMORY;")
|
||||
instance.Exec("PRAGMA wal_checkpoint(TRUNCATE);")
|
||||
default:
|
||||
log.Fatalf("Unsupported database type: %s (supported: %s)", flags.DatabaseType, flags.SupportedDatabaseTypes())
|
||||
}
|
||||
if err := migrations.Run(migrations.Context{DB: instance}); err != nil {
|
||||
log.Fatalf("Failed to run startup migrations: %v", err)
|
||||
}
|
||||
config.SetDb(instance)
|
||||
// 自动迁移模型
|
||||
err = instance.AutoMigrate(
|
||||
&models.User{},
|
||||
&models.Client{},
|
||||
&models.Record{},
|
||||
&models.GPURecord{},
|
||||
&models.Log{},
|
||||
&models.Clipboard{},
|
||||
&models.LoadNotification{},
|
||||
&models.OfflineNotification{},
|
||||
&models.TrafficReportNotification{},
|
||||
&models.PingRecord{},
|
||||
&models.PingTask{},
|
||||
&models.OidcProvider{},
|
||||
&models.MessageSenderProvider{},
|
||||
&models.ThemeConfiguration{},
|
||||
)
|
||||
if err != nil {
|
||||
log.Fatalf("Failed to create tables: %v", err)
|
||||
}
|
||||
err = instance.Table("records_long_term").AutoMigrate(
|
||||
&models.Record{},
|
||||
)
|
||||
if err != nil {
|
||||
log.Printf("Failed to create records_long_term table, it may already exist: %v", err)
|
||||
}
|
||||
err = instance.Table("gpu_records_long_term").AutoMigrate(
|
||||
&models.GPURecord{},
|
||||
)
|
||||
if err != nil {
|
||||
log.Printf("Failed to create gpu_records_long_term table, it may already exist: %v", err)
|
||||
}
|
||||
err = instance.AutoMigrate(
|
||||
&models.Session{},
|
||||
)
|
||||
if err != nil {
|
||||
log.Printf("Failed to create Session table, it may already exist: %v", err)
|
||||
}
|
||||
err = instance.AutoMigrate(
|
||||
&models.Task{},
|
||||
&models.TaskResult{},
|
||||
)
|
||||
if err != nil {
|
||||
log.Printf("Failed to create Task and TaskResult table, it may already exist: %v", err)
|
||||
}
|
||||
|
||||
// Manually create composite indexes
|
||||
if flags.IsSQLite() {
|
||||
instance.Exec("CREATE INDEX IF NOT EXISTS idx_record_client_time ON records(client, time)")
|
||||
instance.Exec("CREATE INDEX IF NOT EXISTS idx_record_lt_client_time ON records_long_term(client, time)")
|
||||
instance.Exec("CREATE INDEX IF NOT EXISTS idx_gpu_record_client_time ON gpu_records(client, time)")
|
||||
instance.Exec("CREATE INDEX IF NOT EXISTS idx_gpu_record_lt_client_time ON gpu_records_long_term(client, time)")
|
||||
instance.Exec("CREATE INDEX IF NOT EXISTS idx_ping_record_client_time ON ping_records(client, time)")
|
||||
}
|
||||
|
||||
initErr = doInitialize()
|
||||
})
|
||||
return initErr
|
||||
}
|
||||
|
||||
// GetDBInstance 返回全局数据库实例。
|
||||
// 为兼容既有大量调用点,这里保留“出错即退出”的语义;
|
||||
// 需要错误处理的启动流程应优先调用 Initialize()。
|
||||
func GetDBInstance() *gorm.DB {
|
||||
if err := Initialize(); err != nil {
|
||||
logger.Fatalf("dbcore", "Failed to initialize database: %v", err)
|
||||
}
|
||||
return instance
|
||||
}
|
||||
|
||||
// Close 关闭底层数据库连接,供关闭流程调用。
|
||||
func Close() error {
|
||||
if instance == nil {
|
||||
return nil
|
||||
}
|
||||
sqlDB, err := instance.DB()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return sqlDB.Close()
|
||||
}
|
||||
|
||||
func doInitialize() error {
|
||||
var err error
|
||||
|
||||
// 在数据库初始化前执行:如果存在 ./data/backup.zip,则进行恢复逻辑
|
||||
func() {
|
||||
backupZipPath := filepath.Join(".", "data", "backup.zip")
|
||||
if _, statErr := os.Stat(backupZipPath); statErr == nil {
|
||||
// 4. 将当前数据快照保存到 ./data/backup/,并保留已有归档。
|
||||
backupDir := filepath.Join(".", "data", "backup")
|
||||
if err := os.MkdirAll(backupDir, 0755); err != nil {
|
||||
logger.Errorf("dbcore", "[restore] failed to create backup dir: %v", err)
|
||||
} else {
|
||||
tsName := time.Now().UTC().Format("20060102-150405")
|
||||
bakPath := filepath.Join(backupDir, fmt.Sprintf("pre-restore-%s.zip", tsName))
|
||||
if zipErr := zipDirectoryExcluding("./data", bakPath, map[string]struct{}{backupZipPath: {}, backupDir: {}}); zipErr != nil {
|
||||
logger.Errorf("dbcore", "[restore] failed to zip current data: %v", zipErr)
|
||||
} else {
|
||||
logger.Infof("dbcore", "[restore] current data zipped to %s", bakPath)
|
||||
}
|
||||
}
|
||||
|
||||
// 5. 删除数据文件,但保留归档目录和待恢复的 backup.zip。
|
||||
if delErr := removeAllInDirExcept("./data", map[string]struct{}{backupZipPath: {}, backupDir: {}}); delErr != nil {
|
||||
logger.Errorf("dbcore", "[restore] failed to cleanup data dir: %v", delErr)
|
||||
}
|
||||
|
||||
// 6. 解压 ./data/backup.zip 到 ./data
|
||||
if unzipErr := unzipToDir(backupZipPath, "./data"); unzipErr != nil {
|
||||
logger.Errorf("dbcore", "[restore] failed to unzip backup into data: %v", unzipErr)
|
||||
} else {
|
||||
logger.Infof("dbcore", "[restore] backup.zip extracted to ./data")
|
||||
}
|
||||
|
||||
// 7. 删除 ./data/backup.zip
|
||||
if rmErr := os.Remove(backupZipPath); rmErr != nil {
|
||||
logger.Errorf("dbcore", "[restore] failed to remove backup.zip: %v", rmErr)
|
||||
} else {
|
||||
logger.Infof("dbcore", "[restore] backup.zip removed")
|
||||
}
|
||||
// 8. 删除标记
|
||||
if rmErr := os.Remove("./data/komari-backup-markup"); rmErr != nil {
|
||||
logger.Errorf("dbcore", "[restore] failed to remove komari-backup-markup: %v", rmErr)
|
||||
} else {
|
||||
logger.Infof("dbcore", "[restore] komari-backup-markup removed")
|
||||
}
|
||||
}
|
||||
}()
|
||||
|
||||
// 记录“打开数据库之前”komari.db 是否已存在,用于区分全新安装与旧版升级。
|
||||
// 必须在(可能的)恢复逻辑之后、gorm.Open 之前采集:恢复会解压出旧库,
|
||||
// gorm.Open 会创建空库。
|
||||
if _, statErr := os.Stat(resolveDatabaseFile()); statErr == nil {
|
||||
dbFileExistedAtStartup = true
|
||||
}
|
||||
|
||||
logConfig := &gorm.Config{
|
||||
Logger: logger.NewGormLogger(),
|
||||
NowFunc: func() time.Time { return time.Now().UTC() },
|
||||
}
|
||||
|
||||
// 根据数据库类型选择不同的连接方式
|
||||
switch flags.ApplyDatabaseTypeNormalization() {
|
||||
case flags.DatabaseTypeSQLite:
|
||||
// _txlock=immediate lets writes acquire their lock before reads can
|
||||
// turn into a lock-upgrade conflict. sqlitetune applies the remaining
|
||||
// per-connection PRAGMAs whenever database/sql opens a connection.
|
||||
dsn := buildSQLiteDSN(flags.DatabaseFile)
|
||||
sqlDB, dbErr := sqlitetune.Open(dsn, mainSQLiteOptions())
|
||||
if dbErr != nil {
|
||||
return fmt.Errorf("open SQLite connection pool: %w", dbErr)
|
||||
}
|
||||
// SQLite has one writer. Keeping exactly one durable connection also
|
||||
// keeps connection-local cache and WAL limits stable for the main DB.
|
||||
sqlDB.SetMaxOpenConns(1)
|
||||
sqlDB.SetMaxIdleConns(1)
|
||||
sqlDB.SetConnMaxLifetime(0)
|
||||
sqlDB.SetConnMaxIdleTime(0)
|
||||
|
||||
instance, err = gorm.Open(sqlite.New(sqlite.Config{Conn: sqlDB}), logConfig)
|
||||
if err != nil {
|
||||
_ = sqlDB.Close()
|
||||
return fmt.Errorf("failed to connect to SQLite3 database: %w", err)
|
||||
}
|
||||
if err := instance.Exec("PRAGMA wal_checkpoint(TRUNCATE)").Error; err != nil {
|
||||
logger.Errorf("dbcore", "Failed to checkpoint SQLite WAL at startup: %v", err)
|
||||
}
|
||||
default:
|
||||
return fmt.Errorf("unsupported database type: %s (supported: %s)", flags.DatabaseType, flags.SupportedDatabaseTypes())
|
||||
}
|
||||
if err := migrations.Run(migrations.Context{DB: instance}); err != nil {
|
||||
return fmt.Errorf("failed to run startup migrations: %w", err)
|
||||
}
|
||||
config.SetDb(instance)
|
||||
|
||||
// 配置库就绪后、执行后续 AutoMigrate 之前:
|
||||
// 基于配置中的版本标记检测升级并自动备份 ./data,便于回滚。
|
||||
backupOnVersionUpgrade()
|
||||
|
||||
// 自动迁移模型
|
||||
//
|
||||
// 注意:负载/GPU/ping 历史监控数据运行期全部走 metric store(默认 SQLite
|
||||
// ./data/metrics.db,或配置的 MySQL/PostgreSQL)。旧的 records /
|
||||
// records_long_term / gpu_records / ping_records 表不再建表、不再写入。
|
||||
// 若升级时旧表仍存在,管理员可通过升级向导显式导入并清理。
|
||||
// models.Record / models.PingRecord / models.GPURecord 结构体仍作为
|
||||
// metric store 的读写 DTO 和旧表导入 DTO 保留在 models 包中。
|
||||
|
||||
err = instance.AutoMigrate(
|
||||
&models.User{},
|
||||
&models.Client{},
|
||||
&models.Log{},
|
||||
&models.Clipboard{},
|
||||
&models.LoadNotification{},
|
||||
&models.OfflineNotification{},
|
||||
&models.TrafficReportNotification{},
|
||||
&models.PingTask{},
|
||||
&models.OidcProvider{},
|
||||
&models.MessageSenderProvider{},
|
||||
&models.ThemeConfiguration{},
|
||||
)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to create tables: %w", err)
|
||||
}
|
||||
if err := instance.AutoMigrate(
|
||||
&models.Session{},
|
||||
); err != nil {
|
||||
logger.Errorf("dbcore", "Failed to create Session table, it may already exist: %v", err)
|
||||
}
|
||||
if err := instance.AutoMigrate(
|
||||
&models.Task{},
|
||||
&models.TaskResult{},
|
||||
); err != nil {
|
||||
logger.Errorf("dbcore", "Failed to create Task and TaskResult table, it may already exist: %v", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,92 @@
|
||||
package dbcore
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"strings"
|
||||
"sync"
|
||||
|
||||
"github.com/komari-monitor/komari/cmd/flags"
|
||||
)
|
||||
|
||||
var maintenanceMu sync.Mutex
|
||||
|
||||
// StorageSize returns the bytes occupied by the main SQLite database and its
|
||||
// WAL/SHM sidecar files.
|
||||
func StorageSize() (int64, error) {
|
||||
if !flags.IsSQLite() {
|
||||
return 0, errors.New("main database size is only available for SQLite")
|
||||
}
|
||||
return sqliteFileSetSize(resolveDatabaseFile())
|
||||
}
|
||||
|
||||
// ReclaimSpace checkpoints the main database WAL and rewrites the SQLite file.
|
||||
func ReclaimSpace(ctx context.Context) error {
|
||||
if !flags.IsSQLite() {
|
||||
return errors.New("main database maintenance is only supported for SQLite")
|
||||
}
|
||||
|
||||
maintenanceMu.Lock()
|
||||
defer maintenanceMu.Unlock()
|
||||
|
||||
db, err := GetDBInstance().DB()
|
||||
if err != nil {
|
||||
return fmt.Errorf("get main database connection: %w", err)
|
||||
}
|
||||
if err := checkpointSQLiteWAL(ctx, db); err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := db.ExecContext(ctx, "VACUUM"); err != nil {
|
||||
return fmt.Errorf("vacuum main database: %w", err)
|
||||
}
|
||||
return checkpointSQLiteWAL(ctx, db)
|
||||
}
|
||||
|
||||
func checkpointSQLiteWAL(ctx context.Context, db *sql.DB) error {
|
||||
var busy, logFrames, checkpointedFrames int
|
||||
if err := db.QueryRowContext(ctx, "PRAGMA wal_checkpoint(TRUNCATE)").Scan(&busy, &logFrames, &checkpointedFrames); err != nil {
|
||||
return fmt.Errorf("checkpoint main database WAL: %w", err)
|
||||
}
|
||||
if busy != 0 {
|
||||
return fmt.Errorf(
|
||||
"checkpoint main database WAL: database is busy (%d log frames, %d checkpointed)",
|
||||
logFrames,
|
||||
checkpointedFrames,
|
||||
)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func sqliteFileSetSize(dsn string) (int64, error) {
|
||||
path := strings.TrimPrefix(strings.TrimSpace(dsn), "file:")
|
||||
if index := strings.IndexByte(path, '?'); index >= 0 {
|
||||
path = path[:index]
|
||||
}
|
||||
if path == "" || path == ":memory:" || strings.Contains(strings.ToLower(dsn), "mode=memory") {
|
||||
return 0, errors.New("database is not backed by a local file")
|
||||
}
|
||||
|
||||
var total int64
|
||||
foundDatabase := false
|
||||
for _, suffix := range []string{"", "-wal", "-shm"} {
|
||||
info, err := os.Stat(path + suffix)
|
||||
switch {
|
||||
case err == nil:
|
||||
total += info.Size()
|
||||
if suffix == "" {
|
||||
foundDatabase = true
|
||||
}
|
||||
case errors.Is(err, os.ErrNotExist):
|
||||
continue
|
||||
default:
|
||||
return 0, fmt.Errorf("stat database file %q: %w", path+suffix, err)
|
||||
}
|
||||
}
|
||||
if !foundDatabase {
|
||||
return 0, fmt.Errorf("database file %q does not exist", path)
|
||||
}
|
||||
return total, nil
|
||||
}
|
||||
@@ -0,0 +1,37 @@
|
||||
package dbcore
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestSQLiteFileSetSizeIncludesWALAndSHM(t *testing.T) {
|
||||
databasePath := filepath.Join(t.TempDir(), "komari.db")
|
||||
files := map[string]int{
|
||||
databasePath: 11,
|
||||
databasePath + "-wal": 7,
|
||||
databasePath + "-shm": 5,
|
||||
}
|
||||
for path, size := range files {
|
||||
if err := os.WriteFile(path, make([]byte, size), 0o600); err != nil {
|
||||
t.Fatalf("write fixture %s: %v", path, err)
|
||||
}
|
||||
}
|
||||
|
||||
got, err := sqliteFileSetSize("file:" + filepath.ToSlash(databasePath) + "?mode=rwc")
|
||||
if err != nil {
|
||||
t.Fatalf("sqliteFileSetSize: %v", err)
|
||||
}
|
||||
if got != 23 {
|
||||
t.Fatalf("size = %d, want 23", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSQLiteFileSetSizeRejectsNonFileDatabase(t *testing.T) {
|
||||
for _, dsn := range []string{":memory:", "file:metrics?MODE=MEMORY&cache=shared"} {
|
||||
if _, err := sqliteFileSetSize(dsn); err == nil {
|
||||
t.Fatalf("sqliteFileSetSize(%q) should fail", dsn)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,99 @@
|
||||
package dbcore
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/komari-monitor/komari/internal/sqlitetune"
|
||||
)
|
||||
|
||||
func TestMainSQLiteConnectorAppliesBoundedSettings(t *testing.T) {
|
||||
databasePath := filepath.Join(t.TempDir(), "komari.db")
|
||||
db, err := sqlitetune.Open(buildSQLiteDSN(databasePath), mainSQLiteOptions())
|
||||
if err != nil {
|
||||
t.Fatalf("open SQLite connector: %v", err)
|
||||
}
|
||||
defer db.Close()
|
||||
|
||||
db.SetMaxOpenConns(1)
|
||||
db.SetMaxIdleConns(1)
|
||||
|
||||
if got := sqlitePragmaInt(t, db, "busy_timeout"); got != mainSQLiteBusyTimeout.Milliseconds() {
|
||||
t.Fatalf("busy_timeout = %d, want %d", got, mainSQLiteBusyTimeout.Milliseconds())
|
||||
}
|
||||
if got := sqlitePragmaInt(t, db, "synchronous"); got != 1 {
|
||||
t.Fatalf("synchronous = %d, want NORMAL (1)", got)
|
||||
}
|
||||
if got := sqlitePragmaInt(t, db, "cache_size"); got != -mainSQLiteCacheSizeKB {
|
||||
t.Fatalf("cache_size = %d, want %d", got, -mainSQLiteCacheSizeKB)
|
||||
}
|
||||
if got := sqlitePragmaInt(t, db, "mmap_size"); got != 0 {
|
||||
t.Fatalf("mmap_size = %d, want 0", got)
|
||||
}
|
||||
if got := sqlitePragmaInt(t, db, "temp_store"); got != 1 {
|
||||
t.Fatalf("temp_store = %d, want FILE (1)", got)
|
||||
}
|
||||
if got := sqlitePragmaInt(t, db, "cache_spill"); got == 0 {
|
||||
t.Fatal("cache_spill is disabled")
|
||||
}
|
||||
if got := sqlitePragmaInt(t, db, "wal_autocheckpoint"); got != mainSQLiteWALAutoCheckpoint {
|
||||
t.Fatalf("wal_autocheckpoint = %d, want %d", got, mainSQLiteWALAutoCheckpoint)
|
||||
}
|
||||
if got := sqlitePragmaInt(t, db, "journal_size_limit"); got != mainSQLiteJournalSizeLimit {
|
||||
t.Fatalf("journal_size_limit = %d, want %d", got, mainSQLiteJournalSizeLimit)
|
||||
}
|
||||
|
||||
var journalMode string
|
||||
if err := db.QueryRow("PRAGMA journal_mode").Scan(&journalMode); err != nil {
|
||||
t.Fatalf("read journal_mode: %v", err)
|
||||
}
|
||||
if !strings.EqualFold(journalMode, "wal") {
|
||||
t.Fatalf("journal_mode = %q, want WAL", journalMode)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMainSQLiteAutoCheckpointKeepsWALBounded(t *testing.T) {
|
||||
databasePath := filepath.Join(t.TempDir(), "komari.db")
|
||||
db, err := sqlitetune.Open(buildSQLiteDSN(databasePath), mainSQLiteOptions())
|
||||
if err != nil {
|
||||
t.Fatalf("open SQLite connector: %v", err)
|
||||
}
|
||||
defer db.Close()
|
||||
|
||||
db.SetMaxOpenConns(1)
|
||||
db.SetMaxIdleConns(1)
|
||||
if _, err := db.Exec("CREATE TABLE checkpoint_test (payload BLOB NOT NULL)"); err != nil {
|
||||
t.Fatalf("create checkpoint test table: %v", err)
|
||||
}
|
||||
|
||||
// This produces well beyond one auto-checkpoint interval. The checkpoint
|
||||
// below reports the WAL frame count observed before its own work begins.
|
||||
for range 1024 {
|
||||
if _, err := db.Exec("INSERT INTO checkpoint_test (payload) VALUES (zeroblob(4096))"); err != nil {
|
||||
t.Fatalf("insert checkpoint test row: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
var busy, logFrames, checkpointedFrames int
|
||||
if err := db.QueryRow("PRAGMA wal_checkpoint(PASSIVE)").Scan(&busy, &logFrames, &checkpointedFrames); err != nil {
|
||||
t.Fatalf("inspect automatic checkpoint: %v", err)
|
||||
}
|
||||
if busy != 0 {
|
||||
t.Fatalf("automatic checkpoint left a busy WAL: %d", busy)
|
||||
}
|
||||
if logFrames >= 2*mainSQLiteWALAutoCheckpoint {
|
||||
t.Fatalf("WAL retained %d frames after %d-frame auto-checkpoint interval", logFrames, mainSQLiteWALAutoCheckpoint)
|
||||
}
|
||||
}
|
||||
|
||||
func sqlitePragmaInt(t *testing.T, db *sql.DB, pragma string) int64 {
|
||||
t.Helper()
|
||||
|
||||
var value int64
|
||||
if err := db.QueryRow("PRAGMA " + pragma).Scan(&value); err != nil {
|
||||
t.Fatalf("read %s: %v", pragma, err)
|
||||
}
|
||||
return value
|
||||
}
|
||||
@@ -0,0 +1,95 @@
|
||||
package metricstore
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
|
||||
"github.com/komari-monitor/komari/pkg/metric"
|
||||
)
|
||||
|
||||
var (
|
||||
ErrStoreNotInitialized = errors.New("metric store not initialized")
|
||||
ErrStoreBusy = errors.New("metric store is busy")
|
||||
)
|
||||
|
||||
// StorageInfo describes the physical storage owned by the active metric store.
|
||||
// Size remains useful even when the store points at an external database because
|
||||
// pkg/metric limits its query to the three tables managed by this Store.
|
||||
type StorageInfo struct {
|
||||
Driver metric.Driver
|
||||
Action metric.MaintenanceAction
|
||||
Size int64
|
||||
}
|
||||
|
||||
// MaintenanceResult keeps measurement failures separate from the maintenance
|
||||
// error. A database may allow table maintenance while denying catalog queries,
|
||||
// and callers should still be able to report that the operation succeeded.
|
||||
type MaintenanceResult struct {
|
||||
Driver metric.Driver
|
||||
Action metric.MaintenanceAction
|
||||
Before int64
|
||||
After int64
|
||||
BeforeSizeError error
|
||||
AfterSizeError error
|
||||
}
|
||||
|
||||
// InspectStorage reads physical storage information while preventing a store
|
||||
// reload from closing the active connection underneath the query.
|
||||
func InspectStorage(ctx context.Context) (StorageInfo, error) {
|
||||
if err := storeOperations.AcquireShared(ctx); err != nil {
|
||||
return StorageInfo{}, fmt.Errorf("wait for metric store operations before inspection: %w", err)
|
||||
}
|
||||
defer storeOperations.ReleaseShared()
|
||||
|
||||
storeMu.RLock()
|
||||
activeStore := store
|
||||
storeMu.RUnlock()
|
||||
if activeStore == nil {
|
||||
return StorageInfo{}, ErrStoreNotInitialized
|
||||
}
|
||||
|
||||
info := StorageInfo{
|
||||
Driver: activeStore.Driver(),
|
||||
Action: activeStore.MaintenanceAction(),
|
||||
}
|
||||
size, err := activeStore.StorageSize(ctx)
|
||||
info.Size = size
|
||||
return info, err
|
||||
}
|
||||
|
||||
// ReclaimSpace performs the driver-specific physical maintenance operation.
|
||||
// It takes the exclusive operation lock, so a table/file rewrite cannot run
|
||||
// concurrently with report writes or compaction.
|
||||
func ReclaimSpace(ctx context.Context) (MaintenanceResult, error) {
|
||||
if !storeOperations.TryAcquire() {
|
||||
storeMu.RLock()
|
||||
defer storeMu.RUnlock()
|
||||
if store == nil {
|
||||
return MaintenanceResult{}, ErrStoreNotInitialized
|
||||
}
|
||||
return MaintenanceResult{
|
||||
Driver: store.Driver(),
|
||||
Action: store.MaintenanceAction(),
|
||||
BeforeSizeError: ErrStoreBusy,
|
||||
AfterSizeError: ErrStoreBusy,
|
||||
}, ErrStoreBusy
|
||||
}
|
||||
defer storeOperations.Release()
|
||||
|
||||
storeMu.RLock()
|
||||
activeStore := store
|
||||
storeMu.RUnlock()
|
||||
if activeStore == nil {
|
||||
return MaintenanceResult{}, ErrStoreNotInitialized
|
||||
}
|
||||
|
||||
result := MaintenanceResult{
|
||||
Driver: activeStore.Driver(),
|
||||
Action: activeStore.MaintenanceAction(),
|
||||
}
|
||||
result.Before, result.BeforeSizeError = activeStore.StorageSize(ctx)
|
||||
maintenanceErr := activeStore.ReclaimSpace(ctx)
|
||||
result.After, result.AfterSizeError = activeStore.StorageSize(ctx)
|
||||
return result, maintenanceErr
|
||||
}
|
||||
@@ -0,0 +1,226 @@
|
||||
package metricstore
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"errors"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/komari-monitor/komari/pkg/metric"
|
||||
)
|
||||
|
||||
func TestInspectAndReclaimStorage(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
s, err := metric.Open(ctx, metric.SQLite(":memory:"))
|
||||
if err != nil {
|
||||
t.Fatalf("open metric store: %v", err)
|
||||
}
|
||||
installTestStore(t, s)
|
||||
|
||||
info, err := InspectStorage(ctx)
|
||||
if err != nil {
|
||||
t.Fatalf("inspect storage: %v", err)
|
||||
}
|
||||
if info.Driver != metric.DriverSQLite || info.Action != metric.MaintenanceVacuum {
|
||||
t.Fatalf("unexpected storage info: %#v", info)
|
||||
}
|
||||
if info.Size != 0 {
|
||||
t.Fatalf("in-memory storage size = %d, want 0", info.Size)
|
||||
}
|
||||
|
||||
result, err := ReclaimSpace(ctx)
|
||||
if err != nil {
|
||||
t.Fatalf("reclaim space: %v", err)
|
||||
}
|
||||
if result.Driver != metric.DriverSQLite || result.Action != metric.MaintenanceVacuum {
|
||||
t.Fatalf("unexpected maintenance result: %#v", result)
|
||||
}
|
||||
if result.BeforeSizeError != nil || result.AfterSizeError != nil {
|
||||
t.Fatalf("unexpected size errors: before=%v after=%v", result.BeforeSizeError, result.AfterSizeError)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRetryMetricWALCheckpointDefersBusyReader(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
path := filepath.Join(t.TempDir(), "checkpoint.db")
|
||||
s, err := metric.Open(ctx, metric.SQLite(path, metric.WithSQLiteWALAutoCheckpoint(1_000_000)))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { _ = s.Close() })
|
||||
if err := s.CreateMetric(ctx, metric.Definition{Name: "before", Type: metric.TypeGauge, RetentionDays: 1}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
reader, err := sql.Open("sqlite3", path)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { _ = reader.Close() })
|
||||
tx, err := reader.BeginTx(ctx, &sql.TxOptions{ReadOnly: true})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var definitions int
|
||||
if err := tx.QueryRowContext(ctx, `SELECT COUNT(*) FROM metric_definitions`).Scan(&definitions); err != nil {
|
||||
_ = tx.Rollback()
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := s.CreateMetric(ctx, metric.Definition{Name: "after", Type: metric.TypeGauge, RetentionDays: 1}); err != nil {
|
||||
_ = tx.Rollback()
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
previousPending := checkpointPending
|
||||
checkpointPending = true
|
||||
t.Cleanup(func() { checkpointPending = previousPending })
|
||||
retryMetricWALCheckpoint(ctx, s)
|
||||
if !checkpointPending {
|
||||
_ = tx.Rollback()
|
||||
t.Fatal("busy reader unexpectedly cleared pending checkpoint")
|
||||
}
|
||||
if err := tx.Rollback(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
retryMetricWALCheckpoint(ctx, s)
|
||||
if checkpointPending {
|
||||
t.Fatal("checkpoint remained pending after reader released")
|
||||
}
|
||||
}
|
||||
|
||||
func TestReclaimSpaceReportsBusyStore(t *testing.T) {
|
||||
s, err := metric.Open(context.Background(), metric.SQLite(":memory:"))
|
||||
if err != nil {
|
||||
t.Fatalf("open metric store: %v", err)
|
||||
}
|
||||
installTestStore(t, s)
|
||||
|
||||
if err := storeOperations.AcquireShared(context.Background()); err != nil {
|
||||
t.Fatalf("acquire shared store operation gate: %v", err)
|
||||
}
|
||||
defer storeOperations.ReleaseShared()
|
||||
|
||||
result, err := ReclaimSpace(context.Background())
|
||||
if !errors.Is(err, ErrStoreBusy) {
|
||||
t.Fatalf("reclaim error = %v, want %v", err, ErrStoreBusy)
|
||||
}
|
||||
if result.Driver != metric.DriverSQLite || result.Action != metric.MaintenanceVacuum {
|
||||
t.Fatalf("busy result lost store metadata: %#v", result)
|
||||
}
|
||||
if !errors.Is(result.BeforeSizeError, ErrStoreBusy) || !errors.Is(result.AfterSizeError, ErrStoreBusy) {
|
||||
t.Fatalf("busy result should mark both measurements unavailable: %#v", result)
|
||||
}
|
||||
compactCtx, cancel := context.WithTimeout(context.Background(), time.Second)
|
||||
defer cancel()
|
||||
if _, err := Compact(compactCtx, time.Now()); err != nil {
|
||||
t.Fatalf("compact should not wait for report writes: %v", err)
|
||||
}
|
||||
if !compactOperations.TryAcquire() {
|
||||
t.Fatal("acquire compact operation gate")
|
||||
}
|
||||
defer compactOperations.Release()
|
||||
if _, err := Compact(context.Background(), time.Now()); !errors.Is(err, ErrCompactInProgress) {
|
||||
t.Fatalf("overlapping compact error = %v, want %v", err, ErrCompactInProgress)
|
||||
}
|
||||
}
|
||||
|
||||
func TestInspectStorageRequiresInitializedStore(t *testing.T) {
|
||||
storeMu.Lock()
|
||||
previous := store
|
||||
store = nil
|
||||
storeMu.Unlock()
|
||||
t.Cleanup(func() {
|
||||
storeMu.Lock()
|
||||
store = previous
|
||||
storeMu.Unlock()
|
||||
})
|
||||
|
||||
if _, err := InspectStorage(context.Background()); !errors.Is(err, ErrStoreNotInitialized) {
|
||||
t.Fatalf("inspect error = %v, want %v", err, ErrStoreNotInitialized)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCloseStoreContextCancelsMigrationBeforeTakingStoreLock(t *testing.T) {
|
||||
s, err := metric.Open(context.Background(), metric.SQLite(":memory:"))
|
||||
if err != nil {
|
||||
t.Fatalf("open metric store: %v", err)
|
||||
}
|
||||
installTestStore(t, s)
|
||||
|
||||
migrationCtx, cancelMigration := context.WithCancel(context.Background())
|
||||
done := make(chan struct{})
|
||||
if !storeOperations.TryAcquire() {
|
||||
t.Fatal("acquire store operation gate")
|
||||
}
|
||||
storeMigMu.Lock()
|
||||
storeMigCancel = cancelMigration
|
||||
storeMigDone = done
|
||||
storeMigMu.Unlock()
|
||||
|
||||
go func() {
|
||||
<-migrationCtx.Done()
|
||||
storeOperations.Release()
|
||||
storeMigMu.Lock()
|
||||
storeMigCancel = nil
|
||||
storeMigDone = nil
|
||||
close(done)
|
||||
storeMigMu.Unlock()
|
||||
}()
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), time.Second)
|
||||
defer cancel()
|
||||
if err := CloseStoreContext(ctx); err != nil {
|
||||
t.Fatalf("close store: %v", err)
|
||||
}
|
||||
if migrationCtx.Err() == nil {
|
||||
t.Fatal("store migration was not canceled before close")
|
||||
}
|
||||
if err := Reload(context.Background()); !errors.Is(err, ErrStoreBusy) {
|
||||
t.Fatalf("reload after close error = %v, want %v", err, ErrStoreBusy)
|
||||
}
|
||||
}
|
||||
|
||||
func TestStoreOperationWaitsRespectContext(t *testing.T) {
|
||||
if !storeOperations.TryAcquire() {
|
||||
t.Fatal("acquire store operation gate")
|
||||
}
|
||||
defer storeOperations.Release()
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
cancel()
|
||||
if _, err := InspectStorage(ctx); !errors.Is(err, context.Canceled) {
|
||||
t.Fatalf("inspect error = %v, want context canceled", err)
|
||||
}
|
||||
if err := Reload(ctx); !errors.Is(err, context.Canceled) {
|
||||
t.Fatalf("reload error = %v, want context canceled", err)
|
||||
}
|
||||
if err := CloseStoreContext(ctx); !errors.Is(err, context.Canceled) {
|
||||
t.Fatalf("close error = %v, want context canceled", err)
|
||||
}
|
||||
}
|
||||
|
||||
func installTestStore(t *testing.T, s *metric.Store) {
|
||||
t.Helper()
|
||||
storeMigMu.Lock()
|
||||
previousClosing := storeClosing
|
||||
storeClosing = false
|
||||
storeMigMu.Unlock()
|
||||
storeMu.Lock()
|
||||
previous := store
|
||||
previousFingerprint := storeFingerprint
|
||||
store = s
|
||||
storeFingerprint = "test|memory"
|
||||
storeMu.Unlock()
|
||||
t.Cleanup(func() {
|
||||
storeMigMu.Lock()
|
||||
storeClosing = previousClosing
|
||||
storeMigMu.Unlock()
|
||||
storeMu.Lock()
|
||||
store = previous
|
||||
storeFingerprint = previousFingerprint
|
||||
storeMu.Unlock()
|
||||
_ = s.Close()
|
||||
})
|
||||
}
|
||||
@@ -0,0 +1,63 @@
|
||||
package metricstore
|
||||
|
||||
const (
|
||||
MetricCPU = "cpu.usage"
|
||||
MetricGPU = "gpu.usage"
|
||||
MetricGPUDeviceUsage = "gpu.device.usage"
|
||||
MetricGPUMem = "gpu.memory.used"
|
||||
MetricGPUMemTotal = "gpu.memory.total"
|
||||
MetricGPUTemp = "gpu.temperature"
|
||||
MetricRAM = "memory.used"
|
||||
MetricSwap = "swap.used"
|
||||
MetricLoad = "load.average"
|
||||
MetricDisk = "disk.used"
|
||||
MetricNetIn = "net.in.rate"
|
||||
MetricNetOut = "net.out.rate"
|
||||
MetricNetTotalUp = "net.total.up"
|
||||
MetricNetTotalDown = "net.total.down"
|
||||
MetricTrafficUp = "traffic.up"
|
||||
MetricTrafficDown = "traffic.down"
|
||||
MetricProcess = "process.count"
|
||||
MetricConnections = "connections.tcp"
|
||||
MetricConnectionsUDP = "connections.udp"
|
||||
MetricPingLatency = "ping.latency_ms"
|
||||
MetricPingLoss = "ping.loss"
|
||||
)
|
||||
|
||||
// loadRecordMetricNames are the entity-level metrics used to reconstruct the
|
||||
// legacy Record response shape.
|
||||
var loadRecordMetricNames = []string{
|
||||
MetricCPU, MetricGPU, MetricRAM, MetricSwap, MetricLoad, MetricDisk, MetricNetIn, MetricNetOut,
|
||||
MetricNetTotalUp, MetricNetTotalDown, MetricTrafficUp, MetricTrafficDown,
|
||||
MetricProcess, MetricConnections, MetricConnectionsUDP,
|
||||
}
|
||||
|
||||
var obsoleteBuiltinMetricNames = []string{
|
||||
"memory.total",
|
||||
"swap.total",
|
||||
"temperature",
|
||||
"disk.total",
|
||||
}
|
||||
|
||||
// gpuDeviceRecordMetricNames are stored separately from the entity-level GPU
|
||||
// average and are included when deleting all system records.
|
||||
var gpuDeviceRecordMetricNames = []string{
|
||||
MetricGPUDeviceUsage, MetricGPUMem, MetricGPUMemTotal, MetricGPUTemp,
|
||||
}
|
||||
|
||||
var recordMetricNames = joinMetricNames(loadRecordMetricNames, gpuDeviceRecordMetricNames)
|
||||
|
||||
// Ping has an independent retention and cleanup boundary.
|
||||
var pingMetricNames = []string{MetricPingLatency, MetricPingLoss}
|
||||
|
||||
func joinMetricNames(groups ...[]string) []string {
|
||||
total := 0
|
||||
for _, group := range groups {
|
||||
total += len(group)
|
||||
}
|
||||
names := make([]string, 0, total)
|
||||
for _, group := range groups {
|
||||
names = append(names, group...)
|
||||
}
|
||||
return names
|
||||
}
|
||||
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,568 @@
|
||||
package metricstore
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"errors"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/komari-monitor/komari/database/models"
|
||||
"github.com/komari-monitor/komari/pkg/metric"
|
||||
v1 "github.com/komari-monitor/komari/protocol/v1"
|
||||
)
|
||||
|
||||
func TestDefaultRollupPolicy(t *testing.T) {
|
||||
policy := defaultRollupPolicy()
|
||||
if err := policy.Validate(); err != nil {
|
||||
t.Fatalf("default rollup policy should validate: %v", err)
|
||||
}
|
||||
if policy.RawRetention != DefaultRollupRawRetention {
|
||||
t.Fatalf("raw retention = %s, want %s", policy.RawRetention, DefaultRollupRawRetention)
|
||||
}
|
||||
if len(policy.Tiers) != 4 {
|
||||
t.Fatalf("expected 4 rollup tiers, got %d", len(policy.Tiers))
|
||||
}
|
||||
|
||||
wantIntervals := []time.Duration{time.Minute, 5 * time.Minute, time.Hour, 24 * time.Hour}
|
||||
wantRetentions := []time.Duration{600 * time.Minute, 600 * 5 * time.Minute, 600 * time.Hour, 100 * 365 * 24 * time.Hour}
|
||||
for i := range wantIntervals {
|
||||
if policy.Tiers[i].Interval != wantIntervals[i] {
|
||||
t.Fatalf("tier %d interval = %s, want %s", i, policy.Tiers[i].Interval, wantIntervals[i])
|
||||
}
|
||||
if policy.Tiers[i].Retention != wantRetentions[i] {
|
||||
t.Fatalf("tier %d retention = %s, want %s", i, policy.Tiers[i].Retention, wantRetentions[i])
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildMetricConfigEnablesDefaultRollupPolicy(t *testing.T) {
|
||||
cfg, err := buildMetricConfig(&MetricStoreConfig{
|
||||
Driver: "sqlite",
|
||||
DSN: ":memory:",
|
||||
TablePrefix: "metric_",
|
||||
}, false)
|
||||
if err != nil {
|
||||
t.Fatalf("build metric config: %v", err)
|
||||
}
|
||||
if !cfg.RollupPolicy.Enabled() {
|
||||
t.Fatal("expected default rollup policy to be enabled")
|
||||
}
|
||||
if cfg.RollupPolicy.RawRetention != DefaultRollupRawRetention {
|
||||
t.Fatalf("raw retention = %s, want %s", cfg.RollupPolicy.RawRetention, DefaultRollupRawRetention)
|
||||
}
|
||||
if cfg.SQLite.ReadPoolSize != 2 {
|
||||
t.Fatalf("metric store read pool = %d, want fixed size 2", cfg.SQLite.ReadPoolSize)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildMetricConfigLeavesFinalRetentionToMetricDefinition(t *testing.T) {
|
||||
cfg, err := buildMetricConfig(&MetricStoreConfig{
|
||||
Driver: "sqlite",
|
||||
DSN: ":memory:",
|
||||
}, false)
|
||||
if err != nil {
|
||||
t.Fatalf("build metric config: %v", err)
|
||||
}
|
||||
wantRollupRetention := 100 * 365 * 24 * time.Hour
|
||||
lastTier := cfg.RollupPolicy.Tiers[len(cfg.RollupPolicy.Tiers)-1]
|
||||
if lastTier.Retention != wantRollupRetention {
|
||||
t.Fatalf("rollup retention = %s, want %s", lastTier.Retention, wantRollupRetention)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildMetricConfigAlwaysEnablesDownsampling(t *testing.T) {
|
||||
cfg, err := buildMetricConfig(&MetricStoreConfig{
|
||||
Driver: "sqlite",
|
||||
DSN: ":memory:",
|
||||
}, false)
|
||||
if err != nil {
|
||||
t.Fatalf("build metric config: %v", err)
|
||||
}
|
||||
if !cfg.RollupPolicy.Enabled() {
|
||||
t.Fatal("expected rollup policy to be enabled")
|
||||
}
|
||||
}
|
||||
|
||||
func TestGetPingRecordsReadsRollupsAfterRawCompaction(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
now := time.Now().UTC().Truncate(time.Second)
|
||||
s, err := metric.Open(ctx, metric.SQLite(":memory:",
|
||||
metric.WithMaxOpenConns(1),
|
||||
metric.WithRollupPolicy(defaultRollupPolicy()),
|
||||
))
|
||||
if err != nil {
|
||||
t.Fatalf("open metric store: %v", err)
|
||||
}
|
||||
defer s.Close()
|
||||
if err := s.UpsertMetric(ctx, metric.Definition{
|
||||
Name: MetricPingLatency,
|
||||
Type: metric.TypeGauge,
|
||||
RetentionDays: 30,
|
||||
}); err != nil {
|
||||
t.Fatalf("create ping metric: %v", err)
|
||||
}
|
||||
if err := s.WriteBatch(ctx, []metric.Point{
|
||||
{MetricName: MetricPingLatency, EntityID: "node-a", Timestamp: now.Add(-20 * time.Minute), Value: 20, Tags: map[string]string{"task_id": "7"}},
|
||||
{MetricName: MetricPingLatency, EntityID: "node-a", Timestamp: now.Add(-10 * time.Minute), Value: 10, Tags: map[string]string{"task_id": "7"}},
|
||||
{MetricName: MetricPingLatency, EntityID: "node-a", Timestamp: now.Add(-5 * time.Minute), Value: 5, Tags: map[string]string{"task_id": "7"}},
|
||||
}); err != nil {
|
||||
t.Fatalf("write ping points: %v", err)
|
||||
}
|
||||
if _, err := s.Compact(ctx, now); err != nil {
|
||||
t.Fatalf("compact ping points: %v", err)
|
||||
}
|
||||
|
||||
storeMu.Lock()
|
||||
oldStore := store
|
||||
store = s
|
||||
storeMu.Unlock()
|
||||
defer func() {
|
||||
storeMu.Lock()
|
||||
store = oldStore
|
||||
storeMu.Unlock()
|
||||
}()
|
||||
|
||||
records, err := GetPingRecords(ctx, "node-a", 7, now.Add(-30*time.Minute), now)
|
||||
if err != nil {
|
||||
t.Fatalf("get ping records: %v", err)
|
||||
}
|
||||
if len(records) != 3 {
|
||||
t.Fatalf("expected 3 ping records across raw and rollup data, got %d: %#v", len(records), records)
|
||||
}
|
||||
if records[0].Value != 5 || records[1].Value != 10 || records[2].Value != 20 {
|
||||
t.Fatalf("unexpected ping values in descending order: %#v", records)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCreateMetricDefinitionsUsesExplicitRetentionAndPreservesOverrides(t *testing.T) {
|
||||
if defaultBuiltinMetricRetentionDays != 1 {
|
||||
t.Fatalf("default built-in metric retention = %d, want 1 day", defaultBuiltinMetricRetentionDays)
|
||||
}
|
||||
|
||||
ctx := context.Background()
|
||||
s, err := metric.Open(ctx, metric.SQLite(":memory:", metric.WithMaxOpenConns(1)))
|
||||
if err != nil {
|
||||
t.Fatalf("open metric store: %v", err)
|
||||
}
|
||||
defer s.Close()
|
||||
|
||||
if err := createMetricDefinitions(ctx, s); err != nil {
|
||||
t.Fatalf("create definitions: %v", err)
|
||||
}
|
||||
defs, err := s.ListMetrics(ctx)
|
||||
if err != nil {
|
||||
t.Fatalf("list definitions: %v", err)
|
||||
}
|
||||
if len(defs) != 21 {
|
||||
t.Fatalf("definition count = %d, want 21", len(defs))
|
||||
}
|
||||
for _, def := range defs {
|
||||
if def.RetentionDays != defaultBuiltinMetricRetentionDays {
|
||||
t.Fatalf("%s retention = %d, want %d", def.Name, def.RetentionDays, defaultBuiltinMetricRetentionDays)
|
||||
}
|
||||
}
|
||||
|
||||
cpu, err := s.GetMetric(ctx, MetricCPU)
|
||||
if err != nil {
|
||||
t.Fatalf("get cpu definition: %v", err)
|
||||
}
|
||||
cpu.RetentionDays = 60
|
||||
if err := s.UpsertMetric(ctx, cpu); err != nil {
|
||||
t.Fatalf("override cpu retention: %v", err)
|
||||
}
|
||||
if err := createMetricDefinitions(ctx, s); err != nil {
|
||||
t.Fatalf("recreate definitions: %v", err)
|
||||
}
|
||||
cpu, err = s.GetMetric(ctx, MetricCPU)
|
||||
if err != nil {
|
||||
t.Fatalf("reload cpu definition: %v", err)
|
||||
}
|
||||
if cpu.RetentionDays != 60 {
|
||||
t.Fatalf("cpu retention = %d, want preserved override 60", cpu.RetentionDays)
|
||||
}
|
||||
if _, err := s.SetMetricRetention(ctx, MetricCPU, 0); err != nil {
|
||||
t.Fatalf("disable cpu retention: %v", err)
|
||||
}
|
||||
if err := createMetricDefinitions(ctx, s); err != nil {
|
||||
t.Fatalf("refresh disabled definition: %v", err)
|
||||
}
|
||||
cpu, err = s.GetMetric(ctx, MetricCPU)
|
||||
if err != nil {
|
||||
t.Fatalf("reload disabled cpu definition: %v", err)
|
||||
}
|
||||
if cpu.RetentionDays != 0 {
|
||||
t.Fatalf("cpu retention = %d, want preserved disabled state", cpu.RetentionDays)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCreateMetricDefinitionsRemovesObsoleteMetrics(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
s, err := metric.Open(ctx, metric.SQLite(":memory:", metric.WithMaxOpenConns(1)))
|
||||
if err != nil {
|
||||
t.Fatalf("open metric store: %v", err)
|
||||
}
|
||||
defer s.Close()
|
||||
if err := s.CreateMetric(ctx, metric.Definition{Name: "memory.total", Type: metric.TypeGauge, RetentionDays: 1}); err != nil {
|
||||
t.Fatalf("create obsolete definition: %v", err)
|
||||
}
|
||||
if err := s.Write(ctx, metric.Point{MetricName: "memory.total", EntityID: "node-a", Timestamp: time.Now().UTC(), Value: 1024}); err != nil {
|
||||
t.Fatalf("write obsolete point: %v", err)
|
||||
}
|
||||
if err := createMetricDefinitions(ctx, s); err != nil {
|
||||
t.Fatalf("refresh built-in definitions: %v", err)
|
||||
}
|
||||
if _, err := s.GetMetric(ctx, "memory.total"); !errors.Is(err, metric.ErrNotFound) {
|
||||
t.Fatalf("obsolete definition error = %v, want ErrNotFound", err)
|
||||
}
|
||||
points, err := s.Query(ctx, metric.Query{MetricName: "memory.total", EntityID: "node-a", Start: time.Now().UTC().Add(-time.Hour), End: time.Now().UTC().Add(time.Hour)})
|
||||
if err != nil {
|
||||
t.Fatalf("query obsolete points: %v", err)
|
||||
}
|
||||
if len(points) != 0 {
|
||||
t.Fatalf("obsolete points remain: %#v", points)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCreateMetricDefinitionsUsesLegacySpanOnlyForNewDefinitions(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
s, err := metric.Open(ctx, metric.SQLite(":memory:", metric.WithMaxOpenConns(1)))
|
||||
if err != nil {
|
||||
t.Fatalf("open metric store: %v", err)
|
||||
}
|
||||
defer s.Close()
|
||||
|
||||
if err := createMetricDefinitionsWithDefaultRetention(ctx, s, 10); err != nil {
|
||||
t.Fatalf("create migration definitions: %v", err)
|
||||
}
|
||||
defs, err := s.ListMetrics(ctx)
|
||||
if err != nil {
|
||||
t.Fatalf("list migration definitions: %v", err)
|
||||
}
|
||||
for _, def := range defs {
|
||||
if def.RetentionDays != 10 {
|
||||
t.Fatalf("%s retention = %d, want legacy span 10", def.Name, def.RetentionDays)
|
||||
}
|
||||
}
|
||||
|
||||
cpu, err := s.GetMetric(ctx, MetricCPU)
|
||||
if err != nil {
|
||||
t.Fatalf("get CPU definition: %v", err)
|
||||
}
|
||||
cpu.RetentionDays = 3
|
||||
if err := s.UpsertMetric(ctx, cpu); err != nil {
|
||||
t.Fatalf("override CPU retention: %v", err)
|
||||
}
|
||||
if err := createMetricDefinitionsWithDefaultRetention(ctx, s, 20); err != nil {
|
||||
t.Fatalf("refresh migration definitions: %v", err)
|
||||
}
|
||||
cpu, err = s.GetMetric(ctx, MetricCPU)
|
||||
if err != nil {
|
||||
t.Fatalf("reload CPU definition: %v", err)
|
||||
}
|
||||
if cpu.RetentionDays != 3 {
|
||||
t.Fatalf("existing CPU retention = %d, want preserved 3", cpu.RetentionDays)
|
||||
}
|
||||
}
|
||||
|
||||
func TestGetRetentionSummaryUsesAllMetricDefinitions(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
s, err := metric.Open(ctx, metric.SQLite(":memory:", metric.WithMaxOpenConns(1)))
|
||||
if err != nil {
|
||||
t.Fatalf("open metric store: %v", err)
|
||||
}
|
||||
defer s.Close()
|
||||
|
||||
storeMu.Lock()
|
||||
oldStore := store
|
||||
store = s
|
||||
storeMu.Unlock()
|
||||
defer func() {
|
||||
storeMu.Lock()
|
||||
store = oldStore
|
||||
storeMu.Unlock()
|
||||
}()
|
||||
|
||||
empty, err := GetRetentionSummary(ctx)
|
||||
if err != nil {
|
||||
t.Fatalf("summarize empty store: %v", err)
|
||||
}
|
||||
if empty.AllPositive || empty.MaxDays != 0 {
|
||||
t.Fatalf("unexpected empty summary: %#v", empty)
|
||||
}
|
||||
for _, def := range []metric.Definition{
|
||||
{Name: "short", Type: metric.TypeGauge, RetentionDays: 7},
|
||||
{Name: "long", Type: metric.TypeGauge, RetentionDays: 60},
|
||||
} {
|
||||
if err := s.UpsertMetric(ctx, def); err != nil {
|
||||
t.Fatalf("upsert %s: %v", def.Name, err)
|
||||
}
|
||||
}
|
||||
summary, err := GetRetentionSummary(ctx)
|
||||
if err != nil {
|
||||
t.Fatalf("summarize definitions: %v", err)
|
||||
}
|
||||
if !summary.AllPositive || summary.MaxDays != 60 {
|
||||
t.Fatalf("unexpected summary: %#v", summary)
|
||||
}
|
||||
if _, err := s.SetMetricRetention(ctx, "short", 0); err != nil {
|
||||
t.Fatalf("disable short metric: %v", err)
|
||||
}
|
||||
summary, err = GetRetentionSummary(ctx)
|
||||
if err != nil {
|
||||
t.Fatalf("summarize disabled metric: %v", err)
|
||||
}
|
||||
if summary.AllPositive || summary.MaxDays != 60 {
|
||||
t.Fatalf("unexpected disabled summary: %#v", summary)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSummarizeRetentionDefinitionsRequiresEveryMetricToBePositive(t *testing.T) {
|
||||
summary := summarizeRetentionDefinitions([]metric.Definition{
|
||||
{Name: "enabled", RetentionDays: 30},
|
||||
{Name: "disabled", RetentionDays: 0},
|
||||
{Name: "long", RetentionDays: 60},
|
||||
})
|
||||
if summary.AllPositive || summary.MaxDays != 60 {
|
||||
t.Fatalf("unexpected summary: %#v", summary)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCompactCleansPointsOutsideFixedRawWindow(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
s, err := metric.Open(ctx, metric.SQLite(":memory:", metric.WithMaxOpenConns(1), metric.WithRollupPolicy(metric.RollupPolicy{})))
|
||||
if err != nil {
|
||||
t.Fatalf("open metric store: %v", err)
|
||||
}
|
||||
if err := s.UpsertMetric(ctx, metric.Definition{
|
||||
Name: "raw.metric",
|
||||
Type: metric.TypeGauge,
|
||||
RetentionDays: 1,
|
||||
}); err != nil {
|
||||
t.Fatalf("upsert metric: %v", err)
|
||||
}
|
||||
|
||||
now := time.Now().UTC()
|
||||
if err := s.WriteBatch(ctx, []metric.Point{
|
||||
{MetricName: "raw.metric", EntityID: "node", Timestamp: now.Add(-11 * time.Minute), Value: 1},
|
||||
{MetricName: "raw.metric", EntityID: "node", Timestamp: now.Add(-30 * time.Second), Value: 2},
|
||||
}); err != nil {
|
||||
t.Fatalf("write points: %v", err)
|
||||
}
|
||||
|
||||
storeMu.Lock()
|
||||
oldStore := store
|
||||
oldCompactAt := compactAt
|
||||
store = s
|
||||
compactAt = 0
|
||||
storeMu.Unlock()
|
||||
defer func() {
|
||||
storeMu.Lock()
|
||||
store = oldStore
|
||||
compactAt = oldCompactAt
|
||||
storeMu.Unlock()
|
||||
_ = s.Close()
|
||||
}()
|
||||
|
||||
if _, err := Compact(ctx, now); err != nil {
|
||||
t.Fatalf("compact: %v", err)
|
||||
}
|
||||
points, err := s.Query(ctx, metric.Query{
|
||||
MetricName: "raw.metric",
|
||||
EntityID: "node",
|
||||
Start: now.Add(-time.Hour),
|
||||
End: now,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("query points: %v", err)
|
||||
}
|
||||
if len(points) != 1 || points[0].Value != 2 {
|
||||
t.Fatalf("expected only the retained raw point, got %#v", points)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCompactContinuesAfterOneMetricFails(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
dsn := filepath.Join(t.TempDir(), "compact.db")
|
||||
s, err := metric.Open(ctx, metric.SQLite(dsn,
|
||||
metric.WithMaxOpenConns(1),
|
||||
metric.WithRollupPolicy(defaultRollupPolicy()),
|
||||
))
|
||||
if err != nil {
|
||||
t.Fatalf("open metric store: %v", err)
|
||||
}
|
||||
t.Cleanup(func() { _ = s.Close() })
|
||||
for _, name := range []string{"a.invalid", "b.healthy"} {
|
||||
if err := s.CreateMetric(ctx, metric.Definition{Name: name, Type: metric.TypeGauge, RetentionDays: 1}); err != nil {
|
||||
t.Fatalf("create metric %s: %v", name, err)
|
||||
}
|
||||
}
|
||||
|
||||
now := time.Date(2026, 7, 20, 12, 0, 0, 0, time.UTC)
|
||||
old := now.Add(-48 * time.Hour)
|
||||
if err := s.WriteBatch(ctx, []metric.Point{
|
||||
{MetricName: "a.invalid", EntityID: "node", Timestamp: old, Value: 1},
|
||||
{MetricName: "b.healthy", EntityID: "node", Timestamp: old, Value: 2},
|
||||
}); err != nil {
|
||||
t.Fatalf("write compact fixtures: %v", err)
|
||||
}
|
||||
|
||||
rawDB, err := sql.Open("sqlite3", dsn)
|
||||
if err != nil {
|
||||
t.Fatalf("open raw sqlite connection: %v", err)
|
||||
}
|
||||
_, err = rawDB.ExecContext(ctx, `CREATE TRIGGER fail_invalid_metric_rollup
|
||||
BEFORE DELETE ON metric_rollups
|
||||
WHEN OLD.series_id IN (SELECT id FROM metric_series WHERE metric_name = 'a.invalid')
|
||||
BEGIN SELECT RAISE(FAIL, 'forced compact failure'); END`)
|
||||
_ = rawDB.Close()
|
||||
if err != nil {
|
||||
t.Fatalf("create compact failure trigger: %v", err)
|
||||
}
|
||||
|
||||
storeMu.Lock()
|
||||
previousStore := store
|
||||
previousCompactAt := compactAt
|
||||
store = s
|
||||
compactAt = 0
|
||||
storeMu.Unlock()
|
||||
t.Cleanup(func() {
|
||||
storeMu.Lock()
|
||||
store = previousStore
|
||||
compactAt = previousCompactAt
|
||||
storeMu.Unlock()
|
||||
})
|
||||
|
||||
if _, err := Compact(ctx, now); err == nil {
|
||||
t.Fatal("expected malformed metric to fail compaction")
|
||||
}
|
||||
points, err := s.Query(ctx, metric.Query{
|
||||
MetricName: "b.healthy",
|
||||
EntityID: "node",
|
||||
Start: old.Add(-time.Minute),
|
||||
End: now,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("query healthy rollups: %v", err)
|
||||
}
|
||||
if len(points) != 0 {
|
||||
t.Fatalf("healthy metric was blocked by another metric failure: %#v", points)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCompactKeepsRotatingCursorAfterFullCycle(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
s, err := metric.Open(ctx, metric.SQLite(":memory:",
|
||||
metric.WithMaxOpenConns(1),
|
||||
metric.WithRollupPolicy(defaultRollupPolicy()),
|
||||
))
|
||||
if err != nil {
|
||||
t.Fatalf("open metric store: %v", err)
|
||||
}
|
||||
for _, name := range []string{"a.metric", "b.metric", "c.metric"} {
|
||||
if err := s.UpsertMetric(ctx, metric.Definition{Name: name, Type: metric.TypeGauge}); err != nil {
|
||||
t.Fatalf("upsert metric %s: %v", name, err)
|
||||
}
|
||||
}
|
||||
|
||||
storeMu.Lock()
|
||||
oldStore := store
|
||||
oldCompactAt := compactAt
|
||||
store = s
|
||||
compactAt = 1
|
||||
storeMu.Unlock()
|
||||
defer func() {
|
||||
storeMu.Lock()
|
||||
store = oldStore
|
||||
compactAt = oldCompactAt
|
||||
storeMu.Unlock()
|
||||
_ = s.Close()
|
||||
}()
|
||||
|
||||
if _, err := Compact(ctx, time.Date(2026, 7, 11, 12, 0, 0, 0, time.UTC)); err != nil {
|
||||
t.Fatalf("compact: %v", err)
|
||||
}
|
||||
if compactAt != 1 {
|
||||
t.Fatalf("compact cursor = %d, want 1 after a complete rotated cycle", compactAt)
|
||||
}
|
||||
}
|
||||
|
||||
func TestGetRecordsByClientAndTimeReadsRollupsAfterRawCompaction(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
s, err := metric.Open(ctx, metric.SQLite(":memory:",
|
||||
metric.WithMaxOpenConns(1),
|
||||
metric.WithRollupPolicy(defaultRollupPolicy()),
|
||||
))
|
||||
if err != nil {
|
||||
t.Fatalf("open metric store: %v", err)
|
||||
}
|
||||
if err := createMetricDefinitions(ctx, s); err != nil {
|
||||
t.Fatalf("create metric definitions: %v", err)
|
||||
}
|
||||
|
||||
storeMu.Lock()
|
||||
oldStore := store
|
||||
oldCompactAt := compactAt
|
||||
store = s
|
||||
compactAt = 0
|
||||
storeMu.Unlock()
|
||||
defer func() {
|
||||
storeMu.Lock()
|
||||
store = oldStore
|
||||
compactAt = oldCompactAt
|
||||
storeMu.Unlock()
|
||||
_ = s.Close()
|
||||
}()
|
||||
|
||||
now := time.Now().UTC().Truncate(time.Minute)
|
||||
ts := now.Add(-time.Hour)
|
||||
rec := models.Record{
|
||||
Client: "node-a",
|
||||
Time: ts,
|
||||
Cpu: 42.5,
|
||||
Ram: 123456,
|
||||
RamTotal: 999999,
|
||||
Disk: 456789,
|
||||
DiskTotal: 777777,
|
||||
Load: 0.75,
|
||||
Connections: 321,
|
||||
ConnectionsUdp: 12,
|
||||
}
|
||||
if _, err := WriteReport(ctx, v1.Report{
|
||||
UUID: rec.Client,
|
||||
UpdatedAt: ts,
|
||||
CPU: v1.CPUReport{Usage: float64(rec.Cpu)},
|
||||
Ram: v1.RamReport{Used: rec.Ram, Total: rec.RamTotal},
|
||||
Load: v1.LoadReport{Load1: float64(rec.Load)},
|
||||
Disk: v1.DiskReport{Used: rec.Disk, Total: rec.DiskTotal},
|
||||
Process: rec.Process,
|
||||
Connections: v1.ConnectionsReport{
|
||||
TCP: rec.Connections,
|
||||
UDP: rec.ConnectionsUdp,
|
||||
},
|
||||
}); err != nil {
|
||||
t.Fatalf("write record: %v", err)
|
||||
}
|
||||
if _, err := s.Compact(ctx, now); err != nil {
|
||||
t.Fatalf("compact raw into rollup: %v", err)
|
||||
}
|
||||
got, err := GetRecordsByClientAndTime(ctx, rec.Client, ts.Add(-time.Minute), now)
|
||||
if err != nil {
|
||||
t.Fatalf("get records: %v", err)
|
||||
}
|
||||
if len(got) != 1 {
|
||||
t.Fatalf("expected 1 reconstructed record from rollup, got %d: %#v", len(got), got)
|
||||
}
|
||||
if got[0].Cpu == 0 || got[0].Ram == 0 || got[0].Disk == 0 || got[0].Connections == 0 {
|
||||
t.Fatalf("record was not reconstructed from rollup: %#v", got[0])
|
||||
}
|
||||
|
||||
all, err := GetRecordsByTime(ctx, ts.Add(-time.Minute), now)
|
||||
if err != nil {
|
||||
t.Fatalf("get all records: %v", err)
|
||||
}
|
||||
if len(all) != 1 || all[0].Client != rec.Client || all[0].Cpu == 0 {
|
||||
t.Fatalf("all-client records were not reconstructed from rollup: %#v", all)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,149 @@
|
||||
package metricstore
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
"github.com/komari-monitor/komari/pkg/metric"
|
||||
logger "github.com/komari-monitor/komari/utils/log"
|
||||
)
|
||||
|
||||
const storeMigrationBatchSize = 500
|
||||
|
||||
// configFromFingerprint 从目标指纹(driver|dsn)重建一个 MetricStoreConfig,
|
||||
// 用于以只读方式打开“上一次的 metrics 目标库”。表前缀、保留天数、连接数等
|
||||
// 沿用当前配置(切换后端时这些通常不变)。
|
||||
func configFromFingerprint(fingerprint string, base *MetricStoreConfig) (*MetricStoreConfig, error) {
|
||||
idx := strings.Index(fingerprint, "|")
|
||||
if idx < 0 {
|
||||
return nil, fmt.Errorf("invalid target fingerprint: %q", fingerprint)
|
||||
}
|
||||
driver := fingerprint[:idx]
|
||||
dsn := fingerprint[idx+1:]
|
||||
if strings.TrimSpace(driver) == "" {
|
||||
return nil, fmt.Errorf("empty driver in target fingerprint: %q", fingerprint)
|
||||
}
|
||||
return &MetricStoreConfig{
|
||||
Driver: driver,
|
||||
DSN: dsn,
|
||||
TablePrefix: base.TablePrefix,
|
||||
MaxOpenConns: base.MaxOpenConns,
|
||||
MaxIdleConns: base.MaxIdleConns,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// openSourceStore 打开一个已存在的 metrics 目标库作为数据搬运的源库读取。
|
||||
//
|
||||
// 使用 autoMigrate=true 确保规范化表和索引存在;创建语句不会删除已有数据。
|
||||
// 当源库文件/表不存在(例如老快照记录了 completed 但 metrics.db 缺失)时,
|
||||
// 创建空表可让后续 ListMetrics 返回空集,把“无历史可迁移”识别为正常情况。
|
||||
func openSourceStore(ctx context.Context, cfg *MetricStoreConfig) (*metric.Store, error) {
|
||||
metricCfg, err := buildMetricConfig(cfg, true)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return metric.Open(ctx, metricCfg)
|
||||
}
|
||||
|
||||
// defaultSQLiteFingerprint 返回默认 SQLite metrics 库(./data/metrics.db)的目标指纹。
|
||||
// 老快照的 metrics 数据固定落在该 SQLite 文件,用于在缺失指纹时推断上一个源库。
|
||||
func defaultSQLiteFingerprint() string {
|
||||
return targetFingerprint(&MetricStoreConfig{Driver: "sqlite", DSN: "./data/metrics.db"})
|
||||
}
|
||||
|
||||
// migrateFromPreviousStore 打开由 prevFingerprint 指定的上一个 metrics 目标库作为源,
|
||||
// 把其中的全部指标定义和持久化 rollup 搬运到当前目标 dst(例如 SQLite
|
||||
// metrics.db → MySQL/PostgreSQL)。
|
||||
//
|
||||
// 该过程幂等:dst 以 (series, resolution, labels, bucket) upsert 写入,中断后重启可安全重跑。
|
||||
func migrateFromPreviousStore(prevFingerprint string, cfg *MetricStoreConfig, dst *metric.Store) error {
|
||||
prevCfg, err := configFromFingerprint(prevFingerprint, cfg)
|
||||
if err != nil {
|
||||
return fmt.Errorf("parse previous metrics target %q: %w", prevFingerprint, err)
|
||||
}
|
||||
|
||||
ctx := context.Background()
|
||||
src, err := openSourceStore(ctx, prevCfg)
|
||||
if err != nil {
|
||||
return fmt.Errorf("open previous metrics store (%s): %w", prevFingerprint, err)
|
||||
}
|
||||
defer src.Close()
|
||||
|
||||
logger.Infof("metricstore", "Migrating metrics from previous store %s to current target...", prevFingerprint)
|
||||
total, err := MigrateBetweenStores(ctx, src, dst)
|
||||
if err != nil {
|
||||
return fmt.Errorf("store-to-store metrics migration failed: %w", err)
|
||||
}
|
||||
logger.Infof("metricstore", "Store-to-store metrics migration completed (%d rollups)", total)
|
||||
return nil
|
||||
}
|
||||
|
||||
// storeMigrationObserver 在 store-to-store 迁移过程中接收进度回调。
|
||||
// - currentMetric:当前正在搬运的指标名。
|
||||
// - metricIndex:该指标在全部指标中的序号(0 起),即已完成的指标数。
|
||||
// - totalMetrics:指标定义总数。
|
||||
// - addedPoints:本次新写入目标库的 rollup 行数(用于外部累计)。
|
||||
type storeMigrationObserver func(currentMetric string, metricIndex, totalMetrics int, addedPoints int64)
|
||||
|
||||
// MigrateBetweenStores 把 src metric store 的全部指标定义与持久化 rollup 搬运到 dst。
|
||||
//
|
||||
// 用于 metrics 后端切换(例如默认 SQLite metrics.db → MySQL/PostgreSQL):
|
||||
// - 先在 dst 建立/更新全部指标定义;
|
||||
// - 再按指标流式读取源库 rollup 并分批写入 dst。
|
||||
//
|
||||
// rollup 在 dst 以 (series, resolution, labels, bucket) upsert 写入,因此整个
|
||||
// 过程可安全重试。返回搬运的 rollup 行数。
|
||||
func MigrateBetweenStores(ctx context.Context, src, dst *metric.Store) (int64, error) {
|
||||
return migrateBetweenStores(ctx, src, dst, nil)
|
||||
}
|
||||
|
||||
// migrateBetweenStores 是 MigrateBetweenStores 的内部实现,额外接受一个可选的进度
|
||||
// 观察者 observe(为 nil 时行为与旧版本完全一致)。WebUI/API 手动触发的迁移传入
|
||||
// 回调以实时更新进度。
|
||||
func migrateBetweenStores(ctx context.Context, src, dst *metric.Store, observe storeMigrationObserver) (int64, error) {
|
||||
if src == nil || dst == nil {
|
||||
return 0, fmt.Errorf("source or destination metric store is nil")
|
||||
}
|
||||
|
||||
defs, err := src.ListMetrics(ctx)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("list source metrics: %w", err)
|
||||
}
|
||||
|
||||
var total int64
|
||||
for i, def := range defs {
|
||||
if observe != nil {
|
||||
// 进入下一个指标:已完成 i 个指标。
|
||||
observe(def.Name, i, len(defs), 0)
|
||||
}
|
||||
// 目标库先建立指标定义,保证后续写入的指标存在。
|
||||
if err := dst.UpsertMetric(ctx, def); err != nil {
|
||||
return total, fmt.Errorf("upsert metric %q on target: %w", def.Name, err)
|
||||
}
|
||||
if def.RetentionDays == 0 {
|
||||
continue
|
||||
}
|
||||
var migrated int64
|
||||
_, err := src.ExportRollups(ctx, def.Name, storeMigrationBatchSize, func(batch []metric.PersistedRollup) error {
|
||||
if err := dst.ImportRollups(ctx, batch); err != nil {
|
||||
return fmt.Errorf("write metric %q rollup batch to target: %w", def.Name, err)
|
||||
}
|
||||
count := int64(len(batch))
|
||||
migrated += count
|
||||
total += count
|
||||
if observe != nil {
|
||||
observe(def.Name, i, len(defs), count)
|
||||
}
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
return total, fmt.Errorf("export metric %q rollups: %w", def.Name, err)
|
||||
}
|
||||
if migrated > 0 {
|
||||
logger.Infof("metricstore", "[store-migration] metric %q: migrated %d rollups", def.Name, migrated)
|
||||
}
|
||||
}
|
||||
|
||||
return total, nil
|
||||
}
|
||||
@@ -0,0 +1,152 @@
|
||||
package metricstore
|
||||
|
||||
import (
|
||||
"context"
|
||||
"path/filepath"
|
||||
"reflect"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/komari-monitor/komari/pkg/metric"
|
||||
)
|
||||
|
||||
func TestMigrateBetweenStoresCopiesPersistedRollupsAfterReopen(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
policy := defaultRollupPolicy()
|
||||
sourcePath := filepath.Join(t.TempDir(), "source.db")
|
||||
source, err := metric.Open(ctx, metric.SQLite(sourcePath,
|
||||
metric.WithMaxOpenConns(1),
|
||||
metric.WithMaxIdleConns(1),
|
||||
metric.WithRollupPolicy(policy),
|
||||
))
|
||||
if err != nil {
|
||||
t.Fatalf("open source: %v", err)
|
||||
}
|
||||
|
||||
definitions := []metric.Definition{
|
||||
{Name: "a.metric", Description: "migrated definition", Type: metric.TypeGauge, Unit: "%", RetentionDays: 30, Metadata: map[string]string{"owner": "test"}},
|
||||
{Name: "z.empty", Type: metric.TypeCounter, Unit: "bytes", RetentionDays: 7, Metadata: map[string]string{}},
|
||||
}
|
||||
for _, definition := range definitions {
|
||||
if err := source.UpsertMetric(ctx, definition); err != nil {
|
||||
t.Fatalf("upsert source metric %q: %v", definition.Name, err)
|
||||
}
|
||||
}
|
||||
|
||||
base := time.Date(2026, 7, 26, 12, 0, 0, 0, time.UTC)
|
||||
digest := metric.NewTDigest(policy.Compression)
|
||||
for _, value := range []float64{10, 20, 30} {
|
||||
digest.Add(value, 1)
|
||||
}
|
||||
sourceRollups := []metric.PersistedRollup{
|
||||
{
|
||||
MetricName: "a.metric", EntityID: "agent-1", Tags: map[string]string{"device": "cpu0"}, Labels: map[string]string{"zone": "ap"},
|
||||
Resolution: time.Minute, Bucket: base, Count: 3, Sum: 60, SumSq: 1400, Min: 10, Max: 30,
|
||||
FirstValue: 10, FirstTime: base.Add(time.Second), LastValue: 30, LastTime: base.Add(40 * time.Second),
|
||||
Digest: digest.Encode(), CreatedAt: base.Add(time.Hour),
|
||||
},
|
||||
{
|
||||
MetricName: "a.metric", EntityID: "agent-2", Tags: map[string]string{}, Labels: map[string]string{"zone": "eu"},
|
||||
Resolution: time.Hour, Bucket: base, Count: 12, Sum: 60, SumSq: 300, Min: 5, Max: 5,
|
||||
FirstValue: 5, FirstTime: base.Add(time.Minute), LastValue: 5, LastTime: base.Add(50 * time.Minute),
|
||||
CreatedAt: base.Add(time.Hour),
|
||||
},
|
||||
}
|
||||
if err := source.ImportRollups(ctx, sourceRollups); err != nil {
|
||||
t.Fatalf("seed source rollups: %v", err)
|
||||
}
|
||||
if err := source.Close(); err != nil {
|
||||
t.Fatalf("close source: %v", err)
|
||||
}
|
||||
|
||||
source, err = metric.Open(ctx, metric.SQLite(sourcePath,
|
||||
metric.WithMaxOpenConns(1),
|
||||
metric.WithMaxIdleConns(1),
|
||||
metric.WithRollupPolicy(policy),
|
||||
))
|
||||
if err != nil {
|
||||
t.Fatalf("reopen source: %v", err)
|
||||
}
|
||||
defer source.Close()
|
||||
raw, err := source.Query(ctx, metric.Query{
|
||||
MetricName: "a.metric", Start: base.Add(-time.Hour), End: base.Add(2 * time.Hour),
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("query reopened raw window: %v", err)
|
||||
}
|
||||
if len(raw) != 0 {
|
||||
t.Fatalf("reopened source raw points = %d, want 0", len(raw))
|
||||
}
|
||||
|
||||
target, err := metric.Open(ctx, metric.SQLite(filepath.Join(t.TempDir(), "target.db"),
|
||||
metric.WithMaxOpenConns(1),
|
||||
metric.WithMaxIdleConns(1),
|
||||
metric.WithRollupPolicy(policy),
|
||||
))
|
||||
if err != nil {
|
||||
t.Fatalf("open target: %v", err)
|
||||
}
|
||||
defer target.Close()
|
||||
|
||||
type progressEvent struct {
|
||||
metricName string
|
||||
metricIndex, totalMetrics int
|
||||
added int64
|
||||
}
|
||||
var events []progressEvent
|
||||
total, err := migrateBetweenStores(ctx, source, target, func(name string, index, definitions int, added int64) {
|
||||
events = append(events, progressEvent{name, index, definitions, added})
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("migrate stores: %v", err)
|
||||
}
|
||||
if total != int64(len(sourceRollups)) {
|
||||
t.Fatalf("migrated rollups = %d, want %d", total, len(sourceRollups))
|
||||
}
|
||||
wantEvents := []progressEvent{
|
||||
{metricName: "a.metric", metricIndex: 0, totalMetrics: 2},
|
||||
{metricName: "a.metric", metricIndex: 0, totalMetrics: 2, added: 2},
|
||||
{metricName: "z.empty", metricIndex: 1, totalMetrics: 2},
|
||||
}
|
||||
if !reflect.DeepEqual(events, wantEvents) {
|
||||
t.Fatalf("migration events = %#v, want %#v", events, wantEvents)
|
||||
}
|
||||
|
||||
for _, want := range definitions {
|
||||
got, err := target.GetMetric(ctx, want.Name)
|
||||
if err != nil {
|
||||
t.Fatalf("get target metric %q: %v", want.Name, err)
|
||||
}
|
||||
if got.Name != want.Name || got.Description != want.Description || got.Type != want.Type || got.Unit != want.Unit || got.RetentionDays != want.RetentionDays || !reflect.DeepEqual(got.Metadata, want.Metadata) {
|
||||
t.Fatalf("target definition = %#v, want mutable fields from %#v", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
exported := exportMigrationTestRollups(t, target, "a.metric")
|
||||
if !reflect.DeepEqual(exported, sourceRollups) {
|
||||
t.Fatalf("target rollups differ:\n got: %#v\nwant: %#v", exported, sourceRollups)
|
||||
}
|
||||
secondTotal, err := MigrateBetweenStores(ctx, source, target)
|
||||
if err != nil {
|
||||
t.Fatalf("repeat migration: %v", err)
|
||||
}
|
||||
if secondTotal != total {
|
||||
t.Fatalf("repeat migrated rollups = %d, want %d", secondTotal, total)
|
||||
}
|
||||
if got := exportMigrationTestRollups(t, target, "a.metric"); !reflect.DeepEqual(got, sourceRollups) {
|
||||
t.Fatalf("repeat migration changed target rollups: %#v", got)
|
||||
}
|
||||
}
|
||||
|
||||
func exportMigrationTestRollups(t *testing.T, store *metric.Store, metricName string) []metric.PersistedRollup {
|
||||
t.Helper()
|
||||
var result []metric.PersistedRollup
|
||||
_, err := store.ExportRollups(context.Background(), metricName, 1, func(batch []metric.PersistedRollup) error {
|
||||
result = append(result, batch...)
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("export %q rollups: %v", metricName, err)
|
||||
}
|
||||
return result
|
||||
}
|
||||
@@ -0,0 +1,40 @@
|
||||
package metricstore
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"golang.org/x/sync/semaphore"
|
||||
)
|
||||
|
||||
// storeOperationGate keeps an active Store stable while allowing regular
|
||||
// reads, report writes, and compaction to proceed together. Store replacement
|
||||
// and maintenance take the whole gate exclusively.
|
||||
type storeOperationGate struct {
|
||||
semaphore *semaphore.Weighted
|
||||
}
|
||||
|
||||
const storeOperationGateWeight int64 = 1024
|
||||
|
||||
func newStoreOperationGate() *storeOperationGate {
|
||||
return &storeOperationGate{semaphore: semaphore.NewWeighted(storeOperationGateWeight)}
|
||||
}
|
||||
|
||||
func (g *storeOperationGate) Acquire(ctx context.Context) error {
|
||||
return g.semaphore.Acquire(ctx, storeOperationGateWeight)
|
||||
}
|
||||
|
||||
func (g *storeOperationGate) TryAcquire() bool {
|
||||
return g.semaphore.TryAcquire(storeOperationGateWeight)
|
||||
}
|
||||
|
||||
func (g *storeOperationGate) Release() {
|
||||
g.semaphore.Release(storeOperationGateWeight)
|
||||
}
|
||||
|
||||
func (g *storeOperationGate) AcquireShared(ctx context.Context) error {
|
||||
return g.semaphore.Acquire(ctx, 1)
|
||||
}
|
||||
|
||||
func (g *storeOperationGate) ReleaseShared() {
|
||||
g.semaphore.Release(1)
|
||||
}
|
||||
@@ -0,0 +1,164 @@
|
||||
package metricstore
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"errors"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"github.com/komari-monitor/komari/internal/config"
|
||||
"github.com/komari-monitor/komari/pkg/metric"
|
||||
"gorm.io/driver/sqlite"
|
||||
"gorm.io/gorm"
|
||||
)
|
||||
|
||||
func prepareRecoveryTest(t *testing.T) {
|
||||
t.Helper()
|
||||
configDB, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{})
|
||||
if err != nil {
|
||||
t.Fatalf("open config db: %v", err)
|
||||
}
|
||||
config.SetDb(configDB)
|
||||
|
||||
storeInitMu.Lock()
|
||||
storeMigMu.Lock()
|
||||
previousClosing := storeClosing
|
||||
storeClosing = false
|
||||
storeMigMu.Unlock()
|
||||
storeMu.Lock()
|
||||
previousStore := store
|
||||
previousFingerprint := storeFingerprint
|
||||
store = nil
|
||||
storeFingerprint = ""
|
||||
storeMu.Unlock()
|
||||
storeInitMu.Unlock()
|
||||
|
||||
t.Cleanup(func() {
|
||||
_ = CloseStoreContext(context.Background())
|
||||
storeMigMu.Lock()
|
||||
storeClosing = previousClosing
|
||||
storeMigMu.Unlock()
|
||||
storeMu.Lock()
|
||||
store = previousStore
|
||||
storeFingerprint = previousFingerprint
|
||||
storeMu.Unlock()
|
||||
})
|
||||
}
|
||||
|
||||
func TestRecoverStoreFailureKeepsCurrentConfig(t *testing.T) {
|
||||
prepareRecoveryTest(t)
|
||||
if err := config.SetMany(map[string]any{
|
||||
MetricDBDriverKey: "sqlite",
|
||||
MetricDBDSNKey: "old.db",
|
||||
}); err != nil {
|
||||
t.Fatalf("save original config: %v", err)
|
||||
}
|
||||
|
||||
missing := "file:" + filepath.ToSlash(filepath.Join(t.TempDir(), "missing.db")) + "?mode=ro"
|
||||
err := RecoverStore(context.Background(), &MetricStoreConfig{Driver: "sqlite", DSN: missing})
|
||||
if err == nil {
|
||||
t.Fatal("recovery unexpectedly opened a missing read-only database")
|
||||
}
|
||||
if got, _ := config.GetAs[string](MetricDBDSNKey, ""); got != "old.db" {
|
||||
t.Fatalf("DSN changed after failed recovery: %q", got)
|
||||
}
|
||||
if GetStore() != nil {
|
||||
t.Fatal("failed recovery installed a store")
|
||||
}
|
||||
}
|
||||
|
||||
func TestRecoverStoreRecordsManualMigrationSource(t *testing.T) {
|
||||
prepareRecoveryTest(t)
|
||||
dsn := filepath.ToSlash(filepath.Join(t.TempDir(), "recovered.db"))
|
||||
t.Cleanup(func() { _ = CloseStoreContext(context.Background()) })
|
||||
cfg := &MetricStoreConfig{Driver: "sqlite", DSN: dsn}
|
||||
if err := RecoverStore(context.Background(), cfg); err != nil {
|
||||
t.Fatalf("recover store: %v", err)
|
||||
}
|
||||
if GetStore() == nil {
|
||||
t.Fatal("successful recovery did not install a store")
|
||||
}
|
||||
wantTarget := targetFingerprint(cfg)
|
||||
if got, _ := config.GetAs[string](MigrationTargetKey, ""); got != wantTarget {
|
||||
t.Fatalf("migration target = %q, want %q", got, wantTarget)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRecoverStorePersistsLegacyStoreForStructureGuide(t *testing.T) {
|
||||
prepareRecoveryTest(t)
|
||||
dsn := filepath.ToSlash(filepath.Join(t.TempDir(), "legacy.db"))
|
||||
db, err := sql.Open("sqlite3", dsn)
|
||||
if err != nil {
|
||||
t.Fatalf("open legacy store: %v", err)
|
||||
}
|
||||
if _, err := db.Exec(`CREATE TABLE metric_definitions (
|
||||
name TEXT PRIMARY KEY, type TEXT NOT NULL, unit TEXT NOT NULL,
|
||||
description TEXT NOT NULL, retention_days INTEGER NOT NULL,
|
||||
metadata TEXT NOT NULL, created_at INTEGER NOT NULL, updated_at INTEGER NOT NULL
|
||||
)`); err != nil {
|
||||
_ = db.Close()
|
||||
t.Fatalf("create legacy definition table: %v", err)
|
||||
}
|
||||
if err := db.Close(); err != nil {
|
||||
t.Fatalf("close legacy store: %v", err)
|
||||
}
|
||||
|
||||
cfg := &MetricStoreConfig{Driver: "sqlite", DSN: dsn}
|
||||
if err := RecoverStore(context.Background(), cfg); err != nil {
|
||||
t.Fatalf("recover legacy store: %v", err)
|
||||
}
|
||||
if GetStore() != nil {
|
||||
t.Fatal("legacy recovery installed the store before structure upgrade")
|
||||
}
|
||||
if got, _ := config.GetAs[string](MetricDBDSNKey, ""); got != dsn {
|
||||
t.Fatalf("saved legacy DSN = %q, want %q", got, dsn)
|
||||
}
|
||||
required, err := StructureUpgradeRequired(context.Background())
|
||||
if err != nil {
|
||||
t.Fatalf("detect structure upgrade after recovery: %v", err)
|
||||
}
|
||||
if !required {
|
||||
t.Fatal("legacy recovery did not lead to the structure upgrade guide")
|
||||
}
|
||||
}
|
||||
|
||||
func TestReloadRejectsLegacyStoreForStructureUpgrade(t *testing.T) {
|
||||
prepareRecoveryTest(t)
|
||||
active, err := metric.Open(context.Background(), metric.SQLite(":memory:"))
|
||||
if err != nil {
|
||||
t.Fatalf("open active metric store: %v", err)
|
||||
}
|
||||
installTestStore(t, active)
|
||||
|
||||
dsn := filepath.ToSlash(filepath.Join(t.TempDir(), "legacy.db"))
|
||||
db, err := sql.Open("sqlite3", dsn)
|
||||
if err != nil {
|
||||
t.Fatalf("open legacy store: %v", err)
|
||||
}
|
||||
if _, err := db.Exec(`CREATE TABLE metric_definitions (
|
||||
name TEXT PRIMARY KEY, type TEXT NOT NULL, unit TEXT NOT NULL,
|
||||
description TEXT NOT NULL, retention_days INTEGER NOT NULL,
|
||||
metadata TEXT NOT NULL, created_at INTEGER NOT NULL, updated_at INTEGER NOT NULL
|
||||
)`); err != nil {
|
||||
_ = db.Close()
|
||||
t.Fatalf("create legacy definition table: %v", err)
|
||||
}
|
||||
if err := db.Close(); err != nil {
|
||||
t.Fatalf("close legacy store: %v", err)
|
||||
}
|
||||
if err := config.SetMany(map[string]any{
|
||||
MetricDBDriverKey: "sqlite",
|
||||
MetricDBDSNKey: dsn,
|
||||
}); err != nil {
|
||||
t.Fatalf("save legacy metric store config: %v", err)
|
||||
}
|
||||
|
||||
err = Reload(context.Background())
|
||||
if !errors.Is(err, ErrStructureUpgradeRequired) {
|
||||
t.Fatalf("reload error = %v, want %v", err, ErrStructureUpgradeRequired)
|
||||
}
|
||||
if GetStore() != active {
|
||||
t.Fatal("legacy target replaced the active metric store during hot reload")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,22 @@
|
||||
package metricstore
|
||||
|
||||
import (
|
||||
"regexp"
|
||||
"strings"
|
||||
)
|
||||
|
||||
var (
|
||||
metricCredentialPattern = regexp.MustCompile(`(?i)\b(password|passwd)\s*=\s*('[^']*'|"[^"]*"|[^\s,;)&]+)`)
|
||||
metricURLCredentialPattern = regexp.MustCompile(`(?i)([a-z0-9_.-]+):([^@\s]+)@`)
|
||||
)
|
||||
|
||||
// RedactConnectionError removes the supplied DSN and common password fragments
|
||||
// before a connection error is shown to an administrator or written to a log.
|
||||
func RedactConnectionError(message, dsn string) string {
|
||||
message = strings.TrimSpace(message)
|
||||
if dsn != "" {
|
||||
message = strings.ReplaceAll(message, dsn, "[redacted]")
|
||||
}
|
||||
message = metricCredentialPattern.ReplaceAllString(message, "$1=[redacted]")
|
||||
return metricURLCredentialPattern.ReplaceAllString(message, "$1:[redacted]@")
|
||||
}
|
||||
@@ -0,0 +1,505 @@
|
||||
package metricstore
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
logger "github.com/komari-monitor/komari/utils/log"
|
||||
"strconv"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/komari-monitor/komari/database/models"
|
||||
"github.com/komari-monitor/komari/pkg/metric"
|
||||
v1 "github.com/komari-monitor/komari/protocol/v1"
|
||||
)
|
||||
|
||||
type reportTrafficState struct {
|
||||
mu sync.Mutex
|
||||
|
||||
reportTrafficValues
|
||||
}
|
||||
|
||||
type reportTrafficValues struct {
|
||||
initialized bool
|
||||
timestamp time.Time
|
||||
hasUp bool
|
||||
totalUp int64
|
||||
hasDown bool
|
||||
totalDown int64
|
||||
}
|
||||
|
||||
var reportTrafficStates sync.Map
|
||||
|
||||
const (
|
||||
reportBatchInterval = 3 * time.Second
|
||||
reportBatchQueueSize = 4096
|
||||
pingBatchMaxRecords = 512
|
||||
reportBatchWriteTimeout = 10 * time.Second
|
||||
)
|
||||
|
||||
var (
|
||||
reportBatcherMu sync.Mutex
|
||||
reportBatcher *reportBatchWorker
|
||||
)
|
||||
|
||||
var (
|
||||
ErrReportBatchQueueFull = errors.New("metric report batch queue is full")
|
||||
ErrReportBatchStopped = errors.New("metric report batcher is stopped")
|
||||
ErrPingBatchQueueFull = errors.New("ping batch queue is full")
|
||||
)
|
||||
|
||||
type reportBatchRequest struct {
|
||||
ctx context.Context
|
||||
done chan error
|
||||
stop bool
|
||||
}
|
||||
|
||||
type reportBatchWorker struct {
|
||||
mu sync.Mutex
|
||||
queue chan v1.Report
|
||||
pingQueue chan models.PingRecord
|
||||
requests chan reportBatchRequest
|
||||
done chan struct{}
|
||||
stopping bool
|
||||
}
|
||||
|
||||
// StartReportBatcher starts the shared report writer. Exact samples are kept in
|
||||
// the short raw window while the same writes update in-memory minute buckets.
|
||||
func StartReportBatcher() {
|
||||
reportBatcherMu.Lock()
|
||||
defer reportBatcherMu.Unlock()
|
||||
if reportBatcher != nil {
|
||||
return
|
||||
}
|
||||
worker := &reportBatchWorker{
|
||||
queue: make(chan v1.Report, reportBatchQueueSize),
|
||||
pingQueue: make(chan models.PingRecord, reportBatchQueueSize),
|
||||
requests: make(chan reportBatchRequest, 1),
|
||||
done: make(chan struct{}),
|
||||
}
|
||||
reportBatcher = worker
|
||||
go worker.run()
|
||||
}
|
||||
|
||||
// StopReportBatcher stops the report writer after flushing all queued reports.
|
||||
func StopReportBatcher(ctx context.Context) error {
|
||||
reportBatcherMu.Lock()
|
||||
worker := reportBatcher
|
||||
if worker == nil {
|
||||
reportBatcherMu.Unlock()
|
||||
return nil
|
||||
}
|
||||
worker.mu.Lock()
|
||||
worker.stopping = true
|
||||
worker.mu.Unlock()
|
||||
reportBatcherMu.Unlock()
|
||||
|
||||
request := reportBatchRequest{ctx: ctx, done: make(chan error, 1), stop: true}
|
||||
select {
|
||||
case worker.requests <- request:
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
}
|
||||
select {
|
||||
case err := <-request.done:
|
||||
<-worker.done
|
||||
reportBatcherMu.Lock()
|
||||
if reportBatcher == worker {
|
||||
reportBatcher = nil
|
||||
}
|
||||
reportBatcherMu.Unlock()
|
||||
return err
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
}
|
||||
}
|
||||
|
||||
// FlushReportBatch synchronously flushes the current queue. It is useful for
|
||||
// controlled handoff points and deterministic tests; normal operation uses the
|
||||
// worker ticker and the active mode's flush interval.
|
||||
func FlushReportBatch(ctx context.Context) error {
|
||||
reportBatcherMu.Lock()
|
||||
worker := reportBatcher
|
||||
reportBatcherMu.Unlock()
|
||||
if worker == nil {
|
||||
return nil
|
||||
}
|
||||
request := reportBatchRequest{ctx: ctx, done: make(chan error, 1)}
|
||||
worker.mu.Lock()
|
||||
stopping := worker.stopping
|
||||
worker.mu.Unlock()
|
||||
if stopping {
|
||||
return ErrReportBatchStopped
|
||||
}
|
||||
select {
|
||||
case worker.requests <- request:
|
||||
case <-worker.done:
|
||||
return ErrReportBatchStopped
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
}
|
||||
select {
|
||||
case err := <-request.done:
|
||||
return err
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
}
|
||||
}
|
||||
|
||||
// WriteReport persists one agent report and adds it to in-memory minute
|
||||
// summaries using the same server receive time. Traffic deltas remain summable
|
||||
// after rollup.
|
||||
func WriteReport(ctx context.Context, report v1.Report) (v1.Report, error) {
|
||||
if report.UUID == "" {
|
||||
return v1.Report{}, fmt.Errorf("report UUID is required")
|
||||
}
|
||||
if report.UpdatedAt.IsZero() {
|
||||
return v1.Report{}, fmt.Errorf("report receive time is required")
|
||||
}
|
||||
report.UpdatedAt = report.UpdatedAt.UTC()
|
||||
if GetStore() == nil {
|
||||
return v1.Report{}, fmt.Errorf("metric store not enabled")
|
||||
}
|
||||
|
||||
reportBatcherMu.Lock()
|
||||
worker := reportBatcher
|
||||
reportBatcherMu.Unlock()
|
||||
if worker != nil {
|
||||
if err := worker.enqueue(ctx, report); err != nil {
|
||||
return v1.Report{}, err
|
||||
}
|
||||
return report, nil
|
||||
}
|
||||
|
||||
saved, err := writeReportBatch(ctx, []v1.Report{report})
|
||||
if err != nil {
|
||||
return v1.Report{}, err
|
||||
}
|
||||
return saved[0], nil
|
||||
}
|
||||
|
||||
func (w *reportBatchWorker) enqueue(ctx context.Context, report v1.Report) error {
|
||||
w.mu.Lock()
|
||||
defer w.mu.Unlock()
|
||||
if w.stopping {
|
||||
return ErrReportBatchStopped
|
||||
}
|
||||
if err := ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
select {
|
||||
case w.queue <- report:
|
||||
return nil
|
||||
default:
|
||||
return ErrReportBatchQueueFull
|
||||
}
|
||||
}
|
||||
|
||||
func (w *reportBatchWorker) run() {
|
||||
ticker := time.NewTicker(reportBatchInterval)
|
||||
defer ticker.Stop()
|
||||
|
||||
var pending []v1.Report
|
||||
var pendingPings []models.PingRecord
|
||||
for {
|
||||
select {
|
||||
case request := <-w.requests:
|
||||
pending = append(pending, drainReportQueue(w.queue, reportBatchQueueSize)...)
|
||||
pendingPings = append(pendingPings, drainPingQueue(w.pingQueue, reportBatchQueueSize)...)
|
||||
err := errors.Join(
|
||||
writePendingReports(request.ctx, &pending),
|
||||
writePendingPingRecords(request.ctx, &pendingPings),
|
||||
)
|
||||
if request.stop {
|
||||
if err != nil {
|
||||
logger.Errorf("metricstore", "failed to flush metric report batch during shutdown: %v", err)
|
||||
}
|
||||
close(w.done)
|
||||
request.done <- err
|
||||
return
|
||||
}
|
||||
request.done <- err
|
||||
case <-ticker.C:
|
||||
pendingPings = append(pendingPings, drainPingQueue(w.pingQueue, reportBatchQueueSize)...)
|
||||
if err := writePendingPingRecords(context.Background(), &pendingPings); err != nil {
|
||||
logger.Errorf("metricstore", "failed to flush ping batch: %v", err)
|
||||
}
|
||||
pending = append(pending, drainReportQueue(w.queue, reportBatchQueueSize)...)
|
||||
if err := writePendingReports(context.Background(), &pending); err != nil {
|
||||
logger.Errorf("metricstore", "failed to flush metric report batch: %v", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (w *reportBatchWorker) enqueuePing(ctx context.Context, record models.PingRecord) error {
|
||||
w.mu.Lock()
|
||||
defer w.mu.Unlock()
|
||||
if w.stopping {
|
||||
return ErrReportBatchStopped
|
||||
}
|
||||
if err := ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
select {
|
||||
case w.pingQueue <- record:
|
||||
return nil
|
||||
default:
|
||||
return ErrPingBatchQueueFull
|
||||
}
|
||||
}
|
||||
|
||||
func drainReportQueue(queue <-chan v1.Report, limit int) []v1.Report {
|
||||
if limit <= 0 {
|
||||
return nil
|
||||
}
|
||||
reports := make([]v1.Report, 0, limit)
|
||||
for len(reports) < limit {
|
||||
select {
|
||||
case report := <-queue:
|
||||
reports = append(reports, report)
|
||||
default:
|
||||
return reports
|
||||
}
|
||||
}
|
||||
return reports
|
||||
}
|
||||
|
||||
func drainPingQueue(queue <-chan models.PingRecord, limit int) []models.PingRecord {
|
||||
if limit <= 0 {
|
||||
return nil
|
||||
}
|
||||
records := make([]models.PingRecord, 0, limit)
|
||||
for len(records) < limit {
|
||||
select {
|
||||
case record := <-queue:
|
||||
records = append(records, record)
|
||||
default:
|
||||
return records
|
||||
}
|
||||
}
|
||||
return records
|
||||
}
|
||||
|
||||
func writePendingReports(ctx context.Context, pending *[]v1.Report) error {
|
||||
if len(*pending) == 0 {
|
||||
return nil
|
||||
}
|
||||
batchSize := len(*pending)
|
||||
for len(*pending) > 0 {
|
||||
if batchSize > len(*pending) {
|
||||
batchSize = len(*pending)
|
||||
}
|
||||
writeCtx, cancel := context.WithTimeout(ctx, reportBatchWriteTimeout)
|
||||
_, err := writeReportBatch(writeCtx, (*pending)[:batchSize])
|
||||
cancel()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
*pending = (*pending)[batchSize:]
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func writePendingPingRecords(ctx context.Context, pending *[]models.PingRecord) error {
|
||||
for len(*pending) > 0 {
|
||||
batchSize := len(*pending)
|
||||
if batchSize > pingBatchMaxRecords {
|
||||
batchSize = pingBatchMaxRecords
|
||||
}
|
||||
writeCtx, cancel := context.WithTimeout(ctx, reportBatchWriteTimeout)
|
||||
err := writePingRecords(writeCtx, (*pending)[:batchSize])
|
||||
cancel()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
*pending = (*pending)[batchSize:]
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func writeReportBatch(ctx context.Context, reports []v1.Report) ([]v1.Report, error) {
|
||||
if len(reports) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
if err := storeOperations.AcquireShared(ctx); err != nil {
|
||||
return nil, fmt.Errorf("wait for metric store operation before writing reports: %w", err)
|
||||
}
|
||||
defer storeOperations.ReleaseShared()
|
||||
|
||||
s := GetStore()
|
||||
if s == nil {
|
||||
return nil, fmt.Errorf("metric store not enabled")
|
||||
}
|
||||
|
||||
prepared := make([]v1.Report, len(reports))
|
||||
copy(prepared, reports)
|
||||
points := make([]metric.Point, 0, len(reports)*20)
|
||||
pendingStates := make(map[*reportTrafficState]reportTrafficValues)
|
||||
for i, report := range prepared {
|
||||
stateValue, _ := reportTrafficStates.LoadOrStore(report.UUID, &reportTrafficState{})
|
||||
state := stateValue.(*reportTrafficState)
|
||||
values, ok := pendingStates[state]
|
||||
if !ok {
|
||||
state.mu.Lock()
|
||||
values = state.reportTrafficValues
|
||||
state.mu.Unlock()
|
||||
}
|
||||
if !values.initialized {
|
||||
totalUp, hasUp, err := latestReportCounter(ctx, s, MetricNetTotalUp, report.UUID, report.UpdatedAt)
|
||||
if err == nil {
|
||||
values.totalUp = totalUp
|
||||
values.hasUp = hasUp
|
||||
} else if ctx.Err() == nil {
|
||||
logger.Errorf("metricstore", "failed to restore previous upload counter for %s: %v", report.UUID, err)
|
||||
}
|
||||
totalDown, hasDown, err := latestReportCounter(ctx, s, MetricNetTotalDown, report.UUID, report.UpdatedAt)
|
||||
if err == nil {
|
||||
values.totalDown = totalDown
|
||||
values.hasDown = hasDown
|
||||
} else if ctx.Err() == nil {
|
||||
logger.Errorf("metricstore", "failed to restore previous download counter for %s: %v", report.UUID, err)
|
||||
}
|
||||
if err := ctx.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
values.initialized = true
|
||||
}
|
||||
|
||||
if !values.timestamp.IsZero() && !report.UpdatedAt.After(values.timestamp) {
|
||||
report.UpdatedAt = values.timestamp.Add(time.Millisecond)
|
||||
}
|
||||
trafficUp := int64(0)
|
||||
if values.hasUp {
|
||||
trafficUp = TrafficCounterDelta(report.Network.TotalUp, values.totalUp)
|
||||
}
|
||||
trafficDown := int64(0)
|
||||
if values.hasDown {
|
||||
trafficDown = TrafficCounterDelta(report.Network.TotalDown, values.totalDown)
|
||||
}
|
||||
points = append(points, reportMetricPoints(report, trafficUp, trafficDown)...)
|
||||
values.timestamp = report.UpdatedAt
|
||||
values.hasUp = true
|
||||
values.totalUp = report.Network.TotalUp
|
||||
values.hasDown = true
|
||||
values.totalDown = report.Network.TotalDown
|
||||
pendingStates[state] = values
|
||||
prepared[i] = report
|
||||
}
|
||||
|
||||
if err := s.WriteBatch(ctx, points); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for state, values := range pendingStates {
|
||||
state.mu.Lock()
|
||||
state.reportTrafficValues = values
|
||||
state.mu.Unlock()
|
||||
}
|
||||
return prepared, nil
|
||||
}
|
||||
|
||||
func reportMetricPoints(report v1.Report, trafficUp, trafficDown int64) []metric.Point {
|
||||
entityID := report.UUID
|
||||
ts := report.UpdatedAt
|
||||
points := []metric.Point{
|
||||
{MetricName: MetricCPU, EntityID: entityID, Timestamp: ts, Value: report.CPU.Usage},
|
||||
{MetricName: MetricRAM, EntityID: entityID, Timestamp: ts, Value: float64(report.Ram.Used)},
|
||||
{MetricName: MetricSwap, EntityID: entityID, Timestamp: ts, Value: float64(report.Swap.Used)},
|
||||
{MetricName: MetricLoad, EntityID: entityID, Timestamp: ts, Value: report.Load.Load1},
|
||||
{MetricName: MetricDisk, EntityID: entityID, Timestamp: ts, Value: float64(report.Disk.Used)},
|
||||
{MetricName: MetricNetIn, EntityID: entityID, Timestamp: ts, Value: float64(report.Network.Down)},
|
||||
{MetricName: MetricNetOut, EntityID: entityID, Timestamp: ts, Value: float64(report.Network.Up)},
|
||||
{MetricName: MetricNetTotalUp, EntityID: entityID, Timestamp: ts, Value: float64(report.Network.TotalUp)},
|
||||
{MetricName: MetricNetTotalDown, EntityID: entityID, Timestamp: ts, Value: float64(report.Network.TotalDown)},
|
||||
{MetricName: MetricTrafficUp, EntityID: entityID, Timestamp: ts, Value: float64(trafficUp)},
|
||||
{MetricName: MetricTrafficDown, EntityID: entityID, Timestamp: ts, Value: float64(trafficDown)},
|
||||
{MetricName: MetricProcess, EntityID: entityID, Timestamp: ts, Value: float64(report.Process)},
|
||||
{MetricName: MetricConnections, EntityID: entityID, Timestamp: ts, Value: float64(report.Connections.TCP)},
|
||||
{MetricName: MetricConnectionsUDP, EntityID: entityID, Timestamp: ts, Value: float64(report.Connections.UDP)},
|
||||
}
|
||||
if report.GPU == nil {
|
||||
return points
|
||||
}
|
||||
points = append(points, metric.Point{MetricName: MetricGPU, EntityID: entityID, Timestamp: ts, Value: report.GPU.AverageUsage})
|
||||
for deviceIndex, gpu := range report.GPU.DetailedInfo {
|
||||
tags := map[string]string{
|
||||
"device_index": strconv.Itoa(deviceIndex),
|
||||
"device_name": gpu.Name,
|
||||
}
|
||||
points = append(points,
|
||||
metric.Point{MetricName: MetricGPUMem, EntityID: entityID, Timestamp: ts, Value: float64(gpu.MemoryUsed), Tags: tags},
|
||||
metric.Point{MetricName: MetricGPUMemTotal, EntityID: entityID, Timestamp: ts, Value: float64(gpu.MemoryTotal), Tags: tags},
|
||||
metric.Point{MetricName: MetricGPUDeviceUsage, EntityID: entityID, Timestamp: ts, Value: gpu.Utilization, Tags: tags},
|
||||
metric.Point{MetricName: MetricGPUTemp, EntityID: entityID, Timestamp: ts, Value: float64(gpu.Temperature), Tags: tags},
|
||||
)
|
||||
}
|
||||
return points
|
||||
}
|
||||
|
||||
func latestReportCounter(ctx context.Context, s *metric.Store, metricName, entityID string, before time.Time) (int64, bool, error) {
|
||||
point, ok, err := s.LatestBefore(ctx, metricName, entityID, before)
|
||||
if err != nil {
|
||||
return 0, false, err
|
||||
}
|
||||
if !ok {
|
||||
return 0, false, nil
|
||||
}
|
||||
return int64(point.Value), true, nil
|
||||
}
|
||||
|
||||
// GetLatestTrafficBefore returns the latest retained upload/download counters
|
||||
// before a boundary, transparently reading raw points or rollup summaries.
|
||||
func GetLatestTrafficBefore(ctx context.Context, entityIDs []string, before time.Time) (map[string]models.Record, error) {
|
||||
s := GetStore()
|
||||
if s == nil {
|
||||
return nil, fmt.Errorf("metric store not enabled")
|
||||
}
|
||||
result := make(map[string]models.Record, len(entityIDs))
|
||||
for _, entityID := range entityIDs {
|
||||
if entityID == "" {
|
||||
continue
|
||||
}
|
||||
up, hasUp, err := latestReportCounter(ctx, s, MetricNetTotalUp, entityID, before)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
down, hasDown, err := latestReportCounter(ctx, s, MetricNetTotalDown, entityID, before)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if !hasUp && !hasDown {
|
||||
continue
|
||||
}
|
||||
result[entityID] = models.Record{
|
||||
Client: entityID,
|
||||
Time: before.UTC().Add(-time.Nanosecond),
|
||||
NetTotalUp: up,
|
||||
NetTotalDown: down,
|
||||
}
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
// TrafficCounterDelta returns a reset-aware increase between two cumulative
|
||||
// traffic counters. After a reset, the current counter is the new increase.
|
||||
func TrafficCounterDelta(current, previous int64) int64 {
|
||||
if current < 0 || previous < 0 {
|
||||
return 0
|
||||
}
|
||||
if current >= previous {
|
||||
return current - previous
|
||||
}
|
||||
return current
|
||||
}
|
||||
|
||||
func deleteReportTrafficState(entityID string) {
|
||||
reportTrafficStates.Delete(entityID)
|
||||
}
|
||||
|
||||
func clearReportTrafficStates() {
|
||||
reportTrafficStates.Range(func(key, _ any) bool {
|
||||
reportTrafficStates.Delete(key)
|
||||
return true
|
||||
})
|
||||
}
|
||||
@@ -0,0 +1,440 @@
|
||||
package metricstore
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/komari-monitor/komari/database/models"
|
||||
"github.com/komari-monitor/komari/pkg/metric"
|
||||
v1 "github.com/komari-monitor/komari/protocol/v1"
|
||||
)
|
||||
|
||||
func useReportTestStore(t *testing.T, policy *metric.RollupPolicy) *metric.Store {
|
||||
t.Helper()
|
||||
ctx := context.Background()
|
||||
opts := []metric.Option{metric.WithMaxOpenConns(1)}
|
||||
if policy != nil {
|
||||
opts = append(opts, metric.WithRollupPolicy(*policy))
|
||||
}
|
||||
dsn := fmt.Sprintf("file:report-%d?mode=memory&cache=shared", time.Now().UnixNano())
|
||||
s, err := metric.Open(ctx, metric.SQLite(dsn, opts...))
|
||||
if err != nil {
|
||||
t.Fatalf("open metric store: %v", err)
|
||||
}
|
||||
if err := createMetricDefinitions(ctx, s); err != nil {
|
||||
_ = s.Close()
|
||||
t.Fatalf("create metric definitions: %v", err)
|
||||
}
|
||||
|
||||
storeMu.Lock()
|
||||
previous := store
|
||||
store = s
|
||||
storeMu.Unlock()
|
||||
clearReportTrafficStates()
|
||||
t.Cleanup(func() {
|
||||
clearReportTrafficStates()
|
||||
storeMu.Lock()
|
||||
store = previous
|
||||
storeMu.Unlock()
|
||||
_ = s.Close()
|
||||
})
|
||||
return s
|
||||
}
|
||||
|
||||
func TestWriteReportStoresMinuteMetricsAndResetAwareTraffic(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
policy := defaultRollupPolicy()
|
||||
s := useReportTestStore(t, &policy)
|
||||
base := time.Now().UTC().Truncate(time.Minute).Add(5 * time.Second)
|
||||
now := base.Add(45 * time.Second)
|
||||
|
||||
report := v1.Report{
|
||||
UUID: "node-a",
|
||||
UpdatedAt: base,
|
||||
CPU: v1.CPUReport{Usage: 12.5},
|
||||
Ram: v1.RamReport{Used: 100, Total: 1000},
|
||||
Swap: v1.RamReport{Used: 20, Total: 200},
|
||||
Load: v1.LoadReport{Load1: 0.5},
|
||||
Disk: v1.DiskReport{Used: 300, Total: 3000},
|
||||
Network: v1.NetworkReport{Up: 3, Down: 4, TotalUp: 100, TotalDown: 200},
|
||||
Process: 7,
|
||||
Connections: v1.ConnectionsReport{TCP: 8, UDP: 9},
|
||||
GPU: &v1.GPUDetailReport{
|
||||
AverageUsage: 25,
|
||||
DetailedInfo: []v1.GPUDeviceInfo{{
|
||||
Name: "GPU 0", MemoryUsed: 400, MemoryTotal: 800, Utilization: 30, Temperature: 55,
|
||||
}},
|
||||
},
|
||||
}
|
||||
if _, err := WriteReport(ctx, report); err != nil {
|
||||
t.Fatalf("write first report: %v", err)
|
||||
}
|
||||
|
||||
report.UpdatedAt = base.Add(3 * time.Second)
|
||||
report.Network.TotalUp = 150
|
||||
report.Network.TotalDown = 260
|
||||
if _, err := WriteReport(ctx, report); err != nil {
|
||||
t.Fatalf("write second report: %v", err)
|
||||
}
|
||||
|
||||
report.UpdatedAt = base.Add(6 * time.Second)
|
||||
report.Network.TotalUp = 20
|
||||
report.Network.TotalDown = 30
|
||||
if _, err := WriteReport(ctx, report); err != nil {
|
||||
t.Fatalf("write reset report: %v", err)
|
||||
}
|
||||
|
||||
assertMetricValues(t, s, MetricTrafficUp, report.UUID, base.Add(-time.Second), base.Add(time.Minute), []float64{0, 50, 20})
|
||||
assertMetricValues(t, s, MetricTrafficDown, report.UUID, base.Add(-time.Second), base.Add(time.Minute), []float64{0, 60, 30})
|
||||
assertMetricValues(t, s, MetricNetTotalUp, report.UUID, base.Add(-time.Second), base.Add(time.Minute), []float64{100, 150, 20})
|
||||
assertMetricAggregate(t, s, MetricTrafficUp, report.UUID, base.Add(-time.Second), base.Add(time.Minute), metric.AggSum, 70, 3)
|
||||
assertMetricAggregate(t, s, MetricTrafficDown, report.UUID, base.Add(-time.Second), base.Add(time.Minute), metric.AggSum, 90, 3)
|
||||
|
||||
gpuPoints, err := s.Query(ctx, metric.Query{
|
||||
MetricName: MetricGPUDeviceUsage,
|
||||
EntityID: report.UUID,
|
||||
Start: base.Add(-time.Second),
|
||||
End: base.Add(time.Minute),
|
||||
Tags: map[string]string{"device_index": "0"},
|
||||
Order: metric.OrderAsc,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("query GPU points: %v", err)
|
||||
}
|
||||
if len(gpuPoints) != 3 || !gpuPoints[2].Timestamp.Equal(base.Add(6*time.Second)) || gpuPoints[2].Tags["device_name"] != "GPU 0" {
|
||||
t.Fatalf("unexpected GPU points: %#v", gpuPoints)
|
||||
}
|
||||
|
||||
if _, err := s.Compact(ctx, now); err != nil {
|
||||
t.Fatalf("compact reports: %v", err)
|
||||
}
|
||||
deleteReportTrafficState(report.UUID)
|
||||
report.UpdatedAt = now
|
||||
report.Network.TotalUp = 35
|
||||
report.Network.TotalDown = 50
|
||||
if _, err := WriteReport(ctx, report); err != nil {
|
||||
t.Fatalf("write after restoring rollup baseline: %v", err)
|
||||
}
|
||||
assertMetricValues(t, s, MetricTrafficUp, report.UUID, now.Add(-time.Second), now.Add(time.Second), []float64{15})
|
||||
assertMetricValues(t, s, MetricTrafficDown, report.UUID, now.Add(-time.Second), now.Add(time.Second), []float64{20})
|
||||
}
|
||||
|
||||
func TestWriteReportSkipsMetricsWithoutAgentData(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
s := useReportTestStore(t, nil)
|
||||
timestamp := time.Now().UTC()
|
||||
if _, err := WriteReport(ctx, v1.Report{
|
||||
UUID: "node-without-gpu", UpdatedAt: timestamp,
|
||||
}); err != nil {
|
||||
t.Fatalf("write report: %v", err)
|
||||
}
|
||||
points, err := s.Query(ctx, metric.Query{
|
||||
MetricName: MetricGPU, EntityID: "node-without-gpu",
|
||||
Start: timestamp.Add(-time.Second), End: timestamp.Add(time.Second),
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("query GPU metric: %v", err)
|
||||
}
|
||||
if len(points) != 0 {
|
||||
t.Fatalf("GPU metric was written without GPU data: %#v", points)
|
||||
}
|
||||
}
|
||||
|
||||
func TestReportBatcherFlushesQueuedReports(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
s := useReportTestStore(t, nil)
|
||||
StartReportBatcher()
|
||||
t.Cleanup(func() {
|
||||
if err := StopReportBatcher(ctx); err != nil {
|
||||
t.Errorf("stop report batcher: %v", err)
|
||||
}
|
||||
})
|
||||
|
||||
base := time.Now().UTC().Truncate(time.Minute).Add(10 * time.Second)
|
||||
first := v1.Report{
|
||||
UUID: "batched-node",
|
||||
UpdatedAt: base,
|
||||
CPU: v1.CPUReport{Usage: 10},
|
||||
Network: v1.NetworkReport{TotalUp: 100, TotalDown: 200},
|
||||
}
|
||||
second := first
|
||||
second.UpdatedAt = base.Add(3 * time.Second)
|
||||
second.CPU.Usage = 20
|
||||
second.Network.TotalUp = 150
|
||||
second.Network.TotalDown = 260
|
||||
|
||||
if _, err := WriteReport(ctx, first); err != nil {
|
||||
t.Fatalf("queue first report: %v", err)
|
||||
}
|
||||
if _, err := WriteReport(ctx, second); err != nil {
|
||||
t.Fatalf("queue second report: %v", err)
|
||||
}
|
||||
points, err := s.Query(ctx, metric.Query{
|
||||
MetricName: MetricCPU,
|
||||
EntityID: first.UUID,
|
||||
Start: base.Add(-time.Second),
|
||||
End: base.Add(time.Minute),
|
||||
Order: metric.OrderAsc,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("query before flush: %v", err)
|
||||
}
|
||||
if len(points) != 0 {
|
||||
t.Fatalf("queued reports were written before flush: %#v", points)
|
||||
}
|
||||
|
||||
if err := FlushReportBatch(ctx); err != nil {
|
||||
t.Fatalf("flush report batch: %v", err)
|
||||
}
|
||||
assertMetricValues(t, s, MetricCPU, first.UUID, base.Add(-time.Second), base.Add(time.Minute), []float64{10, 20})
|
||||
assertMetricValues(t, s, MetricTrafficUp, first.UUID, base.Add(-time.Second), base.Add(time.Minute), []float64{0, 50})
|
||||
assertMetricValues(t, s, MetricTrafficDown, first.UUID, base.Add(-time.Second), base.Add(time.Minute), []float64{0, 60})
|
||||
assertMetricAggregate(t, s, MetricCPU, first.UUID, base.Add(-time.Second), base.Add(time.Minute), metric.AggAvg, 15, 2)
|
||||
}
|
||||
|
||||
func TestPingBatcherFlushesLatencyAndLossTogether(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
s := useReportTestStore(t, nil)
|
||||
StartReportBatcher()
|
||||
t.Cleanup(func() {
|
||||
if err := StopReportBatcher(ctx); err != nil {
|
||||
t.Errorf("stop report batcher: %v", err)
|
||||
}
|
||||
})
|
||||
|
||||
base := time.Now().UTC().Truncate(time.Second)
|
||||
records := []models.PingRecord{
|
||||
{Client: "ping-node", TaskId: 7, Time: base, Value: 24},
|
||||
{Client: "ping-node", TaskId: 7, Time: base.Add(time.Minute), Value: -1},
|
||||
{Client: "ping-node", TaskId: 8, Time: base, Value: 31},
|
||||
}
|
||||
for _, record := range records {
|
||||
if err := WritePingRecord(ctx, record); err != nil {
|
||||
t.Fatalf("queue ping record: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
for _, name := range []string{MetricPingLatency, MetricPingLoss} {
|
||||
points, err := s.Query(ctx, metric.Query{
|
||||
MetricName: name,
|
||||
EntityID: "ping-node",
|
||||
Start: base.Add(-time.Second),
|
||||
End: base.Add(2 * time.Minute),
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("query queued %s points: %v", name, err)
|
||||
}
|
||||
if len(points) != 0 {
|
||||
t.Fatalf("queued %s points were written before flush: %#v", name, points)
|
||||
}
|
||||
}
|
||||
|
||||
if err := FlushReportBatch(ctx); err != nil {
|
||||
t.Fatalf("flush ping batch: %v", err)
|
||||
}
|
||||
|
||||
latency, err := s.Query(ctx, metric.Query{
|
||||
MetricName: MetricPingLatency,
|
||||
EntityID: "ping-node",
|
||||
Tags: map[string]string{"task_id": "7"},
|
||||
Start: base.Add(-time.Second),
|
||||
End: base.Add(2 * time.Minute),
|
||||
Order: metric.OrderAsc,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("query flushed latency points: %v", err)
|
||||
}
|
||||
if len(latency) != 2 || latency[0].Value != 24 || latency[1].Value != -1 {
|
||||
t.Fatalf("latency points = %#v, want both original samples", latency)
|
||||
}
|
||||
|
||||
loss, err := s.Query(ctx, metric.Query{
|
||||
MetricName: MetricPingLoss,
|
||||
EntityID: "ping-node",
|
||||
Tags: map[string]string{"task_id": "7"},
|
||||
Start: base.Add(-time.Second),
|
||||
End: base.Add(2 * time.Minute),
|
||||
Order: metric.OrderAsc,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("query flushed loss points: %v", err)
|
||||
}
|
||||
if len(loss) != 2 || loss[0].Value != 0 || loss[1].Value != 1 {
|
||||
t.Fatalf("loss points = %#v, want success and loss samples", loss)
|
||||
}
|
||||
}
|
||||
|
||||
func TestReportBatchKeepsEverySample(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
s := useReportTestStore(t, nil)
|
||||
base := time.Now().UTC().Truncate(time.Second)
|
||||
pending := []v1.Report{
|
||||
{UUID: "node-a", UpdatedAt: base, CPU: v1.CPUReport{Usage: 10}, Network: v1.NetworkReport{TotalUp: 100}},
|
||||
{UUID: "node-a", UpdatedAt: base, CPU: v1.CPUReport{Usage: 20}, Network: v1.NetworkReport{TotalUp: 150}},
|
||||
{UUID: "node-b", UpdatedAt: base, CPU: v1.CPUReport{Usage: 30}, Network: v1.NetworkReport{TotalUp: 200}},
|
||||
{UUID: "node-b", UpdatedAt: base.Add(time.Second), CPU: v1.CPUReport{Usage: 40}, Network: v1.NetworkReport{TotalUp: 260}},
|
||||
}
|
||||
|
||||
if err := writePendingReports(ctx, &pending); err != nil {
|
||||
t.Fatalf("write report batch: %v", err)
|
||||
}
|
||||
if len(pending) != 0 {
|
||||
t.Fatalf("pending reports = %d, want 0", len(pending))
|
||||
}
|
||||
assertMetricValues(t, s, MetricCPU, "node-a", base.Add(-time.Second), base.Add(time.Minute), []float64{10, 20})
|
||||
assertMetricValues(t, s, MetricCPU, "node-b", base.Add(-time.Second), base.Add(time.Minute), []float64{30, 40})
|
||||
}
|
||||
|
||||
func TestReportQueueFullReturnsError(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
useReportTestStore(t, nil)
|
||||
worker := &reportBatchWorker{
|
||||
queue: make(chan v1.Report, 1),
|
||||
requests: make(chan reportBatchRequest, 1),
|
||||
done: make(chan struct{}),
|
||||
}
|
||||
worker.queue <- v1.Report{UUID: "already-queued"}
|
||||
reportBatcherMu.Lock()
|
||||
reportBatcher = worker
|
||||
reportBatcherMu.Unlock()
|
||||
t.Cleanup(func() {
|
||||
reportBatcherMu.Lock()
|
||||
if reportBatcher == worker {
|
||||
reportBatcher = nil
|
||||
}
|
||||
reportBatcherMu.Unlock()
|
||||
})
|
||||
|
||||
report := v1.Report{
|
||||
UUID: "realtime-node",
|
||||
UpdatedAt: time.Now().UTC(),
|
||||
}
|
||||
_, err := WriteReport(ctx, report)
|
||||
if !errors.Is(err, ErrReportBatchQueueFull) {
|
||||
t.Fatalf("queue-full error = %v, want %v", err, ErrReportBatchQueueFull)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRecordReconstructionUsesMetricSpecificAggregation(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
s := useReportTestStore(t, nil)
|
||||
base := time.Now().UTC().Truncate(time.Minute)
|
||||
entityID := "node-aggregation"
|
||||
points := []metric.Point{
|
||||
{MetricName: MetricCPU, EntityID: entityID, Timestamp: base.Add(time.Second), Value: 10},
|
||||
{MetricName: MetricCPU, EntityID: entityID, Timestamp: base.Add(2 * time.Second), Value: 30},
|
||||
{MetricName: MetricNetTotalUp, EntityID: entityID, Timestamp: base.Add(time.Second), Value: 100},
|
||||
{MetricName: MetricNetTotalUp, EntityID: entityID, Timestamp: base.Add(2 * time.Second), Value: 200},
|
||||
{MetricName: MetricTrafficUp, EntityID: entityID, Timestamp: base.Add(time.Second), Value: 10},
|
||||
{MetricName: MetricTrafficUp, EntityID: entityID, Timestamp: base.Add(2 * time.Second), Value: 20},
|
||||
}
|
||||
if err := s.WriteBatch(ctx, points); err != nil {
|
||||
t.Fatalf("write points: %v", err)
|
||||
}
|
||||
|
||||
records, err := GetRecordsByClientAndTime(ctx, entityID, base, base.Add(time.Hour))
|
||||
if err != nil {
|
||||
t.Fatalf("reconstruct records: %v", err)
|
||||
}
|
||||
if len(records) != 1 {
|
||||
t.Fatalf("records = %#v, want one bucket", records)
|
||||
}
|
||||
if records[0].Cpu != 20 || records[0].NetTotalUp != 200 || records[0].TrafficUp != 30 {
|
||||
t.Fatalf("unexpected aggregation result: %#v", records[0])
|
||||
}
|
||||
}
|
||||
|
||||
func TestTrafficCounterDelta(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
current int64
|
||||
previous int64
|
||||
want int64
|
||||
}{
|
||||
{name: "previous zero", current: 120, previous: 0, want: 120},
|
||||
{name: "monotonic counter", current: 250, previous: 200, want: 50},
|
||||
{name: "unchanged counter", current: 100, previous: 100, want: 0},
|
||||
{name: "counter reset", current: 15, previous: 250, want: 15},
|
||||
{name: "negative current", current: -1, previous: 100, want: 0},
|
||||
{name: "negative previous", current: 15, previous: -1, want: 0},
|
||||
}
|
||||
for _, test := range tests {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
if got := TrafficCounterDelta(test.current, test.previous); got != test.want {
|
||||
t.Fatalf("TrafficCounterDelta(%d, %d) = %d, want %d", test.current, test.previous, got, test.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestWriteReportNormalizesReceiveTimeToUTC(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
s := useReportTestStore(t, nil)
|
||||
local := time.FixedZone("UTC+8", 8*60*60)
|
||||
receiveTime := time.Now().In(local).Add(-10 * time.Second)
|
||||
report := v1.Report{
|
||||
UUID: "utc-report",
|
||||
UpdatedAt: receiveTime,
|
||||
CPU: v1.CPUReport{Usage: 10},
|
||||
Network: v1.NetworkReport{TotalUp: 1, TotalDown: 2},
|
||||
}
|
||||
|
||||
saved, err := WriteReport(ctx, report)
|
||||
if err != nil {
|
||||
t.Fatalf("write report: %v", err)
|
||||
}
|
||||
if !saved.UpdatedAt.Equal(receiveTime) || saved.UpdatedAt.Location() != time.UTC {
|
||||
t.Fatalf("saved receive time = %s (%s), want UTC", saved.UpdatedAt, saved.UpdatedAt.Location())
|
||||
}
|
||||
points, err := s.Query(ctx, metric.Query{
|
||||
MetricName: MetricCPU,
|
||||
EntityID: report.UUID,
|
||||
Start: receiveTime.Add(-time.Nanosecond),
|
||||
End: receiveTime.Add(time.Nanosecond),
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("query stored point: %v", err)
|
||||
}
|
||||
if len(points) != 1 || points[0].Timestamp.Location() != time.UTC || points[0].Timestamp.UnixMilli() != receiveTime.UnixMilli() {
|
||||
t.Fatalf("stored points = %#v, want one UTC millisecond point", points)
|
||||
}
|
||||
}
|
||||
|
||||
func assertMetricValues(t *testing.T, s *metric.Store, metricName, entityID string, start, end time.Time, want []float64) {
|
||||
t.Helper()
|
||||
points, err := s.Query(context.Background(), metric.Query{
|
||||
MetricName: metricName,
|
||||
EntityID: entityID,
|
||||
Start: start,
|
||||
End: end,
|
||||
Order: metric.OrderAsc,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("query %s: %v", metricName, err)
|
||||
}
|
||||
if len(points) != len(want) {
|
||||
t.Fatalf("%s point count = %d, want %d: %#v", metricName, len(points), len(want), points)
|
||||
}
|
||||
for i := range want {
|
||||
if points[i].Value != want[i] {
|
||||
t.Fatalf("%s point %d = %v, want %v", metricName, i, points[i].Value, want[i])
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func assertMetricAggregate(t *testing.T, s *metric.Store, metricName, entityID string, start, end time.Time, aggregation metric.Aggregation, want float64, wantCount int) {
|
||||
t.Helper()
|
||||
points, err := s.Series(context.Background(), metric.AggregateQuery{
|
||||
Query: metric.Query{MetricName: metricName, EntityID: entityID, Start: start, End: end},
|
||||
Aggregation: aggregation, Interval: time.Minute, PreserveSeries: true,
|
||||
}, end)
|
||||
if err != nil {
|
||||
t.Fatalf("aggregate %s: %v", metricName, err)
|
||||
}
|
||||
if len(points) != 1 || points[0].Value != want || points[0].Count != wantCount {
|
||||
t.Fatalf("aggregate %s = %#v, want value=%v count=%d", metricName, points, want, wantCount)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,137 @@
|
||||
package metricstore
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
|
||||
"github.com/komari-monitor/komari/internal/config"
|
||||
"github.com/komari-monitor/komari/pkg/metric"
|
||||
)
|
||||
|
||||
// RestructureProgress is the authenticated guide's stable progress payload.
|
||||
type RestructureProgress struct {
|
||||
Phase string
|
||||
CurrentMetric string
|
||||
RowsDone int64
|
||||
RowsTotal int64
|
||||
MetricsDone int
|
||||
MetricsTotal int
|
||||
}
|
||||
|
||||
type RestructureResult struct {
|
||||
BeforeBytes int64
|
||||
AfterBytes int64
|
||||
RowsCopied int64
|
||||
Metrics int
|
||||
}
|
||||
|
||||
// StructureUpgradeRequired checks the configured metric store without creating
|
||||
// tables. It is called before normal metric-store initialization so an existing
|
||||
// installation is never rebuilt at startup without an administrator action.
|
||||
func StructureUpgradeRequired(ctx context.Context) (bool, error) {
|
||||
cfg, err := config.GetManyAs[MetricStoreConfig]()
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
return structureUpgradeRequiredForConfig(ctx, cfg)
|
||||
}
|
||||
|
||||
func structureUpgradeRequiredForConfig(ctx context.Context, cfg *MetricStoreConfig) (bool, error) {
|
||||
metricCfg, err := buildMetricConfig(cfg, false)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
store, err := metric.Open(ctx, metricCfg)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
defer store.Close()
|
||||
return store.NeedsRestructure(ctx)
|
||||
}
|
||||
|
||||
// RestructureConfiguredStore performs the explicit data copy, table swap and
|
||||
// one physical reclaim pass used by the upgrade guide.
|
||||
func RestructureConfiguredStore(ctx context.Context, report func(RestructureProgress)) (RestructureResult, error) {
|
||||
cfg, err := config.GetManyAs[MetricStoreConfig]()
|
||||
if err != nil {
|
||||
return RestructureResult{}, err
|
||||
}
|
||||
metricCfg, err := buildMetricConfig(cfg, false)
|
||||
if err != nil {
|
||||
return RestructureResult{}, err
|
||||
}
|
||||
store, err := metric.Open(ctx, metricCfg)
|
||||
if err != nil {
|
||||
return RestructureResult{}, err
|
||||
}
|
||||
defer store.Close()
|
||||
|
||||
before, err := store.LegacyStorageSize(ctx)
|
||||
if err != nil {
|
||||
return RestructureResult{}, fmt.Errorf("measure legacy metric storage: %w", err)
|
||||
}
|
||||
result, err := store.Restructure(ctx, func(progress metric.RestructureProgress) {
|
||||
if report == nil {
|
||||
return
|
||||
}
|
||||
report(RestructureProgress{Phase: progress.Phase, CurrentMetric: progress.Current, RowsDone: progress.RowsDone, RowsTotal: progress.RowsTotal, MetricsDone: progress.MetricsDone, MetricsTotal: progress.MetricsTotal})
|
||||
})
|
||||
if err != nil {
|
||||
return RestructureResult{}, err
|
||||
}
|
||||
if report != nil {
|
||||
report(RestructureProgress{Phase: "reclaiming", RowsDone: result.RowsCopied, RowsTotal: result.RowsCopied, MetricsDone: result.Metrics, MetricsTotal: result.Metrics})
|
||||
}
|
||||
if err := store.ReclaimSpace(ctx); err != nil {
|
||||
return RestructureResult{}, fmt.Errorf("reclaim metric storage: %w", err)
|
||||
}
|
||||
after, err := store.StorageSize(ctx)
|
||||
if err != nil {
|
||||
return RestructureResult{}, fmt.Errorf("measure rebuilt metric storage: %w", err)
|
||||
}
|
||||
return RestructureResult{BeforeBytes: before, AfterBytes: after, RowsCopied: result.RowsCopied, Metrics: result.Metrics}, nil
|
||||
}
|
||||
|
||||
// DiscardConfiguredStoreHistory completes the same schema upgrade as
|
||||
// RestructureConfiguredStore but drops every historical sample instead of
|
||||
// copying it. Metric definitions remain available for new ingest.
|
||||
func DiscardConfiguredStoreHistory(ctx context.Context, report func(RestructureProgress)) (RestructureResult, error) {
|
||||
cfg, err := config.GetManyAs[MetricStoreConfig]()
|
||||
if err != nil {
|
||||
return RestructureResult{}, err
|
||||
}
|
||||
metricCfg, err := buildMetricConfig(cfg, false)
|
||||
if err != nil {
|
||||
return RestructureResult{}, err
|
||||
}
|
||||
store, err := metric.Open(ctx, metricCfg)
|
||||
if err != nil {
|
||||
return RestructureResult{}, err
|
||||
}
|
||||
defer store.Close()
|
||||
|
||||
before, err := store.LegacyStorageSize(ctx)
|
||||
if err != nil {
|
||||
return RestructureResult{}, fmt.Errorf("measure legacy metric storage: %w", err)
|
||||
}
|
||||
result, err := store.DiscardHistory(ctx, func(progress metric.RestructureProgress) {
|
||||
if report == nil {
|
||||
return
|
||||
}
|
||||
report(RestructureProgress{Phase: progress.Phase, CurrentMetric: progress.Current, RowsDone: progress.RowsDone, RowsTotal: progress.RowsTotal, MetricsDone: progress.MetricsDone, MetricsTotal: progress.MetricsTotal})
|
||||
})
|
||||
if err != nil {
|
||||
return RestructureResult{}, err
|
||||
}
|
||||
if report != nil {
|
||||
report(RestructureProgress{Phase: "reclaiming", RowsDone: result.RowsCopied, RowsTotal: result.RowsCopied, MetricsDone: result.Metrics, MetricsTotal: result.Metrics})
|
||||
}
|
||||
if err := store.ReclaimSpace(ctx); err != nil {
|
||||
return RestructureResult{}, fmt.Errorf("reclaim metric storage: %w", err)
|
||||
}
|
||||
after, err := store.StorageSize(ctx)
|
||||
if err != nil {
|
||||
return RestructureResult{}, fmt.Errorf("measure rebuilt metric storage: %w", err)
|
||||
}
|
||||
return RestructureResult{BeforeBytes: before, AfterBytes: after, RowsCopied: result.RowsCopied, Metrics: result.Metrics}, nil
|
||||
}
|
||||
@@ -0,0 +1,315 @@
|
||||
package metricstore
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
logger "github.com/komari-monitor/komari/utils/log"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/komari-monitor/komari/internal/config"
|
||||
"github.com/komari-monitor/komari/pkg/metric"
|
||||
)
|
||||
|
||||
// store_migration.go
|
||||
//
|
||||
// 面向 WebUI/API 的“存储后端迁移”支撑:把一个 metrics 源库(默认本地 SQLite
|
||||
// ./data/metrics.db)中的全部指标数据搬运到当前正在运行的 metrics 目标库
|
||||
// (通常是用户新配置并热重载后的 MySQL/PostgreSQL)。
|
||||
//
|
||||
// 此迁移仅能由管理员在 WebUI 中「切换数据库后」手动触发,异步执行并可查询进度 /
|
||||
// 取消。服务启动和指标库热重载均不会复制历史数据。数据以 upsert 写入,可安全重复执行。
|
||||
//
|
||||
// 典型流程:
|
||||
// 1. 管理员在设置中把 metric_db_dsn 改为 MySQL/PostgreSQL,保存 → 后端连接测试
|
||||
// 通过后热重载,当前 store 切到远端(此时远端为空)。
|
||||
// 2. 管理员调用 startMetricMigration(不带 source 参数),系统以上一个 metrics
|
||||
// 目标(默认 SQLite metrics.db)为源,把历史数据搬运到远端。
|
||||
// 3. 完成后登记目标指纹,供下一次手动迁移默认选择源库。
|
||||
|
||||
// StoreMigrationProgress 描述一次 store-to-store 迁移的实时进度。
|
||||
type StoreMigrationProgress struct {
|
||||
Status string `json:"status"` // idle, running, completed, failed, canceled
|
||||
SourceDriver string `json:"source_driver"` // 源库驱动
|
||||
SourceDSN string `json:"source_dsn"` // 源库 DSN(脱敏后)
|
||||
TargetDriver string `json:"target_driver"` // 目标库驱动
|
||||
TargetDSN string `json:"target_dsn"` // 目标库 DSN(脱敏后)
|
||||
TotalMetrics int `json:"total_metrics"` // 指标定义总数
|
||||
CurrentMetric string `json:"current_metric"` // 当前正在搬运的指标名
|
||||
MetricsDone int `json:"metrics_done"` // 已完成的指标数
|
||||
MigratedPoints int64 `json:"migrated_points"` // 已搬运的采样点数
|
||||
StartTime time.Time `json:"start_time"` // 开始时间
|
||||
EndTime time.Time `json:"end_time"` // 结束时间
|
||||
Error string `json:"error,omitempty"` // 错误信息
|
||||
}
|
||||
|
||||
var (
|
||||
storeMigMu sync.Mutex
|
||||
storeMigProgress StoreMigrationProgress
|
||||
storeMigCancel context.CancelFunc
|
||||
storeMigDone chan struct{}
|
||||
storeClosing bool
|
||||
)
|
||||
|
||||
// IsStoreMigrationRunning 报告是否有 store-to-store 迁移正在运行。
|
||||
func IsStoreMigrationRunning() bool {
|
||||
storeMigMu.Lock()
|
||||
defer storeMigMu.Unlock()
|
||||
return storeMigCancel != nil
|
||||
}
|
||||
|
||||
// GetStoreMigrationProgress 返回当前 store-to-store 迁移进度快照。
|
||||
func GetStoreMigrationProgress() StoreMigrationProgress {
|
||||
storeMigMu.Lock()
|
||||
defer storeMigMu.Unlock()
|
||||
return storeMigProgress
|
||||
}
|
||||
|
||||
// ResolveStoreMigrationSourceFingerprint 推断本次 store 迁移的源库指纹。
|
||||
//
|
||||
// 优先级:
|
||||
// 1. 显式传入的 driver+dsn(driver 可留空,交由 DSN 推断)。
|
||||
// 2. 已保存的上一个 metrics 目标指纹(MigrationTargetKey)。
|
||||
// 3. 默认 SQLite(./data/metrics.db)。
|
||||
func ResolveStoreMigrationSourceFingerprint(driver, dsn string) string {
|
||||
dsn = strings.TrimSpace(dsn)
|
||||
if dsn != "" {
|
||||
d := ResolveDriverFromConfig(driver, dsn)
|
||||
return fmt.Sprintf("%s|%s", d, dsn)
|
||||
}
|
||||
if saved, _ := config.GetAs[string](MigrationTargetKey, ""); saved != "" {
|
||||
return saved
|
||||
}
|
||||
return defaultSQLiteFingerprint()
|
||||
}
|
||||
|
||||
// StartStoreMigration 异步启动一次 store-to-store 迁移:把源库(由 sourceDriver +
|
||||
// sourceDSN 指定,留空则自动推断)的全部指标数据搬运到当前运行中的 metrics 目标库。
|
||||
//
|
||||
// 返回错误表示“未能启动”(例如已有迁移在跑、源与目标相同、目标未初始化等);
|
||||
// 迁移过程中的错误通过 GetStoreMigrationProgress().Status == "failed" 与 Error 暴露。
|
||||
func StartStoreMigration(sourceDriver, sourceDSN string) error {
|
||||
cfg, err := config.GetManyAs[MetricStoreConfig]()
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to load metric store config: %w", err)
|
||||
}
|
||||
targetFP := targetFingerprint(cfg)
|
||||
sourceFP := ResolveStoreMigrationSourceFingerprint(sourceDriver, sourceDSN)
|
||||
|
||||
if sourceFP == targetFP {
|
||||
return fmt.Errorf("source and target metrics database are the same; nothing to migrate")
|
||||
}
|
||||
|
||||
srcCfg, err := configFromFingerprint(sourceFP, cfg)
|
||||
if err != nil {
|
||||
return fmt.Errorf("resolve source metrics store: %w", err)
|
||||
}
|
||||
if !storeOperations.TryAcquire() {
|
||||
return ErrStoreBusy
|
||||
}
|
||||
exclusiveLeaseHandedOff := false
|
||||
defer func() {
|
||||
if !exclusiveLeaseHandedOff {
|
||||
storeOperations.Release()
|
||||
}
|
||||
}()
|
||||
|
||||
storeMu.RLock()
|
||||
dst := store
|
||||
activeFingerprint := storeFingerprint
|
||||
storeMu.RUnlock()
|
||||
if dst == nil {
|
||||
return ErrStoreNotInitialized
|
||||
}
|
||||
if activeFingerprint != targetFP {
|
||||
return fmt.Errorf("active metric store does not match the configured migration target; wait for hot reload and retry")
|
||||
}
|
||||
|
||||
storeMigMu.Lock()
|
||||
if storeClosing {
|
||||
storeMigMu.Unlock()
|
||||
return ErrStoreBusy
|
||||
}
|
||||
if storeMigCancel != nil {
|
||||
storeMigMu.Unlock()
|
||||
return fmt.Errorf("a store migration is already in progress")
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
done := make(chan struct{})
|
||||
storeMigCancel = cancel
|
||||
storeMigDone = done
|
||||
storeMigProgress = StoreMigrationProgress{
|
||||
Status: "running",
|
||||
SourceDriver: string(ResolveDriverFromConfig(srcCfg.Driver, srcCfg.DSN)),
|
||||
SourceDSN: maskDSN(srcCfg.DSN),
|
||||
TargetDriver: string(ResolveDriverFromConfig(cfg.Driver, cfg.DSN)),
|
||||
TargetDSN: maskDSN(cfg.DSN),
|
||||
StartTime: time.Now().UTC(),
|
||||
}
|
||||
storeMigMu.Unlock()
|
||||
|
||||
exclusiveLeaseHandedOff = true
|
||||
go runStoreMigration(ctx, cancel, done, srcCfg, cfg, dst, targetFP)
|
||||
return nil
|
||||
}
|
||||
|
||||
// runStoreMigration 执行实际的搬运逻辑(在独立 goroutine 中)。
|
||||
func runStoreMigration(ctx context.Context, cancel context.CancelFunc, done chan struct{}, srcCfg *MetricStoreConfig, cfg *MetricStoreConfig, dst *metric.Store, targetFP string) {
|
||||
defer func() {
|
||||
cancel()
|
||||
storeOperations.Release()
|
||||
|
||||
storeMigMu.Lock()
|
||||
storeMigCancel = nil
|
||||
storeMigDone = nil
|
||||
close(done)
|
||||
storeMigMu.Unlock()
|
||||
}()
|
||||
|
||||
src, err := openSourceStore(ctx, srcCfg)
|
||||
if err != nil {
|
||||
finishStoreMigration("failed", fmt.Errorf("open source metrics store: %w", err))
|
||||
return
|
||||
}
|
||||
defer src.Close()
|
||||
|
||||
observe := func(currentMetric string, metricIndex, totalMetrics int, addedPoints int64) {
|
||||
storeMigMu.Lock()
|
||||
storeMigProgress.CurrentMetric = currentMetric
|
||||
storeMigProgress.MetricsDone = metricIndex
|
||||
storeMigProgress.TotalMetrics = totalMetrics
|
||||
storeMigProgress.MigratedPoints += addedPoints
|
||||
storeMigMu.Unlock()
|
||||
}
|
||||
|
||||
total, err := migrateBetweenStores(ctx, src, dst, observe)
|
||||
if err != nil {
|
||||
if ctx.Err() != nil {
|
||||
finishStoreMigration("canceled", nil)
|
||||
logger.Warnf("metricstore", "[store-migration] canceled after %d points", total)
|
||||
return
|
||||
}
|
||||
finishStoreMigration("failed", err)
|
||||
return
|
||||
}
|
||||
|
||||
// 搬运成功:登记当前目标指纹,供下一次手动迁移推断源库。
|
||||
if err := config.Set(MigrationTargetKey, targetFP); err != nil {
|
||||
logger.Errorf("metricstore", "[store-migration] failed to persist migration target fingerprint: %v", err)
|
||||
}
|
||||
finishStoreMigration("completed", nil)
|
||||
logger.Infof("metricstore", "[store-migration] completed via API (%d points) target_driver=%s", total, ResolveDriverFromConfig(cfg.Driver, cfg.DSN))
|
||||
}
|
||||
|
||||
// finishStoreMigration 统一收尾:设置终态状态、结束时间与错误信息。
|
||||
func finishStoreMigration(status string, err error) {
|
||||
storeMigMu.Lock()
|
||||
defer storeMigMu.Unlock()
|
||||
storeMigProgress.Status = status
|
||||
storeMigProgress.EndTime = time.Now().UTC()
|
||||
if err != nil {
|
||||
storeMigProgress.Error = err.Error()
|
||||
}
|
||||
// 完成/取消时把当前指标标记为已全部处理,便于前端进度条归位。
|
||||
if status == "completed" && storeMigProgress.TotalMetrics > 0 {
|
||||
storeMigProgress.MetricsDone = storeMigProgress.TotalMetrics
|
||||
}
|
||||
}
|
||||
|
||||
// CancelStoreMigration 请求取消正在运行的 store-to-store 迁移,并等待其退出。
|
||||
// 由于写入是幂等 upsert,取消后可安全重新发起,不会产生重复数据。
|
||||
func CancelStoreMigration() error {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||||
defer cancel()
|
||||
return stopStoreMigration(ctx, true)
|
||||
}
|
||||
|
||||
func stopStoreMigration(ctx context.Context, requireRunning bool) error {
|
||||
storeMigMu.Lock()
|
||||
cancel := storeMigCancel
|
||||
done := storeMigDone
|
||||
storeMigMu.Unlock()
|
||||
|
||||
if cancel == nil {
|
||||
if requireRunning {
|
||||
return fmt.Errorf("no store migration is currently running")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
cancel()
|
||||
|
||||
select {
|
||||
case <-done:
|
||||
return nil
|
||||
case <-ctx.Done():
|
||||
return fmt.Errorf("wait for store migration to cancel: %w", ctx.Err())
|
||||
}
|
||||
}
|
||||
|
||||
func stopStoreMigrationForClose(ctx context.Context) error {
|
||||
storeMigMu.Lock()
|
||||
storeClosing = true
|
||||
cancel := storeMigCancel
|
||||
done := storeMigDone
|
||||
storeMigMu.Unlock()
|
||||
|
||||
if cancel == nil {
|
||||
return nil
|
||||
}
|
||||
cancel()
|
||||
select {
|
||||
case <-done:
|
||||
return nil
|
||||
case <-ctx.Done():
|
||||
return fmt.Errorf("wait for store migration to cancel: %w", ctx.Err())
|
||||
}
|
||||
}
|
||||
|
||||
// StopStoreMigrationForShutdown prevents a new store migration and waits for
|
||||
// an active one to release its exclusive lease before report batches flush.
|
||||
func StopStoreMigrationForShutdown(ctx context.Context) error {
|
||||
return stopStoreMigrationForClose(ctx)
|
||||
}
|
||||
|
||||
func clearStoreClosing() {
|
||||
storeMigMu.Lock()
|
||||
storeClosing = false
|
||||
storeMigMu.Unlock()
|
||||
}
|
||||
|
||||
func isStoreClosing() bool {
|
||||
storeMigMu.Lock()
|
||||
defer storeMigMu.Unlock()
|
||||
return storeClosing
|
||||
}
|
||||
|
||||
// maskDSN 对 DSN 做粗粒度脱敏,避免在 API 响应/日志中泄露密码。
|
||||
// 处理两类常见格式:URL(scheme://user:pass@host/...)与 key=value(password=...)。
|
||||
func maskDSN(dsn string) string {
|
||||
dsn = strings.TrimSpace(dsn)
|
||||
if dsn == "" {
|
||||
return ""
|
||||
}
|
||||
// key=value 形式:屏蔽 password=... 段。
|
||||
if strings.Contains(dsn, "password=") {
|
||||
parts := strings.Fields(dsn)
|
||||
for i, p := range parts {
|
||||
if strings.HasPrefix(strings.ToLower(p), "password=") {
|
||||
parts[i] = "password=***"
|
||||
}
|
||||
}
|
||||
return strings.Join(parts, " ")
|
||||
}
|
||||
// URL / user:pass@host 形式:屏蔽 user:pass@ 中的密码。
|
||||
if at := strings.LastIndex(dsn, "@"); at > 0 {
|
||||
head := dsn[:at]
|
||||
tail := dsn[at:]
|
||||
if colon := strings.LastIndex(head, ":"); colon >= 0 {
|
||||
// 保留 scheme://user,屏蔽密码。
|
||||
return head[:colon] + ":***" + tail
|
||||
}
|
||||
}
|
||||
return dsn
|
||||
}
|
||||
@@ -1,121 +0,0 @@
|
||||
package models
|
||||
|
||||
import (
|
||||
"database/sql/driver"
|
||||
"fmt"
|
||||
"log"
|
||||
"os"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
// LocalTime is a custom time type for GORM to handle time.Time correctly.
|
||||
// It ensures that all times are stored and retrieved based on the application's configured timezone.
|
||||
type LocalTime time.Time
|
||||
|
||||
// Value implements the driver.Valuer interface.
|
||||
// It converts UTCTime to a TEXT format that the database understands.
|
||||
// The time is formatted as a string in the application's local timezone, without timezone information.
|
||||
func (t LocalTime) Value() (driver.Value, error) {
|
||||
if time.Time(t).IsZero() {
|
||||
return nil, nil
|
||||
}
|
||||
return time.Time(t).In(GetAppLocation()).Format("2006-01-02 15:04:05.0000000"), nil
|
||||
}
|
||||
|
||||
// Scan implements the sql.Scanner interface.
|
||||
// It scans a value from the database into a UTCTime object.
|
||||
func (t *LocalTime) Scan(v interface{}) error {
|
||||
if v == nil {
|
||||
*t = LocalTime(time.Time{})
|
||||
return nil
|
||||
}
|
||||
|
||||
loc := GetAppLocation()
|
||||
|
||||
switch val := v.(type) {
|
||||
case time.Time:
|
||||
// CRITICAL FIX: When the driver reads a timezone-less string (e.g., "16:00:00"),
|
||||
// it often incorrectly assumes it's UTC. We must correct this by re-interpreting
|
||||
// the date and clock values in the application's actual timezone.
|
||||
year, month, day := val.Date()
|
||||
hour, min, sec := val.Clock()
|
||||
nanosec := val.Nanosecond()
|
||||
*t = LocalTime(time.Date(year, month, day, hour, min, sec, nanosec, loc))
|
||||
return nil
|
||||
case []byte:
|
||||
return t.parseTime(string(val), loc)
|
||||
case string:
|
||||
return t.parseTime(val, loc)
|
||||
default:
|
||||
return fmt.Errorf("UTCTime scan source was not string, []byte or time.Time: %T (%v)", v, v)
|
||||
}
|
||||
}
|
||||
|
||||
// parseTime handles parsing a string into UTCTime.
|
||||
func (t *LocalTime) parseTime(timeStr string, loc *time.Location) error {
|
||||
timeStr = strings.TrimSpace(timeStr)
|
||||
if timeStr == "" {
|
||||
*t = LocalTime(time.Time{})
|
||||
return nil
|
||||
}
|
||||
|
||||
layouts := []string{
|
||||
time.RFC3339Nano, time.RFC3339,
|
||||
"2006-01-02 15:04:05.0000000-07:00", "2006-01-02 15:04:05-07:00",
|
||||
"2006-01-02 15:04:05.0000000", "2006-01-02 15:04:05", "2006-01-02",
|
||||
}
|
||||
|
||||
for _, layout := range layouts {
|
||||
if parsedTime, err := time.ParseInLocation(layout, timeStr, loc); err == nil {
|
||||
*t = LocalTime(parsedTime)
|
||||
return nil
|
||||
}
|
||||
}
|
||||
return fmt.Errorf("unable to parse time string '%s' into UTCTime", timeStr)
|
||||
}
|
||||
|
||||
// MarshalJSON implements the json.Marshaler interface.
|
||||
// Serializes UTCTime to JSON in RFC3339 format with the correct timezone offset.
|
||||
func (t LocalTime) MarshalJSON() ([]byte, error) {
|
||||
if time.Time(t).IsZero() {
|
||||
return []byte("null"), nil
|
||||
}
|
||||
formattedTime := time.Time(t).In(GetAppLocation()).Format(time.RFC3339)
|
||||
return []byte(fmt.Sprintf(`"%s"`, formattedTime)), nil
|
||||
}
|
||||
|
||||
// ToTime converts UTCTime to Go's native time.Time type.
|
||||
func (t LocalTime) ToTime() time.Time { return time.Time(t) }
|
||||
|
||||
// FromTime converts Go's native time.Time type to UTCTime.
|
||||
func FromTime(t time.Time) LocalTime { return LocalTime(t) }
|
||||
|
||||
// Now returns the current time in the application's configured location.
|
||||
func Now() LocalTime { return LocalTime(time.Now().In(GetAppLocation())) }
|
||||
|
||||
var (
|
||||
appLocation *time.Location
|
||||
locationOnce sync.Once
|
||||
)
|
||||
|
||||
// GetAppLocation retrieves the application's global timezone from the "TZ" environment variable.
|
||||
func GetAppLocation() *time.Location {
|
||||
locationOnce.Do(func() {
|
||||
tz := os.Getenv("TZ")
|
||||
if tz == "" {
|
||||
tz = "UTC"
|
||||
}
|
||||
loc, err := time.LoadLocation(tz)
|
||||
if err != nil {
|
||||
log.Printf("Warning: Failed to load timezone '%s', falling back to UTC. Error: %v", tz, err)
|
||||
appLocation = time.UTC
|
||||
} else {
|
||||
appLocation = loc
|
||||
}
|
||||
time.Local = appLocation
|
||||
log.Printf("Application timezone is set to '%s'.", appLocation.String())
|
||||
})
|
||||
return appLocation
|
||||
}
|
||||
@@ -1,11 +1,13 @@
|
||||
package models
|
||||
|
||||
import "time"
|
||||
|
||||
type Clipboard struct {
|
||||
Id int `json:"id" gorm:"primaryKey;autoIncrement;unique"`
|
||||
Text string `json:"text" gorm:"type:longtext"`
|
||||
Name string `json:"name" gorm:"type:varchar(255)"`
|
||||
Weight int `json:"weight" gorm:"type:int"`
|
||||
Remark string `json:"remark" gorm:"type:text"`
|
||||
CreatedAt LocalTime `json:"created_at"`
|
||||
UpdatedAt LocalTime `json:"updated_at"`
|
||||
CreatedAt time.Time `json:"created_at"`
|
||||
UpdatedAt time.Time `json:"updated_at"`
|
||||
}
|
||||
@@ -1,10 +1,12 @@
|
||||
package models
|
||||
|
||||
import "time"
|
||||
|
||||
type Log struct {
|
||||
ID uint `json:"id,omitempty" gorm:"primaryKey;autoIncrement"`
|
||||
IP string `json:"ip" gorm:"type:varchar(45);"` // IPv4 or IPv6
|
||||
UUID string `json:"uuid" gorm:"type:varchar(36);"`
|
||||
Message string `json:"message" gorm:"type:text;not null"`
|
||||
MsgType string `json:"msg_type" gorm:"type:varchar(20);not null"`
|
||||
Time LocalTime `json:"time" gorm:"autoCreateTime;not null"`
|
||||
MsgType string `json:"msg_type" gorm:"type:varchar(20);not null;index:idx_logs_msg_type_time,priority:1"`
|
||||
Time time.Time `json:"time" gorm:"autoCreateTime;not null;index:idx_logs_time;index:idx_logs_msg_type_time,priority:2"`
|
||||
}
|
||||
+41
-40
@@ -4,43 +4,44 @@ import (
|
||||
"database/sql/driver"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Client represents a registered client device
|
||||
type Client struct {
|
||||
UUID string `json:"uuid,omitempty" gorm:"type:varchar(36);primaryKey"`
|
||||
Token string `json:"token,omitempty" gorm:"type:varchar(255);unique;not null"`
|
||||
Name string `json:"name" gorm:"type:varchar(100)"`
|
||||
CpuName string `json:"cpu_name" gorm:"type:varchar(100)"`
|
||||
Virtualization string `json:"virtualization" gorm:"type:varchar(50)"`
|
||||
Arch string `json:"arch" gorm:"type:varchar(50)"`
|
||||
CpuCores int `json:"cpu_cores" gorm:"type:int"`
|
||||
CpuPhysicalCores int `json:"cpu_physical_cores" gorm:"type:int"`
|
||||
OS string `json:"os" gorm:"type:varchar(100)"`
|
||||
KernelVersion string `json:"kernel_version" gorm:"type:varchar(100)"`
|
||||
GpuName string `json:"gpu_name" gorm:"type:varchar(100)"`
|
||||
IPv4 string `json:"ipv4,omitempty" gorm:"type:varchar(100)"`
|
||||
IPv6 string `json:"ipv6,omitempty" gorm:"type:varchar(100)"`
|
||||
Region string `json:"region" gorm:"type:varchar(100)"`
|
||||
Remark string `json:"remark,omitempty" gorm:"type:longtext"`
|
||||
PublicRemark string `json:"public_remark,omitempty" gorm:"type:longtext"`
|
||||
MemTotal int64 `json:"mem_total" gorm:"type:bigint"`
|
||||
SwapTotal int64 `json:"swap_total" gorm:"type:bigint"`
|
||||
DiskTotal int64 `json:"disk_total" gorm:"type:bigint"`
|
||||
Version string `json:"version,omitempty" gorm:"type:varchar(100)"`
|
||||
Weight int `json:"weight" gorm:"type:int"`
|
||||
Price float64 `json:"price"`
|
||||
BillingCycle int `json:"billing_cycle"`
|
||||
AutoRenewal bool `json:"auto_renewal" gorm:"default:false"` // 是否自动续费
|
||||
Currency string `json:"currency" gorm:"type:varchar(20);default:'$'"`
|
||||
ExpiredAt LocalTime `json:"expired_at" gorm:"type:timestamp"`
|
||||
Group string `json:"group" gorm:"type:varchar(100)"`
|
||||
Tags string `json:"tags" gorm:"type:text"` // split by ';'
|
||||
Hidden bool `json:"hidden" gorm:"default:false"`
|
||||
TrafficLimit int64 `json:"traffic_limit" gorm:"type:bigint"`
|
||||
TrafficLimitType string `json:"traffic_limit_type" gorm:"type:varchar(10);default:'max'"` // 流量阈值类型:sum max min up down
|
||||
CreatedAt LocalTime `json:"created_at"`
|
||||
UpdatedAt LocalTime `json:"updated_at"`
|
||||
UUID string `json:"uuid,omitempty" gorm:"type:varchar(36);primaryKey"`
|
||||
Token string `json:"token,omitempty" gorm:"type:varchar(255);unique;not null"`
|
||||
Name string `json:"name" gorm:"type:varchar(100)"`
|
||||
CpuName string `json:"cpu_name" gorm:"type:varchar(100)"`
|
||||
Virtualization string `json:"virtualization" gorm:"type:varchar(50)"`
|
||||
Arch string `json:"arch" gorm:"type:varchar(50)"`
|
||||
CpuCores int `json:"cpu_cores" gorm:"type:int"`
|
||||
CpuPhysicalCores int `json:"cpu_physical_cores" gorm:"type:int"`
|
||||
OS string `json:"os" gorm:"type:varchar(100)"`
|
||||
KernelVersion string `json:"kernel_version" gorm:"type:varchar(100)"`
|
||||
GpuName string `json:"gpu_name" gorm:"type:varchar(100)"`
|
||||
IPv4 string `json:"ipv4,omitempty" gorm:"type:varchar(100)"`
|
||||
IPv6 string `json:"ipv6,omitempty" gorm:"type:varchar(100)"`
|
||||
Region string `json:"region" gorm:"type:varchar(100)"`
|
||||
Remark string `json:"remark,omitempty" gorm:"type:longtext"`
|
||||
PublicRemark string `json:"public_remark,omitempty" gorm:"type:longtext"`
|
||||
MemTotal int64 `json:"mem_total" gorm:"type:bigint"`
|
||||
SwapTotal int64 `json:"swap_total" gorm:"type:bigint"`
|
||||
DiskTotal int64 `json:"disk_total" gorm:"type:bigint"`
|
||||
Version string `json:"version,omitempty" gorm:"type:varchar(100)"`
|
||||
Weight int `json:"weight" gorm:"type:int"`
|
||||
Price float64 `json:"price"`
|
||||
BillingCycle int `json:"billing_cycle"`
|
||||
AutoRenewal bool `json:"auto_renewal" gorm:"default:false"` // 是否自动续费
|
||||
Currency string `json:"currency" gorm:"type:varchar(20);default:'$'"`
|
||||
ExpiredAt *time.Time `json:"expired_at" gorm:"type:timestamp"`
|
||||
Group string `json:"group" gorm:"type:varchar(100)"`
|
||||
Tags string `json:"tags" gorm:"type:text"` // split by ';'
|
||||
Hidden bool `json:"hidden" gorm:"default:false"`
|
||||
TrafficLimit int64 `json:"traffic_limit" gorm:"type:bigint"`
|
||||
TrafficLimitType string `json:"traffic_limit_type" gorm:"type:varchar(10);default:'max'"` // 流量阈值类型:sum max min up down
|
||||
CreatedAt time.Time `json:"created_at"`
|
||||
UpdatedAt time.Time `json:"updated_at"`
|
||||
}
|
||||
|
||||
// User represents an authenticated user
|
||||
@@ -52,8 +53,8 @@ type User struct {
|
||||
SSOID string `json:"sso_id" gorm:"type:varchar(100)"` // OAuth provider's user ID
|
||||
TwoFactor string `json:"two_factor,omitempty" gorm:"type:varchar(255)"` // 2FA secret
|
||||
Sessions []Session `json:"sessions,omitempty" gorm:"foreignKey:UUID;references:UUID;constraint:OnDelete:CASCADE,OnUpdate:CASCADE"`
|
||||
CreatedAt LocalTime `json:"created_at"`
|
||||
UpdatedAt LocalTime `json:"updated_at"`
|
||||
CreatedAt time.Time `json:"created_at"`
|
||||
UpdatedAt time.Time `json:"updated_at"`
|
||||
}
|
||||
|
||||
// Session manages user sessions
|
||||
@@ -63,17 +64,17 @@ type Session struct {
|
||||
UserAgent string `json:"user_agent" gorm:"type:text"`
|
||||
Ip string `json:"ip" gorm:"type:varchar(100)"`
|
||||
LoginMethod string `json:"login_method" gorm:"type:varchar(50)"`
|
||||
LatestOnline LocalTime `json:"latest_online" gorm:"type:timestamp"`
|
||||
LatestOnline time.Time `json:"latest_online" gorm:"type:timestamp"`
|
||||
LatestUserAgent string `json:"latest_user_agent" gorm:"type:text"`
|
||||
LatestIp string `json:"latest_ip" gorm:"type:varchar(100)"`
|
||||
Expires LocalTime `json:"expires" gorm:"not null"`
|
||||
CreatedAt LocalTime `json:"created_at"`
|
||||
Expires time.Time `json:"expires" gorm:"not null"`
|
||||
CreatedAt time.Time `json:"created_at"`
|
||||
}
|
||||
|
||||
// Record logs client metrics over time
|
||||
type Record struct {
|
||||
Client string `json:"client" gorm:"type:varchar(36);index"`
|
||||
Time LocalTime `json:"time" gorm:"index"`
|
||||
Time time.Time `json:"time" gorm:"index"`
|
||||
Cpu float32 `json:"cpu" gorm:"type:decimal(5,2)"` // e.g., 75.50%
|
||||
Gpu float32 `json:"gpu" gorm:"type:decimal(5,2)"`
|
||||
Ram int64 `json:"ram" gorm:"type:bigint"`
|
||||
@@ -99,7 +100,7 @@ type Record struct {
|
||||
// GPURecord logs individual GPU metrics over time
|
||||
type GPURecord struct {
|
||||
Client string `json:"client" gorm:"type:varchar(36);index"` // 客户端UUID
|
||||
Time LocalTime `json:"time" gorm:"index"` // 记录时间
|
||||
Time time.Time `json:"time" gorm:"index"` // 记录时间
|
||||
DeviceIndex int `json:"device_index" gorm:"index"` // GPU设备索引 (0,1,2...)
|
||||
DeviceName string `json:"device_name" gorm:"type:varchar(100)"` // GPU型号
|
||||
MemTotal int64 `json:"mem_total" gorm:"type:bigint"` // 显存总量(字节)
|
||||
|
||||
@@ -1,13 +1,15 @@
|
||||
package models
|
||||
|
||||
import "time"
|
||||
|
||||
// Notification 定义了通知相关的数据库模型
|
||||
type OfflineNotification struct {
|
||||
Client string `json:"client" gorm:"type:varchar(36);not null;index;unique;constraint:OnDelete:CASCADE,OnUpdate:CASCADE;foreignKey:client;references:UUID"`
|
||||
ClientInfo Client `json:"client_info,omitempty" gorm:"foreignKey:Client;references:UUID"`
|
||||
Enable bool `json:"enable" gorm:"type:boolean;default:false"`
|
||||
//Cooldown int `json:"cooldown" gorm:"type:int;not null;default:1800"` // 冷却时间(秒),默认 30 分钟
|
||||
GracePeriod int `json:"grace_period" gorm:"type:int;not null;default:180"` // 宽限期(秒),默认 3 分钟
|
||||
LastNotified LocalTime `json:"last_notified"` // 上次通知时间
|
||||
GracePeriod int `json:"grace_period" gorm:"type:int;not null;default:180"` // 宽限期(秒),默认 3 分钟
|
||||
LastNotified *time.Time `json:"last_notified"` // 上次通知时间
|
||||
}
|
||||
|
||||
// LoadNotification 定义了基于资源占用达标时间比的负载通知规则
|
||||
@@ -19,7 +21,7 @@ type LoadNotification struct {
|
||||
Threshold float32 `json:"threshold" gorm:"type:decimal(5,2);not null;default:80.00"` // 阈值百分比
|
||||
Ratio float32 `json:"ratio" gorm:"type:decimal(5,2);not null;default:0.80"` // 达标时间比
|
||||
Interval int `json:"interval" gorm:"type:int;not null;default:15"` // 监测间隔(分钟)
|
||||
LastNotified LocalTime `json:"last_notified"` // 上次通知时间
|
||||
LastNotified *time.Time `json:"last_notified"` // 上次通知时间
|
||||
}
|
||||
|
||||
// TrafficReportNotification 定义了流量定时报告的数据库模型
|
||||
@@ -27,7 +29,7 @@ type TrafficReportNotification struct {
|
||||
Client string `json:"client" gorm:"type:varchar(36);not null;index;unique;constraint:OnDelete:CASCADE,OnUpdate:CASCADE;foreignKey:client;references:UUID"`
|
||||
ClientInfo Client `json:"client_info,omitempty" gorm:"foreignKey:Client;references:UUID"`
|
||||
Enable bool `json:"enable" gorm:"type:boolean;default:false"`
|
||||
Daily bool `json:"daily" gorm:"type:boolean;default:false"` // 日报
|
||||
Weekly bool `json:"weekly" gorm:"type:boolean;default:false"` // 周报
|
||||
Daily bool `json:"daily" gorm:"type:boolean;default:false"` // 日报
|
||||
Weekly bool `json:"weekly" gorm:"type:boolean;default:false"` // 周报
|
||||
Monthly bool `json:"monthly" gorm:"type:boolean;default:false"` // 月报
|
||||
}
|
||||
@@ -1,11 +1,13 @@
|
||||
package models
|
||||
|
||||
import "time"
|
||||
|
||||
type PingRecord struct {
|
||||
Client string `json:"client" gorm:"type:varchar(36);not null;index"`
|
||||
ClientInfo Client `json:"client_info" gorm:"foreignKey:Client;references:UUID;constraint:OnDelete:CASCADE,OnUpdate:CASCADE"`
|
||||
TaskId uint `json:"task_id" gorm:"not null;index"`
|
||||
Task PingTask `json:"task" gorm:"foreignKey:TaskId;references:Id;constraint:OnDelete:CASCADE,OnUpdate:CASCADE;"`
|
||||
Time LocalTime `json:"time" gorm:"index;not null"`
|
||||
Time time.Time `json:"time" gorm:"index;not null"`
|
||||
Value int `json:"value" gorm:"type:int;not null"` // Ping 值,单位毫秒
|
||||
}
|
||||
|
||||
|
||||
@@ -1,5 +1,7 @@
|
||||
package models
|
||||
|
||||
import "time"
|
||||
|
||||
type Task struct {
|
||||
TaskId string `json:"task_id" gorm:"type:varchar(36);primaryKey;unique"`
|
||||
Clients StringArray `json:"clients" gorm:"type:longtext"`
|
||||
@@ -13,6 +15,6 @@ type TaskResult struct {
|
||||
ClientInfo Client `json:"client_info" gorm:"foreignKey:Client;references:UUID;constraint:OnDelete:CASCADE,OnUpdate:CASCADE"`
|
||||
Result string `json:"result" gorm:"type:longtext"`
|
||||
ExitCode *int `json:"exit_code" gorm:"type:int"`
|
||||
FinishedAt *LocalTime `json:"finished_at" gorm:"type:timestamp"`
|
||||
CreatedAt LocalTime `json:"created_at" gorm:"type:timestamp"`
|
||||
FinishedAt *time.Time `json:"finished_at" gorm:"type:timestamp"`
|
||||
CreatedAt time.Time `json:"created_at" gorm:"type:timestamp"`
|
||||
}
|
||||
@@ -0,0 +1,77 @@
|
||||
package models
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"gorm.io/driver/sqlite"
|
||||
"gorm.io/gorm"
|
||||
"gorm.io/gorm/logger"
|
||||
)
|
||||
|
||||
func TestTimeFieldsStoreAndSerializeAsUTCWithNanoseconds(t *testing.T) {
|
||||
dsn := fmt.Sprintf("file:%s?mode=memory&cache=shared", strings.ReplaceAll(t.Name(), "/", "_"))
|
||||
db, err := gorm.Open(sqlite.Open(dsn), &gorm.Config{
|
||||
Logger: logger.Default.LogMode(logger.Silent),
|
||||
NowFunc: func() time.Time { return time.Now().UTC() },
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("open database: %v", err)
|
||||
}
|
||||
if err := db.AutoMigrate(&Log{}); err != nil {
|
||||
t.Fatalf("migrate log: %v", err)
|
||||
}
|
||||
|
||||
stamp := time.Date(2026, 7, 17, 1, 2, 3, 123456789, time.UTC)
|
||||
entry := Log{IP: "127.0.0.1", UUID: "test", Message: "test", MsgType: "info", Time: stamp}
|
||||
if err := db.Create(&entry).Error; err != nil {
|
||||
t.Fatalf("create log: %v", err)
|
||||
}
|
||||
var loaded Log
|
||||
if err := db.First(&loaded, entry.ID).Error; err != nil {
|
||||
t.Fatalf("load log: %v", err)
|
||||
}
|
||||
if !loaded.Time.Equal(stamp) || loaded.Time.Nanosecond() != stamp.Nanosecond() {
|
||||
t.Fatalf("loaded time = %s, want %s", loaded.Time, stamp)
|
||||
}
|
||||
|
||||
encoded, err := json.Marshal(loaded)
|
||||
if err != nil {
|
||||
t.Fatalf("marshal log: %v", err)
|
||||
}
|
||||
if !strings.Contains(string(encoded), `"time":"2026-07-17T01:02:03.123456789Z"`) {
|
||||
t.Fatalf("unexpected JSON: %s", encoded)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNullableTimeFieldReadsAndSerializesNull(t *testing.T) {
|
||||
dsn := fmt.Sprintf("file:%s?mode=memory&cache=shared", strings.ReplaceAll(t.Name(), "/", "_"))
|
||||
db, err := gorm.Open(sqlite.Open(dsn), &gorm.Config{Logger: logger.Default.LogMode(logger.Silent)})
|
||||
if err != nil {
|
||||
t.Fatalf("open database: %v", err)
|
||||
}
|
||||
if err := db.AutoMigrate(&Client{}); err != nil {
|
||||
t.Fatalf("migrate client: %v", err)
|
||||
}
|
||||
if err := db.Exec(`INSERT INTO clients (uuid, token, expired_at) VALUES (?, ?, NULL)`, "nullable", "token").Error; err != nil {
|
||||
t.Fatalf("insert client: %v", err)
|
||||
}
|
||||
|
||||
var client Client
|
||||
if err := db.First(&client, "uuid = ?", "nullable").Error; err != nil {
|
||||
t.Fatalf("load client: %v", err)
|
||||
}
|
||||
if client.ExpiredAt != nil {
|
||||
t.Fatalf("expired_at = %s, want nil", client.ExpiredAt)
|
||||
}
|
||||
encoded, err := json.Marshal(client)
|
||||
if err != nil {
|
||||
t.Fatalf("marshal client: %v", err)
|
||||
}
|
||||
if !strings.Contains(string(encoded), `"expired_at":null`) {
|
||||
t.Fatalf("unexpected JSON: %s", encoded)
|
||||
}
|
||||
}
|
||||
@@ -23,6 +23,11 @@ func GetOidcConfigByName(name string) (*models.OidcProvider, error) {
|
||||
return &config, nil
|
||||
}
|
||||
|
||||
func DeleteOidcConfigByName(name string) error {
|
||||
db := dbcore.GetDBInstance()
|
||||
return db.Delete(&models.OidcProvider{}, "name = ?", name).Error
|
||||
}
|
||||
|
||||
func SaveOidcConfig(config *models.OidcProvider) error {
|
||||
db := dbcore.GetDBInstance()
|
||||
if err := db.Save(config).Error; err != nil {
|
||||
|
||||
+15
-590
@@ -1,609 +1,34 @@
|
||||
package records
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"log"
|
||||
"sort"
|
||||
"strings"
|
||||
"context"
|
||||
"time"
|
||||
|
||||
"gorm.io/gorm"
|
||||
|
||||
"github.com/komari-monitor/komari/cmd/flags"
|
||||
"github.com/komari-monitor/komari/database/dbcore"
|
||||
"github.com/komari-monitor/komari/database/metricstore"
|
||||
"github.com/komari-monitor/komari/database/models"
|
||||
"github.com/komari-monitor/komari/utils"
|
||||
)
|
||||
|
||||
func RecordOne(rec models.Record) error {
|
||||
db := dbcore.GetDBInstance()
|
||||
return db.Create(&rec).Error
|
||||
}
|
||||
|
||||
func RecordGPU(rec models.GPURecord) error {
|
||||
db := dbcore.GetDBInstance()
|
||||
return db.Create(&rec).Error
|
||||
}
|
||||
// 历史监控数据已完全迁移到 metric store(默认 SQLite ./data/metrics.db,或配置的
|
||||
// MySQL/PostgreSQL)。旧的 records / records_long_term / gpu_records / ping_records
|
||||
// 表仅在启动一次性迁移时作为数据源读取,运行期的读写全部走 metric store。
|
||||
//
|
||||
// 因此本文件不再保留对旧表的任何读写、压缩(rollup)与流量增量修复逻辑:
|
||||
// metric store 自行管理 rollup 与保留策略,旧表压缩已无意义。
|
||||
|
||||
func DeleteAll() error {
|
||||
db := dbcore.GetDBInstance()
|
||||
if err := db.Exec("DELETE FROM records_long_term").Error; err != nil {
|
||||
return err
|
||||
}
|
||||
if err := db.Exec("DELETE FROM gpu_records_long_term").Error; err != nil {
|
||||
return err
|
||||
}
|
||||
if err := db.Exec("DELETE FROM gpu_records").Error; err != nil {
|
||||
return err
|
||||
}
|
||||
return db.Exec("DELETE FROM records").Error
|
||||
return metricstore.DeleteAllRecords(context.Background())
|
||||
}
|
||||
|
||||
// GetGPURecordsByClientAndTime 获取GPU记录数据
|
||||
// GetGPURecordsByClientAndTime 获取 GPU 记录数据。
|
||||
func GetGPURecordsByClientAndTime(uuid string, start, end time.Time) ([]models.GPURecord, error) {
|
||||
db := dbcore.GetDBInstance()
|
||||
var records []models.GPURecord
|
||||
|
||||
fourHoursAgo := time.Now().Add(-4*time.Hour - time.Minute)
|
||||
|
||||
var recentRecords []models.GPURecord
|
||||
recentStart := start
|
||||
if end.After(fourHoursAgo) {
|
||||
if recentStart.Before(fourHoursAgo) {
|
||||
recentStart = fourHoursAgo
|
||||
}
|
||||
err := db.Where("client = ? AND time >= ? AND time <= ?", uuid, recentStart, end).
|
||||
Order("time ASC, device_index ASC").Find(&recentRecords).Error
|
||||
if err != nil {
|
||||
log.Printf("Error fetching recent GPU records for client %s between %s and %s: %v", uuid, recentStart, end, err)
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
|
||||
var longTermRecords []models.GPURecord
|
||||
err := db.Table("gpu_records_long_term").Where("client = ? AND time >= ? AND time <= ?", uuid, start, end).
|
||||
Order("time ASC, device_index ASC").Find(&longTermRecords).Error
|
||||
if err != nil {
|
||||
log.Printf("Error fetching long-term GPU records for client %s between %s and %s: %v", uuid, start, end, err)
|
||||
return recentRecords, nil
|
||||
}
|
||||
|
||||
// 合并结果 - 不再需要类型转换
|
||||
records = append(records, recentRecords...)
|
||||
records = append(records, longTermRecords...)
|
||||
|
||||
return records, nil
|
||||
}
|
||||
|
||||
func GetLatestRecord(uuid string) (Record []models.Record, err error) {
|
||||
db := dbcore.GetDBInstance()
|
||||
err = db.Where("client = ?", uuid).Order("time DESC").Limit(1).Find(&Record).Error
|
||||
return
|
||||
}
|
||||
|
||||
func DeleteRecordBefore(before time.Time) error {
|
||||
db := dbcore.GetDBInstance()
|
||||
db.Table("records_long_term").Where("time < ?", before).Delete(&models.Record{})
|
||||
db.Table("gpu_records_long_term").Where("time < ?", before).Delete(&models.GPURecord{})
|
||||
db.Where("time < ?", before).Delete(&models.GPURecord{})
|
||||
return db.Where("time < ?", before).Delete(&models.Record{}).Error
|
||||
return metricstore.GetGPURecordsByClientAndTime(context.Background(), uuid, start, end)
|
||||
}
|
||||
|
||||
func GetRecordsByClientAndTime(uuid string, start, end time.Time) ([]models.Record, error) {
|
||||
db := dbcore.GetDBInstance()
|
||||
var records []models.Record
|
||||
|
||||
fourHoursAgo := time.Now().Add(-4*time.Hour - time.Minute)
|
||||
|
||||
var recentRecords []models.Record
|
||||
recentStart := start
|
||||
if end.After(fourHoursAgo) {
|
||||
if recentStart.Before(fourHoursAgo) {
|
||||
recentStart = fourHoursAgo
|
||||
}
|
||||
err := db.Where("client = ? AND time >= ? AND time <= ?", uuid, recentStart, end).Order("time ASC").Find(&recentRecords).Error
|
||||
if err != nil {
|
||||
log.Printf("Error fetching recent records for client %s between %s and %s: %v", uuid, recentStart, end, err)
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
|
||||
var long_term []models.Record
|
||||
err := db.Table("records_long_term").Where("client = ? AND time >= ? AND time <= ?", uuid, start, end).Order("time ASC").Find(&long_term).Error
|
||||
if err != nil {
|
||||
log.Printf("Error fetching long-term records for client %s between %s and %s: %v", uuid, start, end, err)
|
||||
return recentRecords, nil
|
||||
}
|
||||
|
||||
if len(long_term) == 0 {
|
||||
// 没有查到long_term,返回全部recentRecords
|
||||
records = append(records, recentRecords...)
|
||||
return records, nil
|
||||
}
|
||||
|
||||
// 查到了long_term,recentRecords按15分钟分组,每组只保留一条(取最新一条)
|
||||
grouped := make(map[string]models.Record)
|
||||
for _, rec := range recentRecords {
|
||||
key := rec.Time.ToTime().Truncate(15 * time.Minute).Format(time.RFC3339)
|
||||
if old, ok := grouped[key]; !ok || rec.Time.ToTime().After(old.Time.ToTime()) {
|
||||
grouped[key] = rec
|
||||
}
|
||||
}
|
||||
var groupedList []models.Record
|
||||
for _, rec := range grouped {
|
||||
groupedList = append(groupedList, rec)
|
||||
}
|
||||
sort.Slice(groupedList, func(i, j int) bool {
|
||||
return groupedList[i].Time.ToTime().Before(groupedList[j].Time.ToTime())
|
||||
})
|
||||
records = append(records, groupedList...)
|
||||
records = append(records, long_term...)
|
||||
return records, nil
|
||||
return metricstore.GetRecordsByClientAndTime(context.Background(), uuid, start, end)
|
||||
}
|
||||
|
||||
func GetAllRecords() ([]models.Record, error) {
|
||||
db := dbcore.GetDBInstance()
|
||||
var records []models.Record
|
||||
var long_term []models.Record
|
||||
err := db.Table("records").Order("time ASC").Find(&records).Error
|
||||
if err != nil {
|
||||
log.Printf("Error fetching all records: %v", err)
|
||||
return nil, err
|
||||
}
|
||||
err = db.Table("records_long_term").Order("time ASC").Find(&long_term).Error
|
||||
if err != nil {
|
||||
log.Printf("Error fetching long-term records: %v", err)
|
||||
return records, nil
|
||||
}
|
||||
records = append(records, long_term...)
|
||||
return records, nil
|
||||
}
|
||||
|
||||
// 压缩数据库
|
||||
func CompactRecord() error {
|
||||
db := dbcore.GetDBInstance()
|
||||
err := migrateOldRecords(db)
|
||||
if err != nil {
|
||||
log.Printf("Error migrating old records: %v", err)
|
||||
return err
|
||||
}
|
||||
|
||||
err = migrateGPURecords(db)
|
||||
if err != nil {
|
||||
log.Printf("Error migrating GPU records: %v", err)
|
||||
return err
|
||||
}
|
||||
|
||||
if flags.IsSQLite() {
|
||||
db.Exec("PRAGMA wal_checkpoint(PASSIVE);")
|
||||
}
|
||||
//log.Printf("Record compaction completed")
|
||||
return nil
|
||||
}
|
||||
|
||||
func migrateOldRecords(db *gorm.DB) error {
|
||||
return migrateOldRecordsAt(db, time.Now())
|
||||
}
|
||||
|
||||
func compactRecordCutoff(now time.Time) time.Time {
|
||||
return now.Add(-4 * time.Hour).Truncate(15 * time.Minute)
|
||||
}
|
||||
|
||||
func migrateOldRecordsAt(db *gorm.DB, now time.Time) error {
|
||||
cutoff := compactRecordCutoff(now)
|
||||
|
||||
// 查询 records 表中超过 4 小时的记录
|
||||
var records []models.Record
|
||||
if err := db.Table("records").Where("time < ?", cutoff).Find(&records).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if len(records) == 0 {
|
||||
return nil
|
||||
}
|
||||
previousByClient, err := getPreviousTrafficRecordsBefore(db, records)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
repairZeroTrafficDeltas(records, previousByClient)
|
||||
|
||||
// 按 Client 和 15 分钟时间段分组,并存储所有记录以计算分位数
|
||||
type groupData struct {
|
||||
Cpu []float32
|
||||
Gpu []float32
|
||||
Load []float32
|
||||
Temp []float32
|
||||
Ram []int64
|
||||
RamTotal []int64
|
||||
Swap []int64
|
||||
SwapTotal []int64
|
||||
Disk []int64
|
||||
DiskTotal []int64
|
||||
NetIn []int64
|
||||
NetOut []int64
|
||||
NetTotalUp []int64
|
||||
NetTotalDown []int64
|
||||
TrafficUp int64
|
||||
TrafficDown int64
|
||||
LatestTime time.Time
|
||||
LatestTotalUp int64
|
||||
LatestTotalDown int64
|
||||
Process []int
|
||||
Connections []int
|
||||
ConnectionsUdp []int
|
||||
Uptime []int64
|
||||
}
|
||||
|
||||
groupedRecords := make(map[string]*groupData)
|
||||
for _, record := range records {
|
||||
key := record.Client + "_" + record.Time.ToTime().Truncate(15*time.Minute).Format(time.RFC3339)
|
||||
if _, ok := groupedRecords[key]; !ok {
|
||||
groupedRecords[key] = &groupData{}
|
||||
}
|
||||
data := groupedRecords[key]
|
||||
data.Cpu = append(data.Cpu, record.Cpu)
|
||||
data.Gpu = append(data.Gpu, record.Gpu)
|
||||
data.Load = append(data.Load, record.Load)
|
||||
data.Temp = append(data.Temp, record.Temp)
|
||||
data.Ram = append(data.Ram, record.Ram)
|
||||
data.RamTotal = append(data.RamTotal, record.RamTotal)
|
||||
data.Swap = append(data.Swap, record.Swap)
|
||||
data.SwapTotal = append(data.SwapTotal, record.SwapTotal)
|
||||
data.Disk = append(data.Disk, record.Disk)
|
||||
data.DiskTotal = append(data.DiskTotal, record.DiskTotal)
|
||||
data.NetIn = append(data.NetIn, record.NetIn)
|
||||
data.NetOut = append(data.NetOut, record.NetOut)
|
||||
data.NetTotalUp = append(data.NetTotalUp, record.NetTotalUp)
|
||||
data.NetTotalDown = append(data.NetTotalDown, record.NetTotalDown)
|
||||
data.TrafficUp += record.TrafficUp
|
||||
data.TrafficDown += record.TrafficDown
|
||||
if data.LatestTime.IsZero() || record.Time.ToTime().After(data.LatestTime) {
|
||||
data.LatestTime = record.Time.ToTime()
|
||||
data.LatestTotalUp = record.NetTotalUp
|
||||
data.LatestTotalDown = record.NetTotalDown
|
||||
}
|
||||
data.Process = append(data.Process, record.Process)
|
||||
data.Connections = append(data.Connections, record.Connections)
|
||||
data.ConnectionsUdp = append(data.ConnectionsUdp, record.ConnectionsUdp)
|
||||
//data.Uptime = append(data.Uptime, record.Uptime)
|
||||
}
|
||||
|
||||
getPercentile := func(values []float64, percentile float64) float64 {
|
||||
if len(values) == 0 {
|
||||
return 0
|
||||
}
|
||||
sortedValues := make([]float64, len(values))
|
||||
copy(sortedValues, values)
|
||||
sort.Float64s(sortedValues)
|
||||
index := float64(len(sortedValues)-1) * percentile
|
||||
lowerIndex := int(index)
|
||||
if lowerIndex >= len(sortedValues)-1 {
|
||||
return sortedValues[len(sortedValues)-1]
|
||||
}
|
||||
frac := index - float64(lowerIndex)
|
||||
return sortedValues[lowerIndex] + frac*(sortedValues[lowerIndex+1]-sortedValues[lowerIndex])
|
||||
}
|
||||
|
||||
getIntPercentile := func(values []int64, percentile float64) int64 {
|
||||
if len(values) == 0 {
|
||||
return 0
|
||||
}
|
||||
floats := make([]float64, len(values))
|
||||
for i, v := range values {
|
||||
floats[i] = float64(v)
|
||||
}
|
||||
return int64(getPercentile(floats, percentile))
|
||||
}
|
||||
|
||||
getInt32Percentile := func(values []int, percentile float64) int {
|
||||
if len(values) == 0 {
|
||||
return 0
|
||||
}
|
||||
floats := make([]float64, len(values))
|
||||
for i, v := range values {
|
||||
floats[i] = float64(v)
|
||||
}
|
||||
return int(getPercentile(floats, percentile))
|
||||
}
|
||||
|
||||
return db.Transaction(func(tx *gorm.DB) error {
|
||||
for key, data := range groupedRecords {
|
||||
// 解析 Client 和时间
|
||||
parts := strings.Split(key, "_")
|
||||
clientUUID := parts[0]
|
||||
timeSlot, err := time.Parse(time.RFC3339, strings.Join(parts[1:], "_"))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
cpuFloats := make([]float64, len(data.Cpu))
|
||||
for i, v := range data.Cpu {
|
||||
cpuFloats[i] = float64(v)
|
||||
}
|
||||
gpuFloats := make([]float64, len(data.Gpu))
|
||||
for i, v := range data.Gpu {
|
||||
gpuFloats[i] = float64(v)
|
||||
}
|
||||
loadFloats := make([]float64, len(data.Load))
|
||||
for i, v := range data.Load {
|
||||
loadFloats[i] = float64(v)
|
||||
}
|
||||
tempFloats := make([]float64, len(data.Temp))
|
||||
for i, v := range data.Temp {
|
||||
tempFloats[i] = float64(v)
|
||||
}
|
||||
// 取高位
|
||||
high_percentile := 0.7
|
||||
// 检查 records_long_term 表中是否已存在相同的记录
|
||||
var existingCount int64
|
||||
if err := tx.Table("records_long_term").Where("client = ? AND time = ?", clientUUID, timeSlot).Count(&existingCount).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
newRec := models.Record{
|
||||
Client: clientUUID,
|
||||
Time: models.FromTime(timeSlot),
|
||||
Cpu: float32(getPercentile(cpuFloats, high_percentile)),
|
||||
Gpu: float32(getPercentile(gpuFloats, high_percentile)),
|
||||
Load: float32(getPercentile(loadFloats, high_percentile)),
|
||||
Temp: float32(getPercentile(tempFloats, high_percentile)),
|
||||
Ram: getIntPercentile(data.Ram, high_percentile),
|
||||
RamTotal: getIntPercentile(data.RamTotal, high_percentile),
|
||||
Swap: getIntPercentile(data.Swap, high_percentile),
|
||||
SwapTotal: getIntPercentile(data.SwapTotal, high_percentile),
|
||||
Disk: getIntPercentile(data.Disk, high_percentile),
|
||||
DiskTotal: getIntPercentile(data.DiskTotal, high_percentile),
|
||||
NetIn: getIntPercentile(data.NetIn, 0.2),
|
||||
NetOut: getIntPercentile(data.NetOut, 0.2),
|
||||
NetTotalUp: data.LatestTotalUp,
|
||||
NetTotalDown: data.LatestTotalDown,
|
||||
TrafficUp: data.TrafficUp,
|
||||
TrafficDown: data.TrafficDown,
|
||||
Process: getInt32Percentile(data.Process, high_percentile),
|
||||
Connections: getInt32Percentile(data.Connections, high_percentile),
|
||||
ConnectionsUdp: getInt32Percentile(data.ConnectionsUdp, high_percentile),
|
||||
//Uptime: getIntPercentile(data.Uptime, high_percentile),
|
||||
}
|
||||
|
||||
// 如果记录已存在则更新,否则创建新记录
|
||||
if existingCount > 0 {
|
||||
if err := tx.Table("records_long_term").Where("client = ? AND time = ?", clientUUID, timeSlot).Updates(&newRec).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
} else {
|
||||
if err := tx.Table("records_long_term").Create(&newRec).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 删除 records 表中的旧数据
|
||||
if err := tx.Table("records").Where("time < ?", cutoff.Add(-1*time.Hour)).Delete(&models.Record{}).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
})
|
||||
}
|
||||
|
||||
func getPreviousTrafficRecordsBefore(db *gorm.DB, records []models.Record) (map[string]*models.Record, error) {
|
||||
firstTimeByClient := make(map[string]time.Time)
|
||||
for _, record := range records {
|
||||
recordTime := record.Time.ToTime()
|
||||
firstTime, ok := firstTimeByClient[record.Client]
|
||||
if !ok || recordTime.Before(firstTime) {
|
||||
firstTimeByClient[record.Client] = recordTime
|
||||
}
|
||||
}
|
||||
|
||||
previousByClient := make(map[string]*models.Record, len(firstTimeByClient))
|
||||
for clientUUID, firstTime := range firstTimeByClient {
|
||||
previous, err := getPreviousTrafficRecordBefore(db, clientUUID, firstTime)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if previous != nil {
|
||||
previousByClient[clientUUID] = previous
|
||||
}
|
||||
}
|
||||
return previousByClient, nil
|
||||
}
|
||||
|
||||
func getPreviousTrafficRecordBefore(db *gorm.DB, clientUUID string, before time.Time) (*models.Record, error) {
|
||||
var latest *models.Record
|
||||
for _, table := range []string{"records", "records_long_term"} {
|
||||
var record models.Record
|
||||
queryBefore := before
|
||||
if table == "records_long_term" {
|
||||
queryBefore = before.Truncate(15 * time.Minute)
|
||||
}
|
||||
err := db.Table(table).
|
||||
Where("client = ? AND time < ?", clientUUID, models.FromTime(queryBefore)).
|
||||
Order("time DESC").
|
||||
First(&record).Error
|
||||
if err == nil {
|
||||
if latest == nil || record.Time.ToTime().After(latest.Time.ToTime()) {
|
||||
latest = &record
|
||||
}
|
||||
continue
|
||||
}
|
||||
if !errors.Is(err, gorm.ErrRecordNotFound) {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
return latest, nil
|
||||
}
|
||||
|
||||
func repairZeroTrafficDeltas(records []models.Record, previousByClient map[string]*models.Record) {
|
||||
recordsByClient := make(map[string][]*models.Record)
|
||||
for i := range records {
|
||||
recordsByClient[records[i].Client] = append(recordsByClient[records[i].Client], &records[i])
|
||||
}
|
||||
|
||||
for _, clientRecords := range recordsByClient {
|
||||
sort.Slice(clientRecords, func(i, j int) bool {
|
||||
return clientRecords[i].Time.ToTime().Before(clientRecords[j].Time.ToTime())
|
||||
})
|
||||
var previous *models.Record
|
||||
if previousByClient != nil {
|
||||
previous = previousByClient[clientRecords[0].Client]
|
||||
}
|
||||
for _, current := range clientRecords {
|
||||
if previous == nil {
|
||||
previous = current
|
||||
continue
|
||||
}
|
||||
if current.TrafficUp == 0 {
|
||||
current.TrafficUp = utils.ComputeTrafficDelta(current.NetTotalUp, previous.NetTotalUp)
|
||||
}
|
||||
if current.TrafficDown == 0 {
|
||||
current.TrafficDown = utils.ComputeTrafficDelta(current.NetTotalDown, previous.NetTotalDown)
|
||||
}
|
||||
previous = current
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// migrateGPURecords 压缩GPU记录数据
|
||||
func migrateGPURecords(db *gorm.DB) error {
|
||||
cutoff := compactRecordCutoff(time.Now())
|
||||
|
||||
// 查询超过4小时的GPU记录
|
||||
var gpuRecords []models.GPURecord
|
||||
if err := db.Where("time < ?", cutoff).Find(&gpuRecords).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if len(gpuRecords) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
// 按Client + DeviceIndex + 15分钟时间窗口分组
|
||||
type gpuGroupKey struct {
|
||||
Client string
|
||||
DeviceIndex int
|
||||
TimeSlot time.Time
|
||||
DeviceName string
|
||||
}
|
||||
|
||||
type gpuGroupData struct {
|
||||
MemTotal []int64
|
||||
MemUsed []int64
|
||||
Utilization []float32
|
||||
Temperature []int
|
||||
}
|
||||
|
||||
groupedGPUs := make(map[gpuGroupKey]*gpuGroupData)
|
||||
|
||||
for _, record := range gpuRecords {
|
||||
key := gpuGroupKey{
|
||||
Client: record.Client,
|
||||
DeviceIndex: record.DeviceIndex,
|
||||
TimeSlot: record.Time.ToTime().Truncate(15 * time.Minute),
|
||||
DeviceName: record.DeviceName,
|
||||
}
|
||||
|
||||
if _, ok := groupedGPUs[key]; !ok {
|
||||
groupedGPUs[key] = &gpuGroupData{}
|
||||
}
|
||||
|
||||
data := groupedGPUs[key]
|
||||
data.MemTotal = append(data.MemTotal, record.MemTotal)
|
||||
data.MemUsed = append(data.MemUsed, record.MemUsed)
|
||||
data.Utilization = append(data.Utilization, record.Utilization)
|
||||
data.Temperature = append(data.Temperature, record.Temperature)
|
||||
}
|
||||
|
||||
// 百分位数计算函数 (复用传统Record压缩逻辑)
|
||||
getPercentile := func(values []float64, percentile float64) float64 {
|
||||
if len(values) == 0 {
|
||||
return 0
|
||||
}
|
||||
sortedValues := make([]float64, len(values))
|
||||
copy(sortedValues, values)
|
||||
sort.Float64s(sortedValues)
|
||||
index := float64(len(sortedValues)-1) * percentile
|
||||
lowerIndex := int(index)
|
||||
if lowerIndex >= len(sortedValues)-1 {
|
||||
return sortedValues[len(sortedValues)-1]
|
||||
}
|
||||
frac := index - float64(lowerIndex)
|
||||
return sortedValues[lowerIndex] + frac*(sortedValues[lowerIndex+1]-sortedValues[lowerIndex])
|
||||
}
|
||||
|
||||
getIntPercentile := func(values []int64, percentile float64) int64 {
|
||||
if len(values) == 0 {
|
||||
return 0
|
||||
}
|
||||
floats := make([]float64, len(values))
|
||||
for i, v := range values {
|
||||
floats[i] = float64(v)
|
||||
}
|
||||
return int64(getPercentile(floats, percentile))
|
||||
}
|
||||
|
||||
// 温度数据转换辅助函数
|
||||
convertIntToInt64 := func(values []int) []int64 {
|
||||
result := make([]int64, len(values))
|
||||
for i, v := range values {
|
||||
result[i] = int64(v)
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
getFloat32Percentile := func(values []float32, percentile float64) float32 {
|
||||
if len(values) == 0 {
|
||||
return 0
|
||||
}
|
||||
floats := make([]float64, len(values))
|
||||
for i, v := range values {
|
||||
floats[i] = float64(v)
|
||||
}
|
||||
return float32(getPercentile(floats, percentile))
|
||||
}
|
||||
|
||||
// 保持与传统Record压缩的一致性
|
||||
high_percentile := 0.7
|
||||
|
||||
return db.Transaction(func(tx *gorm.DB) error {
|
||||
for key, data := range groupedGPUs {
|
||||
// 检查是否已存在记录
|
||||
var existingCount int64
|
||||
if err := tx.Table("gpu_records_long_term").Where("client = ? AND device_index = ? AND time = ?",
|
||||
key.Client, key.DeviceIndex, key.TimeSlot).Count(&existingCount).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
compressedGPU := models.GPURecord{
|
||||
Client: key.Client,
|
||||
DeviceIndex: key.DeviceIndex,
|
||||
Time: models.FromTime(key.TimeSlot),
|
||||
DeviceName: key.DeviceName,
|
||||
MemTotal: getIntPercentile(data.MemTotal, high_percentile),
|
||||
MemUsed: getIntPercentile(data.MemUsed, high_percentile),
|
||||
Utilization: getFloat32Percentile(data.Utilization, high_percentile),
|
||||
Temperature: int(getIntPercentile(convertIntToInt64(data.Temperature), high_percentile)),
|
||||
}
|
||||
|
||||
if existingCount > 0 {
|
||||
// 更新已存在记录
|
||||
if err := tx.Table("gpu_records_long_term").Where("client = ? AND device_index = ? AND time = ?",
|
||||
key.Client, key.DeviceIndex, key.TimeSlot).Updates(&compressedGPU).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
} else {
|
||||
// 创建新记录
|
||||
if err := tx.Table("gpu_records_long_term").Create(&compressedGPU).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 删除已压缩的原始GPU数据
|
||||
if err := tx.Where("time < ?", cutoff.Add(-1*time.Hour)).Delete(&models.GPURecord{}).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
})
|
||||
// GetRecordsByTime 获取所有客户端在时间范围内的记录。
|
||||
func GetRecordsByTime(start, end time.Time) ([]models.Record, error) {
|
||||
return metricstore.GetRecordsByTime(context.Background(), start, end)
|
||||
}
|
||||
@@ -1,321 +0,0 @@
|
||||
package records
|
||||
|
||||
import (
|
||||
"encoding/csv"
|
||||
"os"
|
||||
"strconv"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
"gorm.io/driver/sqlite"
|
||||
"gorm.io/gorm"
|
||||
|
||||
"github.com/komari-monitor/komari/database/models"
|
||||
)
|
||||
|
||||
var uuid = "7901508c-304f-49aa-b84f-957c33ae6f8a"
|
||||
|
||||
var _ = func() bool {
|
||||
// 确保 Test 环境中使用 sqlite 内存数据库
|
||||
return true
|
||||
}()
|
||||
|
||||
// TestCompactRecord tests the database compaction logic by inserting 4h30m of data (one record per minute),
|
||||
// then running migrateOldRecords and verifying the aggregation and cleanup.
|
||||
func TestCompactRecord(t *testing.T) {
|
||||
const totalMinutes = 12*60 + 30
|
||||
loc := models.GetAppLocation()
|
||||
now := time.Date(2026, 6, 15, 14, 30, 0, 0, loc)
|
||||
threshold := compactRecordCutoff(now)
|
||||
overlapCutoff := threshold.Add(-1 * time.Hour)
|
||||
|
||||
// 使用 sqlite 内存数据库并迁移表结构
|
||||
db, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{})
|
||||
assert.NoError(t, err)
|
||||
assert.NoError(t, db.AutoMigrate(&models.Record{}))
|
||||
assert.NoError(t, db.Table("records_long_term").AutoMigrate(&models.Record{}))
|
||||
|
||||
expectedGroups := make(map[time.Time]struct{})
|
||||
expectedRawRemain := 0
|
||||
|
||||
// 插入数据
|
||||
for i := 0; i < totalMinutes; i++ {
|
||||
recTime := now.Add(-time.Duration(i) * time.Minute)
|
||||
rec := models.Record{Client: uuid, Time: models.FromTime(recTime), Cpu: float32(i), Gpu: float32(i), Load: float32(i), Temp: float32(i), Ram: int64(i)}
|
||||
err := db.Create(&rec).Error
|
||||
assert.NoError(t, err)
|
||||
|
||||
if recTime.Before(threshold) {
|
||||
slot := recTime.Truncate(15 * time.Minute)
|
||||
expectedGroups[slot] = struct{}{}
|
||||
}
|
||||
if !recTime.Before(overlapCutoff) {
|
||||
expectedRawRemain++
|
||||
}
|
||||
}
|
||||
|
||||
// 导出原始数据到 CSV
|
||||
os.MkdirAll("../../data", 0755)
|
||||
var origRecs []models.Record
|
||||
db.Order("time desc").Find(&origRecs)
|
||||
fOrig, err := os.Create("../../data/original.csv")
|
||||
assert.NoError(t, err)
|
||||
defer fOrig.Close()
|
||||
wOrig := csv.NewWriter(fOrig)
|
||||
defer wOrig.Flush()
|
||||
wOrig.Write([]string{"Client", "Time", "Cpu", "Gpu", "Load", "Temp", "Ram"})
|
||||
for _, r := range origRecs {
|
||||
wOrig.Write([]string{
|
||||
r.Client,
|
||||
r.Time.ToTime().Format(time.RFC3339),
|
||||
strconv.FormatFloat(float64(r.Cpu), 'f', -1, 32),
|
||||
strconv.FormatFloat(float64(r.Gpu), 'f', -1, 32),
|
||||
strconv.FormatFloat(float64(r.Load), 'f', -1, 32),
|
||||
strconv.FormatFloat(float64(r.Temp), 'f', -1, 32),
|
||||
strconv.FormatInt(r.Ram, 10),
|
||||
})
|
||||
}
|
||||
|
||||
// 运行压缩(迁移)逻辑
|
||||
err = migrateOldRecordsAt(db, now)
|
||||
assert.NoError(t, err)
|
||||
|
||||
// 验证 long-term 表中的聚合记录数
|
||||
var longCount int64
|
||||
assert.NoError(t, db.Table("records_long_term").Count(&longCount).Error)
|
||||
assert.Equal(t, int64(len(expectedGroups)), longCount)
|
||||
|
||||
// 验证原始表中剩余记录数
|
||||
var remainCount int64
|
||||
assert.NoError(t, db.Table("records").Count(&remainCount).Error)
|
||||
assert.Equal(t, int64(expectedRawRemain), remainCount)
|
||||
|
||||
// 导出压缩后的数据到 CSV
|
||||
var compRecs []models.Record
|
||||
db.Table("records_long_term").Order("time desc").Find(&compRecs)
|
||||
fComp, err := os.Create("../../data/compressed.csv")
|
||||
assert.NoError(t, err)
|
||||
defer fComp.Close()
|
||||
wComp := csv.NewWriter(fComp)
|
||||
defer wComp.Flush()
|
||||
wComp.Write([]string{"Client", "Time", "Cpu", "Gpu", "Load", "Temp", "Ram"})
|
||||
for _, r := range compRecs {
|
||||
wComp.Write([]string{
|
||||
r.Client,
|
||||
r.Time.ToTime().Format(time.RFC3339),
|
||||
strconv.FormatFloat(float64(r.Cpu), 'f', -1, 32),
|
||||
strconv.FormatFloat(float64(r.Gpu), 'f', -1, 32),
|
||||
strconv.FormatFloat(float64(r.Load), 'f', -1, 32),
|
||||
strconv.FormatFloat(float64(r.Temp), 'f', -1, 32),
|
||||
strconv.FormatInt(r.Ram, 10),
|
||||
})
|
||||
}
|
||||
|
||||
db.Table("records").Order("time desc").Find(&compRecs)
|
||||
fComp, err = os.Create("../../data/compressed_records.csv")
|
||||
assert.NoError(t, err)
|
||||
defer fComp.Close()
|
||||
wComp = csv.NewWriter(fComp)
|
||||
defer wComp.Flush()
|
||||
wComp.Write([]string{"Client", "Time", "Cpu", "Gpu", "Load", "Temp", "Ram"})
|
||||
for _, r := range compRecs {
|
||||
wComp.Write([]string{
|
||||
r.Client,
|
||||
r.Time.ToTime().Format(time.RFC3339),
|
||||
strconv.FormatFloat(float64(r.Cpu), 'f', -1, 32),
|
||||
strconv.FormatFloat(float64(r.Gpu), 'f', -1, 32),
|
||||
strconv.FormatFloat(float64(r.Load), 'f', -1, 32),
|
||||
strconv.FormatFloat(float64(r.Temp), 'f', -1, 32),
|
||||
strconv.FormatInt(r.Ram, 10),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestCompactRecordPreservesExactTrafficDelta(t *testing.T) {
|
||||
loc := models.GetAppLocation()
|
||||
currentTime := time.Date(2026, 6, 15, 14, 30, 0, 0, loc)
|
||||
now := currentTime.Truncate(15 * time.Minute).Add(-5*time.Hour + time.Minute)
|
||||
db, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{})
|
||||
assert.NoError(t, err)
|
||||
assert.NoError(t, db.AutoMigrate(&models.Record{}))
|
||||
assert.NoError(t, db.Table("records_long_term").AutoMigrate(&models.Record{}))
|
||||
|
||||
records := []models.Record{
|
||||
{
|
||||
Client: uuid,
|
||||
Time: models.FromTime(now),
|
||||
NetTotalUp: 100,
|
||||
NetTotalDown: 200,
|
||||
TrafficUp: 0,
|
||||
TrafficDown: 0,
|
||||
},
|
||||
{
|
||||
Client: uuid,
|
||||
Time: models.FromTime(now.Add(5 * time.Minute)),
|
||||
NetTotalUp: 150,
|
||||
NetTotalDown: 260,
|
||||
TrafficUp: 50,
|
||||
TrafficDown: 60,
|
||||
},
|
||||
{
|
||||
Client: uuid,
|
||||
Time: models.FromTime(now.Add(10 * time.Minute)),
|
||||
NetTotalUp: 10,
|
||||
NetTotalDown: 30,
|
||||
TrafficUp: 10,
|
||||
TrafficDown: 30,
|
||||
},
|
||||
}
|
||||
|
||||
for _, rec := range records {
|
||||
assert.NoError(t, db.Create(&rec).Error)
|
||||
}
|
||||
|
||||
assert.NoError(t, migrateOldRecordsAt(db, currentTime))
|
||||
|
||||
var compacted []models.Record
|
||||
assert.NoError(t, db.Table("records_long_term").Find(&compacted).Error)
|
||||
require.Len(t, compacted, 1)
|
||||
assert.Equal(t, int64(60), compacted[0].TrafficUp)
|
||||
assert.Equal(t, int64(90), compacted[0].TrafficDown)
|
||||
assert.Equal(t, int64(10), compacted[0].NetTotalUp)
|
||||
assert.Equal(t, int64(30), compacted[0].NetTotalDown)
|
||||
assert.True(t, compacted[0].Time.ToTime().Equal(records[2].Time.ToTime().Truncate(15*time.Minute)))
|
||||
}
|
||||
|
||||
func TestRepairZeroTrafficDeltasPreservesRawResetDetailBeforeCompaction(t *testing.T) {
|
||||
loc := models.GetAppLocation()
|
||||
start := time.Date(2026, 6, 6, 0, 0, 0, 0, loc)
|
||||
records := []models.Record{
|
||||
{Client: uuid, Time: models.FromTime(start), NetTotalUp: 100, NetTotalDown: 200},
|
||||
{Client: uuid, Time: models.FromTime(start.Add(5 * time.Minute)), NetTotalUp: 140, NetTotalDown: 260},
|
||||
{Client: uuid, Time: models.FromTime(start.Add(10 * time.Minute)), NetTotalUp: 10, NetTotalDown: 20},
|
||||
{Client: uuid, Time: models.FromTime(start.Add(15 * time.Minute)), NetTotalUp: 25, NetTotalDown: 35},
|
||||
}
|
||||
|
||||
repairZeroTrafficDeltas(records, nil)
|
||||
|
||||
assert.Equal(t, int64(0), records[0].TrafficUp)
|
||||
assert.Equal(t, int64(0), records[0].TrafficDown)
|
||||
assert.Equal(t, int64(40), records[1].TrafficUp)
|
||||
assert.Equal(t, int64(60), records[1].TrafficDown)
|
||||
assert.Equal(t, int64(10), records[2].TrafficUp)
|
||||
assert.Equal(t, int64(20), records[2].TrafficDown)
|
||||
assert.Equal(t, int64(15), records[3].TrafficUp)
|
||||
assert.Equal(t, int64(15), records[3].TrafficDown)
|
||||
}
|
||||
|
||||
func TestRepairZeroTrafficDeltasUsesPreviousPersistedBaseline(t *testing.T) {
|
||||
loc := models.GetAppLocation()
|
||||
now := time.Date(2026, 6, 7, 12, 0, 0, 0, loc)
|
||||
db, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{})
|
||||
assert.NoError(t, err)
|
||||
assert.NoError(t, db.AutoMigrate(&models.Record{}))
|
||||
assert.NoError(t, db.Table("records_long_term").AutoMigrate(&models.Record{}))
|
||||
|
||||
assert.NoError(t, db.Table("records_long_term").Create(&models.Record{
|
||||
Client: uuid,
|
||||
Time: models.FromTime(now.Add(-10 * time.Minute)),
|
||||
NetTotalUp: 100,
|
||||
NetTotalDown: 200,
|
||||
}).Error)
|
||||
|
||||
rawRecords := []models.Record{
|
||||
{Client: uuid, Time: models.FromTime(now), NetTotalUp: 150, NetTotalDown: 260},
|
||||
{Client: uuid, Time: models.FromTime(now.Add(5 * time.Minute)), NetTotalUp: 175, NetTotalDown: 300},
|
||||
}
|
||||
|
||||
previousByClient, err := getPreviousTrafficRecordsBefore(db, rawRecords)
|
||||
assert.NoError(t, err)
|
||||
repairZeroTrafficDeltas(rawRecords, previousByClient)
|
||||
|
||||
assert.Equal(t, int64(50), rawRecords[0].TrafficUp)
|
||||
assert.Equal(t, int64(60), rawRecords[0].TrafficDown)
|
||||
assert.Equal(t, int64(25), rawRecords[1].TrafficUp)
|
||||
assert.Equal(t, int64(40), rawRecords[1].TrafficDown)
|
||||
}
|
||||
|
||||
func TestRepairZeroTrafficDeltasIgnoresSameSlotLongTermBaseline(t *testing.T) {
|
||||
loc := models.GetAppLocation()
|
||||
now := time.Date(2026, 6, 7, 12, 0, 30, 0, loc)
|
||||
db, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{})
|
||||
assert.NoError(t, err)
|
||||
assert.NoError(t, db.AutoMigrate(&models.Record{}))
|
||||
assert.NoError(t, db.Table("records_long_term").AutoMigrate(&models.Record{}))
|
||||
|
||||
assert.NoError(t, db.Table("records_long_term").Create(&models.Record{
|
||||
Client: uuid,
|
||||
Time: models.FromTime(now.Truncate(15 * time.Minute)),
|
||||
NetTotalUp: 900,
|
||||
NetTotalDown: 900,
|
||||
}).Error)
|
||||
|
||||
rawRecords := []models.Record{
|
||||
{Client: uuid, Time: models.FromTime(now), NetTotalUp: 150, NetTotalDown: 260},
|
||||
{Client: uuid, Time: models.FromTime(now.Add(5 * time.Minute)), NetTotalUp: 175, NetTotalDown: 300},
|
||||
}
|
||||
|
||||
previousByClient, err := getPreviousTrafficRecordsBefore(db, rawRecords)
|
||||
assert.NoError(t, err)
|
||||
repairZeroTrafficDeltas(rawRecords, previousByClient)
|
||||
|
||||
assert.Equal(t, int64(0), rawRecords[0].TrafficUp)
|
||||
assert.Equal(t, int64(0), rawRecords[0].TrafficDown)
|
||||
assert.Equal(t, int64(25), rawRecords[1].TrafficUp)
|
||||
assert.Equal(t, int64(40), rawRecords[1].TrafficDown)
|
||||
}
|
||||
|
||||
func TestCompactRecordRetainsOneHourOverlapWindow(t *testing.T) {
|
||||
loc := models.GetAppLocation()
|
||||
now := time.Date(2026, 6, 7, 12, 7, 0, 0, loc)
|
||||
cutoff := compactRecordCutoff(now)
|
||||
db, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{})
|
||||
assert.NoError(t, err)
|
||||
assert.NoError(t, db.AutoMigrate(&models.Record{}))
|
||||
assert.NoError(t, db.Table("records_long_term").AutoMigrate(&models.Record{}))
|
||||
|
||||
records := []models.Record{
|
||||
{Client: uuid, Time: models.FromTime(cutoff.Add(-time.Hour - time.Minute)), TrafficUp: 5},
|
||||
{Client: uuid, Time: models.FromTime(cutoff.Add(-30 * time.Minute)), TrafficUp: 7},
|
||||
{Client: uuid, Time: models.FromTime(now.Add(-3 * time.Hour)), TrafficUp: 11},
|
||||
}
|
||||
for _, rec := range records {
|
||||
assert.NoError(t, db.Create(&rec).Error)
|
||||
}
|
||||
|
||||
assert.NoError(t, migrateOldRecordsAt(db, now))
|
||||
|
||||
var remainTimes []models.Record
|
||||
assert.NoError(t, db.Table("records").Order("time ASC").Find(&remainTimes).Error)
|
||||
require.Len(t, remainTimes, 2)
|
||||
assert.True(t, remainTimes[0].Time.ToTime().Equal(records[1].Time.ToTime()))
|
||||
assert.True(t, remainTimes[1].Time.ToTime().Equal(records[2].Time.ToTime()))
|
||||
}
|
||||
|
||||
func TestCompactRecordOnlyMigratesCompleteFifteenMinuteBuckets(t *testing.T) {
|
||||
loc := models.GetAppLocation()
|
||||
now := time.Date(2026, 6, 7, 12, 7, 0, 0, loc)
|
||||
cutoff := compactRecordCutoff(now)
|
||||
db, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{})
|
||||
assert.NoError(t, err)
|
||||
assert.NoError(t, db.AutoMigrate(&models.Record{}))
|
||||
assert.NoError(t, db.Table("records_long_term").AutoMigrate(&models.Record{}))
|
||||
|
||||
compactable := models.Record{Client: uuid, Time: models.FromTime(cutoff.Add(-time.Minute)), TrafficUp: 5}
|
||||
partialSlot := models.Record{Client: uuid, Time: models.FromTime(cutoff.Add(time.Minute)), TrafficUp: 7}
|
||||
assert.NoError(t, db.Create(&compactable).Error)
|
||||
assert.NoError(t, db.Create(&partialSlot).Error)
|
||||
|
||||
assert.NoError(t, migrateOldRecordsAt(db, now))
|
||||
|
||||
var compacted []models.Record
|
||||
assert.NoError(t, db.Table("records_long_term").Order("time ASC").Find(&compacted).Error)
|
||||
require.Len(t, compacted, 1)
|
||||
assert.True(t, compacted[0].Time.ToTime().Equal(compactable.Time.ToTime().Truncate(15*time.Minute)))
|
||||
|
||||
var rawCount int64
|
||||
assert.NoError(t, db.Table("records").Where("time = ?", partialSlot.Time).Count(&rawCount).Error)
|
||||
assert.Equal(t, int64(1), rawCount)
|
||||
}
|
||||
+18
-36
@@ -1,9 +1,11 @@
|
||||
package tasks
|
||||
|
||||
import (
|
||||
"context"
|
||||
"time"
|
||||
|
||||
"github.com/komari-monitor/komari/database/dbcore"
|
||||
"github.com/komari-monitor/komari/database/metricstore"
|
||||
"github.com/komari-monitor/komari/database/models"
|
||||
"github.com/komari-monitor/komari/utils"
|
||||
"gorm.io/gorm"
|
||||
@@ -45,6 +47,12 @@ func AddPingTask(clients []string, defaultOn bool, name string, target, task_typ
|
||||
}
|
||||
|
||||
func DeletePingTask(id []uint) error {
|
||||
// The metric store is independent from the main database, so clean it first
|
||||
// to avoid leaving history that can no longer be addressed through the task.
|
||||
if err := DeletePingRecords(id); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
db := dbcore.GetDBInstance()
|
||||
result := db.Where("id IN ?", id).Delete(&models.PingTask{})
|
||||
if result.RowsAffected == 0 {
|
||||
@@ -125,38 +133,24 @@ func UpdatePingTaskOrder(order map[uint]int) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func SavePingRecord(record models.PingRecord) error {
|
||||
db := dbcore.GetDBInstance()
|
||||
return db.Create(&record).Error
|
||||
}
|
||||
// ping 记录已完全迁移到 metric store(指标 ping.latency_ms),运行期读写全部走
|
||||
// metric store,旧 ping_records 表不再参与。
|
||||
|
||||
func DeletePingRecordsBefore(time time.Time) error {
|
||||
db := dbcore.GetDBInstance()
|
||||
err := db.Where("time < ?", time).Delete(&models.PingRecord{}).Error
|
||||
return err
|
||||
func SavePingRecord(record models.PingRecord) error {
|
||||
return metricstore.WritePingRecord(context.Background(), record)
|
||||
}
|
||||
|
||||
func DeletePingRecords(id []uint) error {
|
||||
db := dbcore.GetDBInstance()
|
||||
result := db.Where("task_id IN ?", id).Delete(&models.PingRecord{})
|
||||
if result.RowsAffected == 0 {
|
||||
return gorm.ErrRecordNotFound
|
||||
}
|
||||
return result.Error
|
||||
return metricstore.DeletePingRecordsByTask(context.Background(), id)
|
||||
}
|
||||
|
||||
func DeleteAllPingRecords() error {
|
||||
db := dbcore.GetDBInstance()
|
||||
result := db.Exec("DELETE FROM ping_records")
|
||||
if result.RowsAffected == 0 {
|
||||
return gorm.ErrRecordNotFound
|
||||
}
|
||||
return result.Error
|
||||
return metricstore.DeleteAllPingRecords(context.Background())
|
||||
}
|
||||
|
||||
func ReloadPingSchedule() error {
|
||||
db := dbcore.GetDBInstance()
|
||||
var pingTasks []models.PingTask
|
||||
if err := db.Find(&pingTasks).Error; err != nil {
|
||||
pingTasks, err := GetAllPingTasks()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return utils.ReloadPingSchedule(pingTasks)
|
||||
@@ -201,17 +195,5 @@ func AddDefaultOnClientUUID(uuid string) error {
|
||||
}
|
||||
|
||||
func GetPingRecords(uuid string, taskId int, start, end time.Time) ([]models.PingRecord, error) {
|
||||
db := dbcore.GetDBInstance()
|
||||
var records []models.PingRecord
|
||||
dbQuery := db.Model(&models.PingRecord{})
|
||||
if uuid != "" {
|
||||
dbQuery = dbQuery.Where("client = ?", uuid)
|
||||
}
|
||||
if taskId >= 0 {
|
||||
dbQuery = dbQuery.Where("task_id = ?", uint(taskId))
|
||||
}
|
||||
if err := dbQuery.Where("time >= ? AND time <= ?", start, end).Order("time DESC").Find(&records).Error; err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return records, nil
|
||||
return metricstore.GetPingRecords(context.Background(), uuid, taskId, start, end)
|
||||
}
|
||||
+4
-15
@@ -26,7 +26,7 @@ func CreateTask(taskId string, clients []string, command string) error {
|
||||
Result: "",
|
||||
ExitCode: nil,
|
||||
FinishedAt: nil,
|
||||
CreatedAt: models.FromTime(time.Now()),
|
||||
CreatedAt: time.Now().UTC(),
|
||||
})
|
||||
}
|
||||
if len(taskResults) > 0 {
|
||||
@@ -57,13 +57,6 @@ func GetSpecificTaskResult(taskId, clientId string) (*models.TaskResult, error)
|
||||
return &result, nil
|
||||
}
|
||||
|
||||
func GetAllTasksResultByUUID(uuid string) ([]models.TaskResult, error) {
|
||||
var results []models.TaskResult
|
||||
if err := dbcore.GetDBInstance().Where("client = ?", uuid).Find(&results).Error; err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return results, nil
|
||||
}
|
||||
func GetAllTasks() ([]models.Task, error) {
|
||||
var tasks []models.Task
|
||||
if err := dbcore.GetDBInstance().Find(&tasks).Error; err != nil {
|
||||
@@ -79,21 +72,17 @@ func GetTaskResultsByTaskId(taskId string) ([]models.TaskResult, error) {
|
||||
}
|
||||
return results, nil
|
||||
}
|
||||
func DeleteTaskByTaskId(taskId string) error {
|
||||
return dbcore.GetDBInstance().Where("task_id = ?", taskId).Delete(&models.Task{}).Error
|
||||
}
|
||||
|
||||
func SaveTaskResult(taskId, clientId, result string, exitCode int, timestamp models.LocalTime) error {
|
||||
func SaveTaskResult(taskId, clientId, result string, exitCode int, timestamp time.Time) error {
|
||||
return dbcore.GetDBInstance().
|
||||
Model(&models.TaskResult{}).
|
||||
Where("task_id = ? AND client = ?", taskId, clientId).
|
||||
Updates(map[string]interface{}{
|
||||
"result": result,
|
||||
"exit_code": exitCode,
|
||||
"finished_at": timestamp,
|
||||
"finished_at": timestamp.UTC(),
|
||||
}).Error
|
||||
}
|
||||
|
||||
func ClearTaskResultsByTimeBefore(before time.Time) error {
|
||||
return dbcore.GetDBInstance().Where("created_at < ?", before.Format(time.RFC3339)).Delete(&models.TaskResult{}).Error
|
||||
return dbcore.GetDBInstance().Where("created_at < ?", before.UTC()).Delete(&models.TaskResult{}).Error
|
||||
}
|
||||
@@ -0,0 +1,68 @@
|
||||
package tasks
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/komari-monitor/komari/cmd/flags"
|
||||
"github.com/komari-monitor/komari/database/dbcore"
|
||||
"github.com/komari-monitor/komari/database/models"
|
||||
)
|
||||
|
||||
func TestClearTaskResultsByTimeBeforeUsesUTCTimeValue(t *testing.T) {
|
||||
flags.DatabaseType = flags.DatabaseTypeSQLite
|
||||
flags.DatabaseFile = "file:task_cleanup_time?mode=memory&cache=shared"
|
||||
db := dbcore.GetDBInstance()
|
||||
|
||||
taskID := "task-cleanup-time"
|
||||
if err := db.Create(&models.Task{TaskId: taskID, Command: "true"}).Error; err != nil {
|
||||
t.Fatalf("create task: %v", err)
|
||||
}
|
||||
cutoff := time.Date(2026, 7, 17, 12, 0, 0, 123456789, time.UTC)
|
||||
results := []models.TaskResult{
|
||||
{TaskId: taskID, Client: "old", CreatedAt: cutoff.Add(-time.Nanosecond)},
|
||||
{TaskId: taskID, Client: "boundary", CreatedAt: cutoff},
|
||||
{TaskId: taskID, Client: "new", CreatedAt: cutoff.Add(time.Nanosecond)},
|
||||
}
|
||||
if err := db.Create(&results).Error; err != nil {
|
||||
t.Fatalf("create task results: %v", err)
|
||||
}
|
||||
|
||||
localCutoff := cutoff.In(time.FixedZone("UTC+8", 8*60*60))
|
||||
if err := ClearTaskResultsByTimeBefore(localCutoff); err != nil {
|
||||
t.Fatalf("clear task results: %v", err)
|
||||
}
|
||||
var remaining []models.TaskResult
|
||||
if err := db.Where("task_id = ?", taskID).Order("created_at").Find(&remaining).Error; err != nil {
|
||||
t.Fatalf("load remaining results: %v", err)
|
||||
}
|
||||
if len(remaining) != 2 || remaining[0].Client != "boundary" || remaining[1].Client != "new" {
|
||||
t.Fatalf("remaining results = %#v, want boundary and new", remaining)
|
||||
}
|
||||
}
|
||||
|
||||
func TestGetAllPingTasksOrdersByWeightThenID(t *testing.T) {
|
||||
flags.DatabaseType = flags.DatabaseTypeSQLite
|
||||
flags.DatabaseFile = "file:ping_task_order?mode=memory&cache=shared"
|
||||
db := dbcore.GetDBInstance()
|
||||
|
||||
tasks := []models.PingTask{
|
||||
{Name: "third", Weight: 2, Type: "icmp", Target: "third.example", Interval: 60},
|
||||
{Name: "first", Weight: 0, Type: "icmp", Target: "first.example", Interval: 60},
|
||||
{Name: "second", Weight: 0, Type: "icmp", Target: "second.example", Interval: 60},
|
||||
}
|
||||
if err := db.Create(&tasks).Error; err != nil {
|
||||
t.Fatalf("create ping tasks: %v", err)
|
||||
}
|
||||
|
||||
ordered, err := GetAllPingTasks()
|
||||
if err != nil {
|
||||
t.Fatalf("get ordered ping tasks: %v", err)
|
||||
}
|
||||
if len(ordered) != len(tasks) {
|
||||
t.Fatalf("ordered ping task count = %d, want %d", len(ordered), len(tasks))
|
||||
}
|
||||
if ordered[0].Id != tasks[1].Id || ordered[1].Id != tasks[2].Id || ordered[2].Id != tasks[0].Id {
|
||||
t.Fatalf("ping task order = %#v, want ids [%d %d %d]", ordered, tasks[1].Id, tasks[2].Id, tasks[0].Id)
|
||||
}
|
||||
}
|
||||
+13
-16
@@ -1,16 +1,18 @@
|
||||
package database
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"log"
|
||||
logger "github.com/komari-monitor/komari/utils/log"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
|
||||
"github.com/gin-gonic/gin"
|
||||
"github.com/komari-monitor/komari/database/dbcore"
|
||||
"github.com/komari-monitor/komari/database/metricstore"
|
||||
"github.com/komari-monitor/komari/database/models"
|
||||
"github.com/komari-monitor/komari/pkg/config"
|
||||
"github.com/komari-monitor/komari/internal/config"
|
||||
)
|
||||
|
||||
func GetPublicInfo() (map[string]interface{}, error) {
|
||||
@@ -42,15 +44,6 @@ func GetPublicInfo() (map[string]interface{}, error) {
|
||||
if !hasKey("o_auth_provider") {
|
||||
cst.OAuthProvider = "github"
|
||||
}
|
||||
if !hasKey("record_enabled") {
|
||||
cst.RecordEnabled = true
|
||||
}
|
||||
if !hasKey("record_preserve_time") {
|
||||
cst.RecordPreserveTime = 720
|
||||
}
|
||||
if !hasKey("ping_record_preserve_time") {
|
||||
cst.PingRecordPreserveTime = 24
|
||||
}
|
||||
|
||||
// Fallback defaults if we couldn't enumerate keys.
|
||||
if allErr != nil {
|
||||
@@ -61,7 +54,10 @@ func GetPublicInfo() (map[string]interface{}, error) {
|
||||
cst.Description = "Komari Monitor, a simple server monitoring tool."
|
||||
}
|
||||
}
|
||||
|
||||
retention, err := metricstore.GetRetentionSummary(context.Background())
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
db := dbcore.GetDBInstance()
|
||||
tc := models.ThemeConfiguration{}
|
||||
err = db.Model(&models.ThemeConfiguration{}).Where("short = ?", cst.Theme).First(&tc).Error
|
||||
@@ -71,7 +67,7 @@ func GetPublicInfo() (map[string]interface{}, error) {
|
||||
tc_data := gin.H{}
|
||||
err = json.Unmarshal([]byte(tc.Data), &tc_data)
|
||||
if err != nil {
|
||||
log.Printf("%v", err)
|
||||
logger.Infof("database", "%v", err)
|
||||
}
|
||||
// Try to load theme declaration file and merge defaults for managed configuration
|
||||
// Theme declarations live in ./data/theme/<short>/komari-theme.json
|
||||
@@ -135,10 +131,11 @@ func GetPublicInfo() (map[string]interface{}, error) {
|
||||
"oauth_provider": cst.OAuthProvider,
|
||||
"disable_password_login": cst.DisablePasswordLogin,
|
||||
"cors_origin_check_enabled": cst.CorsOriginCheckEnabled,
|
||||
"record_enabled": cst.RecordEnabled,
|
||||
"record_preserve_time": cst.RecordPreserveTime,
|
||||
"ping_record_preserve_time": cst.PingRecordPreserveTime,
|
||||
"record_enabled": retention.AllPositive, // 兼容旧版本主题
|
||||
"record_preserve_time": retention.MaxDays * 24,
|
||||
"ping_record_preserve_time": retention.MaxDays * 24,
|
||||
"private_site": cst.PrivateSite,
|
||||
"visitor_audit_enabled": cst.VisitorAuditEnabled,
|
||||
"theme": cst.Theme,
|
||||
"theme_settings": tc_data,
|
||||
}, nil
|
||||
|
||||
@@ -1,148 +0,0 @@
|
||||
# Komari
|
||||
|
||||

|
||||
|
||||

|
||||
|
||||
Komariは、サーバーのパフォーマンスを監視するためのシンプルで効率的なソリューションを提供することを目的とした、軽量の自己ホスト型サーバー監視ツールです。Webインターフェースを介してサーバーのステータスを表示し、軽量エージェントを介してデータを収集します。
|
||||
|
||||
> [!WARNING]
|
||||
> Komariは、自己ホスト型の監視/制御プログラムであり、所有しているシステム、または管理権限を得ているシステムでのみ使用してください。Komariを武器化したり、許可なく展開、アクセス、永続化、コマンド実行、その他の不正利用に使用したりしないでください。実際の悪用リスクについては、Huntressの分析をご参照ください:[Komari C2 agent abuse](https://www.huntress.com/blog/komari-c2-agent-abuse)。
|
||||
> Komariの展開および運用方法については、利用者自身が責任を負います。開発者は、無許可または不正な利用、およびその結果について責任を負いません。
|
||||
> Windows端末でリモートコントロールを有効にした場合、クライアントはユーザーがログインするたびに、KomariがリモートコントロールソフトウェアであることをWindows通知で知らせます。
|
||||
|
||||
[ドキュメント](https://komari-document.pages.dev/) | [Telegramグループ](https://t.me/komari_monitor)
|
||||
|
||||
## 特徴
|
||||
|
||||
- **軽量で効率的**: リソース消費が少なく、あらゆる規模のサーバーに適しています。
|
||||
- **自己ホスト型**: データプライバシーを完全に制御でき、展開も簡単です。
|
||||
- **Webインターフェース**: 直感的な監視ダッシュボードで、使いやすいです。
|
||||
|
||||
## クイックスタート
|
||||
|
||||
### 0. クラウドホスティングによるワンクリック展開
|
||||
|
||||
- Rainyun - CNY 4.5/月
|
||||
|
||||
[](https://app.rainyun.com/apps/rca/store/6780/NzYxNzAz_)
|
||||
|
||||
- 1Panel アプリストア
|
||||
|
||||
1Panel アプリストアで利用可能です。「アプリストア」>「ユーティリティ」>「Komari」からインストールしてください。
|
||||
|
||||
### 1. ワンクリックインストールスクリプトを使用する
|
||||
|
||||
systemdを使用するディストリビューション(Ubuntu、Debianなど)に適しています。
|
||||
|
||||
```bash
|
||||
curl -fsSL https://raw.githubusercontent.com/komari-monitor/komari/main/install-komari.sh -o install-komari.sh
|
||||
chmod +x install-komari.sh
|
||||
sudo ./install-komari.sh
|
||||
```
|
||||
|
||||
### 2. Docker展開
|
||||
|
||||
1. データディレクトリを作成します:
|
||||
```bash
|
||||
mkdir -p ./data
|
||||
```
|
||||
2. Dockerコンテナを実行します:
|
||||
```bash
|
||||
docker run -d \
|
||||
-p 25774:25774 \
|
||||
-v $(pwd)/data:/app/data \
|
||||
--name komari \
|
||||
ghcr.io/komari-monitor/komari:latest
|
||||
```
|
||||
3. デフォルトのユーザー名とパスワードを表示します:
|
||||
```bash
|
||||
docker logs komari
|
||||
```
|
||||
4. ブラウザで `http://<your_server_ip>:25774` にアクセスします。
|
||||
|
||||
> [!NOTE]
|
||||
> 環境変数 `ADMIN_USERNAME` と `ADMIN_PASSWORD` を使用して、初期のユーザー名とパスワードをカスタマイズすることもできます。
|
||||
|
||||
### 3. バイナリファイル展開
|
||||
|
||||
1. Komariの[GitHubリリース](https://github.com/komari-monitor/komari/releases)ページにアクセスして、お使いのオペレーティングシステム用の最新のバイナリをダウンロードします。
|
||||
2. Komariを実行します:
|
||||
```bash
|
||||
./komari server -l 0.0.0.0:25774
|
||||
```
|
||||
3. ブラウザで `http://<your_server_ip>:25774` にアクセスします。デフォルトのポートは `25774` です。
|
||||
4. デフォルトのユーザー名とパスワードは、起動ログで確認するか、環境変数 `ADMIN_USERNAME` と `ADMIN_PASSWORD` を介して設定できます。
|
||||
|
||||
> [!NOTE]
|
||||
> バイナリに実行権限があることを確認してください(`chmod +x komari`)。データは実行ディレクトリの `data` フォルダに保存されます。
|
||||
|
||||
### 手動ビルド
|
||||
|
||||
#### 依存関係
|
||||
|
||||
- Go 1.18+ および Node.js 20+(手動ビルド用)
|
||||
|
||||
1. フロントエンドの静的ファイルをビルドします:
|
||||
```bash
|
||||
git clone https://github.com/komari-monitor/komari-web
|
||||
cd komari-web
|
||||
npm install
|
||||
npm run build
|
||||
```
|
||||
2. バックエンドをビルドします:
|
||||
```bash
|
||||
git clone https://github.com/komari-monitor/komari
|
||||
cd komari
|
||||
```
|
||||
ステップ1で生成された静的ファイルを `komari` プロジェクトのルートにある `/web/public/defaultTheme/dist` フォルダにコピーし、`komari-theme.json` と `preview.png`/`perview.png` を `/web/public/defaultTheme` にコピーします。
|
||||
```bash
|
||||
go build -o komari
|
||||
```
|
||||
3. 実行:
|
||||
```bash
|
||||
./komari server -l 0.0.0.0:25774
|
||||
```
|
||||
デフォルトのリスニングポートは `25774` です。`http://localhost:25774` にアクセスします。
|
||||
|
||||
## フロントエンド開発ガイド
|
||||
|
||||
[Komariテーマ開発ガイド | Komari](https://komari-document.pages.dev/dev/theme.html)
|
||||
|
||||
[CrowdinでKomariを翻訳する](https://crowdin.com/project/komari/invite?h=cd051bf172c9a9f7f1360e87ffb521692507706)
|
||||
|
||||
## クライアントエージェント開発ガイド
|
||||
|
||||
[Komariエージェント情報レポートおよびイベント処理ドキュメント](https://komari-document.pages.dev/dev/agent.html)
|
||||
|
||||
## 貢献
|
||||
|
||||
IssueやPull Requestを歓迎します!
|
||||
|
||||
## 謝辞
|
||||
|
||||
### 破碎工坊云 (Crash Work)
|
||||
|
||||
[破碎工坊云 - 効率的で安定した、安全な高防御サーバーとCDNソリューションを提供する専門的なクラウドコンピューティングプラットフォーム](https://www.crash.work/)
|
||||
|
||||
### DreamCloud
|
||||
|
||||
[DreamCloud - コストパフォーマンスに優れたアジア太平洋向け高防御直通](https://as211392.com/)
|
||||
|
||||
### 🚀 SharonNetworks スポンサー
|
||||
|
||||
[](https://sharon.io)
|
||||
|
||||
SharonNetworks は、あなたのビジネスの離陸を力強くサポートします!
|
||||
|
||||
アジア太平洋のデータセンターから中国最適化ネットワーク接続を提供。低レイテンシー & 高帯域幅、Tbps 級のローカル洗浄 DDoS 防御で、ビジネスとお客様の体験を守ります。コミュニティ [Telegram グループ](https://t.me/SharonNetwork) に参加すると、チャリティまたはグループ内抽選で無料利用のチャンスがあります。
|
||||
|
||||
### オープンソースコミュニティ
|
||||
|
||||
PR を送ってくれた方、テーマを作成してくれた全ての開発者
|
||||
|
||||
—— そして:こんなに暇でいられる自分に感謝
|
||||
|
||||
## Star履歴
|
||||
|
||||
[](https://www.star-history.com/#komari-monitor/komari&Date)
|
||||
@@ -1,148 +0,0 @@
|
||||
# Komari
|
||||
|
||||

|
||||
|
||||

|
||||
|
||||
Komari 是一款輕量級的自託管伺服器監控工具,旨在提供簡單、高效的伺服器性能監控解決方案。它支援透過 Web 介面查看伺服器狀態,並透過輕量級 Agent 收集數據。
|
||||
|
||||
> [!WARNING]
|
||||
> Komari 是一款自託管的監控/控制程式,僅應部署在你擁有或已獲授權管理的系統上。請勿將 Komari 武器化,或在未獲授權的情況下部署、存取、持久化、執行命令及從事其他濫用行為。關於現實中的濫用風險,可參考 Huntress 的分析:[Komari C2 agent abuse](https://www.huntress.com/blog/komari-c2-agent-abuse)。
|
||||
> 使用者需要自行承擔部署和使用 Komari 的責任。開發者不對未經授權或濫用行為及其後果承擔責任。
|
||||
> 在 Windows 端啟用遠端控制後,客戶端會在每次使用者登入時透過 Windows 通知提醒使用者 Komari 是一款遠端控制軟體。
|
||||
|
||||
[文檔](https://komari-document.pages.dev/) | [Telegram 群組](https://t.me/komari_monitor)
|
||||
|
||||
## 特性
|
||||
|
||||
- **輕量高效**:低資源佔用,適合各種規模的伺服器。
|
||||
- **自託管**:完全掌控數據隱私,部署簡單。
|
||||
- **Web 介面**:直觀的監控儀表盤,易於使用。
|
||||
|
||||
## 快速開始
|
||||
|
||||
### 0. 容器雲一鍵部署
|
||||
|
||||
- 雨雲雲應用 - CNY 4.5/月
|
||||
|
||||
[](https://app.rainyun.com/apps/rca/store/6780/NzYxNzAz_)
|
||||
|
||||
- 1Panel 應用商店
|
||||
|
||||
已上架 1Panel 應用商店,應用商店-實用工具-Komari 即可安裝
|
||||
|
||||
### 1. 使用一鍵安裝腳本
|
||||
|
||||
適用於使用了 systemd 的發行版(Ubuntu、Debian...)。
|
||||
|
||||
```bash
|
||||
curl -fsSL https://raw.githubusercontent.com/komari-monitor/komari/main/install-komari.sh -o install-komari.sh
|
||||
chmod +x install-komari.sh
|
||||
sudo ./install-komari.sh
|
||||
```
|
||||
|
||||
### 2. Docker 部署
|
||||
|
||||
1. 建立資料目錄:
|
||||
```bash
|
||||
mkdir -p ./data
|
||||
```
|
||||
2. 執行 Docker 容器:
|
||||
```bash
|
||||
docker run -d \
|
||||
-p 25774:25774 \
|
||||
-v $(pwd)/data:/app/data \
|
||||
--name komari \
|
||||
ghcr.io/komari-monitor/komari:latest
|
||||
```
|
||||
3. 查看預設帳號和密碼:
|
||||
```bash
|
||||
docker logs komari
|
||||
```
|
||||
4. 在瀏覽器中存取 `http://<your_server_ip>:25774`。
|
||||
|
||||
> [!NOTE]
|
||||
> 你也可以透過環境變數 `ADMIN_USERNAME` 和 `ADMIN_PASSWORD` 自訂初始使用者名稱和密碼。
|
||||
|
||||
### 3. 二進位檔案部署
|
||||
|
||||
1. 存取 Komari 的 [GitHub Release 頁面](https://github.com/komari-monitor/komari/releases) 下載適用於你作業系統的最新二進位檔案。
|
||||
2. 執行 Komari:
|
||||
```bash
|
||||
./komari server -l 0.0.0.0:25774
|
||||
```
|
||||
3. 在瀏覽器中存取 `http://<your_server_ip>:25774`,預設監聽 `25774` 連接埠。
|
||||
4. 預設帳號和密碼可在啟動日誌中查看,或透過環境變數 `ADMIN_USERNAME` 和 `ADMIN_PASSWORD` 設定。
|
||||
|
||||
> [!NOTE]
|
||||
> 確保二進位檔案具有可執行權限(`chmod +x komari`)。資料將保存在執行目錄下的 `data` 資料夾中。
|
||||
|
||||
### 手工建置
|
||||
|
||||
#### 依賴
|
||||
|
||||
- Go 1.18+ 和 Node.js 20+(手工建置)
|
||||
|
||||
1. 建置前端靜態檔案:
|
||||
```bash
|
||||
git clone https://github.com/komari-monitor/komari-web
|
||||
cd komari-web
|
||||
npm install
|
||||
npm run build
|
||||
```
|
||||
2. 建置後端:
|
||||
```bash
|
||||
git clone https://github.com/komari-monitor/komari
|
||||
cd komari
|
||||
```
|
||||
將步驟1中產生的靜態檔案複製到 `komari` 專案根目錄下的 `/web/public/defaultTheme/dist` 資料夾,並將 `komari-theme.json` 與 `preview.png`/`perview.png` 複製到 `/web/public/defaultTheme`。
|
||||
```bash
|
||||
go build -o komari
|
||||
```
|
||||
3. 執行:
|
||||
```bash
|
||||
./komari server -l 0.0.0.0:25774
|
||||
```
|
||||
預設監聽 `25774` 連接埠,存取 `http://localhost:25774`。
|
||||
|
||||
## 前端開發指南
|
||||
|
||||
[Komari 主題開發指南 | Komari](https://komari-document.pages.dev/dev/theme.html)
|
||||
|
||||
[在 Crowdin 上翻譯 Komari](https://crowdin.com/project/komari/invite?h=cd051bf172c9a9f7f1360e87ffb521692507706)
|
||||
|
||||
## 客戶端 Agent 開發指南
|
||||
|
||||
[Komari Agent 資訊上報與事件處理文檔](https://komari-document.pages.dev/dev/agent.html)
|
||||
|
||||
## 貢獻
|
||||
|
||||
歡迎提交 Issue 或 Pull Request!
|
||||
|
||||
## 鳴謝
|
||||
|
||||
### 破碎工坊雲
|
||||
|
||||
[破碎工坊雲 - 專業雲計算服務平台,提供高效、穩定、安全的高防伺服器與CDN解決方案](https://www.crash.work/)
|
||||
|
||||
### DreamCloud
|
||||
|
||||
[DreamCloud - 極高性價比解鎖直連亞太高防](https://as211392.com/)
|
||||
|
||||
### 🚀 由 SharonNetworks 贊助
|
||||
|
||||
[](https://sharon.io)
|
||||
|
||||
SharonNetworks 為您的業務起飛保駕護航!
|
||||
|
||||
亞太資料中心提供頂級的中國優化網路接入 · 低延遲 & 高頻寬 & 提供 Tbps 級本地清洗高防服務,為您的業務保駕護航,為您的客戶提供極致體驗。加入社群 [Telegram 群組](https://t.me/SharonNetwork) 可參與公益募捐或群內抽獎免費使用。
|
||||
|
||||
### 開源社群
|
||||
|
||||
提交 PR、製作主題的各位開發者
|
||||
|
||||
—— 以及:感謝我自己能這麼閒
|
||||
|
||||
## Star History
|
||||
|
||||
[](https://www.star-history.com/#komari-monitor/komari&Date)
|
||||
@@ -1,148 +0,0 @@
|
||||
# Komari
|
||||
|
||||

|
||||
|
||||

|
||||
|
||||
Komari 是一款轻量级的自托管服务器监控工具,旨在提供简单、高效的服务器性能监控解决方案。它支持通过 Web 界面查看服务器状态,并通过轻量级 Agent 收集数据。
|
||||
|
||||
> [!WARNING]
|
||||
> Komari 是一款自托管的监控/控制程序,仅应部署在你拥有或已获得授权管理的系统上。请勿将 Komari 武器化,或在未获授权的情况下部署、访问、持久化、执行命令及从事其他滥用行为。关于现实中的滥用风险,可参考 Huntress 的分析:[Komari C2 agent abuse](https://www.huntress.com/blog/komari-c2-agent-abuse)。
|
||||
> 用户需要自行承担部署和使用 Komari 的责任。开发者不对未经授权或滥用行为及其后果承担责任。
|
||||
> 在 Windows 端开启远程控制后,客户端会在每次用户登录时通过 Windows 通知提醒用户 Komari 是一款远程控制软件。
|
||||
|
||||
[文档](https://komari-document.pages.dev/) | [文档(镜像站 By Geekertao)](https://www.komari.wiki) | [Telegram 群组](https://t.me/komari_monitor)
|
||||
|
||||
## 特性
|
||||
|
||||
- **轻量高效**:低资源占用,适合各种规模的服务器。
|
||||
- **自托管**:完全掌控数据隐私,部署简单。
|
||||
- **Web 界面**:直观的监控仪表盘,易于使用。
|
||||
|
||||
## 快速开始
|
||||
|
||||
### 0. 容器云一键部署
|
||||
|
||||
- 雨云云应用 - CNY 4.5/月
|
||||
|
||||
[](https://app.rainyun.com/apps/rca/store/6780/NzYxNzAz_)
|
||||
|
||||
- 1Panel 应用商店
|
||||
|
||||
已上架1Panel应用商店,应用商店-实用工具-Komari 即可安装
|
||||
|
||||
### 1. 使用一键安装脚本
|
||||
|
||||
适用于使用了 systemd 的发行版(Ubuntu、Debian...)。
|
||||
|
||||
```bash
|
||||
curl -fsSL https://raw.githubusercontent.com/komari-monitor/komari/main/install-komari.sh -o install-komari.sh
|
||||
chmod +x install-komari.sh
|
||||
sudo ./install-komari.sh
|
||||
```
|
||||
|
||||
### 2. Docker 部署
|
||||
|
||||
1. 创建数据目录:
|
||||
```bash
|
||||
mkdir -p ./data
|
||||
```
|
||||
2. 运行 Docker 容器:
|
||||
```bash
|
||||
docker run -d \
|
||||
-p 25774:25774 \
|
||||
-v $(pwd)/data:/app/data \
|
||||
--name komari \
|
||||
ghcr.io/komari-monitor/komari:latest
|
||||
```
|
||||
3. 查看默认账号和密码:
|
||||
```bash
|
||||
docker logs komari
|
||||
```
|
||||
4. 在浏览器中访问 `http://<your_server_ip>:25774`。
|
||||
|
||||
> [!NOTE]
|
||||
> 你也可以通过环境变量 `ADMIN_USERNAME` 和 `ADMIN_PASSWORD` 自定义初始用户名和密码。
|
||||
|
||||
### 3. 二进制文件部署
|
||||
|
||||
1. 访问 Komari 的 [GitHub Release 页面](https://github.com/komari-monitor/komari/releases) 下载适用于你操作系统的最新二进制文件。
|
||||
2. 运行 Komari:
|
||||
```bash
|
||||
./komari server -l 0.0.0.0:25774
|
||||
```
|
||||
3. 在浏览器中访问 `http://<your_server_ip>:25774`,默认监听 `25774` 端口。
|
||||
4. 默认账号和密码可在启动日志中查看,或通过环境变量 `ADMIN_USERNAME` 和 `ADMIN_PASSWORD` 设置。
|
||||
|
||||
> [!NOTE]
|
||||
> 确保二进制文件具有可执行权限(`chmod +x komari`)。数据将保存在运行目录下的 `data` 文件夹中。
|
||||
|
||||
### 手工构建
|
||||
|
||||
#### 依赖
|
||||
|
||||
- Go 1.18+ 和 Node.js 20+(手工构建)
|
||||
|
||||
1. 构建前端静态文件:
|
||||
```bash
|
||||
git clone https://github.com/komari-monitor/komari-web
|
||||
cd komari-web
|
||||
npm install
|
||||
npm run build
|
||||
```
|
||||
2. 构建后端:
|
||||
```bash
|
||||
git clone https://github.com/komari-monitor/komari
|
||||
cd komari
|
||||
```
|
||||
将步骤1中生成的静态文件复制到 `komari` 项目根目录下的 `/web/public/defaultTheme/dist` 文件夹,并将 `komari-theme.json` 与 `preview.png`/`perview.png` 复制到 `/web/public/defaultTheme`。
|
||||
```bash
|
||||
go build -o komari
|
||||
```
|
||||
3. 运行:
|
||||
```bash
|
||||
./komari server -l 0.0.0.0:25774
|
||||
```
|
||||
默认监听 `25774` 端口,访问 `http://localhost:25774`。
|
||||
|
||||
## 前端开发指南
|
||||
|
||||
[Komari 主题开发指南 | Komari](https://komari-document.pages.dev/dev/theme.html)
|
||||
|
||||
[在 Crowdin 上翻译 Komari](https://crowdin.com/project/komari/invite?h=cd051bf172c9a9f7f1360e87ffb521692507706)
|
||||
|
||||
## 客户端 Agent 开发指南
|
||||
|
||||
[Komari Agent 信息上报与事件处理文档](https://komari-document.pages.dev/dev/agent.html)
|
||||
|
||||
## 贡献
|
||||
|
||||
欢迎提交 Issue 或 Pull Request!
|
||||
|
||||
## 鸣谢
|
||||
|
||||
### 破碎工坊云
|
||||
|
||||
[破碎工坊云 - 专业云计算服务平台,提供高效、稳定、安全的高防服务器与CDN解决方案](https://www.crash.work/)
|
||||
|
||||
### DreamCloud
|
||||
|
||||
[DreamCloud - 极高性价比解锁直连亚太高防](https://as211392.com/)
|
||||
|
||||
### 🚀 由 SharonNetworks 赞助
|
||||
|
||||
[](https://sharon.io)
|
||||
|
||||
SharonNetworks 为您的业务起飞保驾护航!
|
||||
|
||||
亚太数据中心提供顶级的中国优化网络接入 · 低延时 & 高带宽 & 提供 Tbps 级本地清洗高防服务,为您的业务保驾护航,为您的客户提供极致体验。加入社区 [Telegram 群组](https://t.me/SharonNetwork) 可参与公益募捐或群内抽奖免费使用。
|
||||
|
||||
### 开源社区
|
||||
|
||||
提交 PR、制作主题的各位开发者
|
||||
|
||||
—— 以及:感谢我自己能这么闲
|
||||
|
||||
## Star History
|
||||
|
||||
[](https://www.star-history.com/#komari-monitor/komari&Date)
|
||||
@@ -3,7 +3,6 @@ module github.com/komari-monitor/komari
|
||||
go 1.25.0
|
||||
|
||||
require (
|
||||
github.com/coreos/go-oidc/v3 v3.15.0
|
||||
github.com/dop251/goja v0.0.0-20251008123653-cf18d89f3cf6
|
||||
github.com/dop251/goja_nodejs v0.0.0-20251015164255-5e94316bedaf
|
||||
github.com/gin-gonic/gin v1.10.0
|
||||
@@ -18,12 +17,12 @@ require (
|
||||
github.com/spf13/cobra v1.9.1
|
||||
github.com/stretchr/testify v1.11.1
|
||||
golang.org/x/image v0.35.0
|
||||
google.golang.org/grpc v1.75.0
|
||||
google.golang.org/protobuf v1.36.6
|
||||
gorm.io/driver/sqlite v1.5.7
|
||||
gorm.io/gorm v1.26.1
|
||||
)
|
||||
|
||||
require golang.org/x/sys v0.33.0 // indirect
|
||||
|
||||
// Misuse of ServerConfig.PublicKeyCallback may cause authorization bypass in golang.org/x/crypto #1
|
||||
// golang.org/x/crypto Vulnerable to Denial of Service (DoS) via Slow or Incomplete Key Exchange #3
|
||||
require golang.org/x/crypto v0.39.0 // indirect
|
||||
@@ -42,12 +41,12 @@ require (
|
||||
github.com/dlclark/regexp2 v1.11.4 // indirect
|
||||
github.com/gabriel-vasile/mimetype v1.4.8 // indirect
|
||||
github.com/gin-contrib/sse v1.0.0 // indirect
|
||||
github.com/go-jose/go-jose/v4 v4.1.1 // indirect
|
||||
github.com/go-playground/locales v0.14.1 // indirect
|
||||
github.com/go-playground/universal-translator v0.18.1 // indirect
|
||||
github.com/go-playground/validator/v10 v10.26.0 // indirect
|
||||
github.com/go-sourcemap/sourcemap v2.1.4+incompatible // indirect
|
||||
github.com/goccy/go-json v0.10.5 // indirect
|
||||
github.com/google/go-cmp v0.7.0 // indirect
|
||||
github.com/google/pprof v0.0.0-20240727154555-813a5fbdbec8 // indirect
|
||||
github.com/inconshreveable/mousetrap v1.1.0 // indirect
|
||||
github.com/jackc/pgpassfile v1.0.0 // indirect
|
||||
@@ -69,10 +68,8 @@ require (
|
||||
github.com/twitchyliquid64/golang-asm v0.15.1 // indirect
|
||||
github.com/ugorji/go/codec v1.2.12 // indirect
|
||||
golang.org/x/arch v0.15.0 // indirect
|
||||
golang.org/x/oauth2 v0.30.0 // indirect
|
||||
golang.org/x/sync v0.19.0 // indirect
|
||||
golang.org/x/sys v0.33.0 // indirect
|
||||
golang.org/x/sync v0.19.0
|
||||
golang.org/x/text v0.33.0 // indirect
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20250707201910-8d1bb00bc6a7 // indirect
|
||||
google.golang.org/protobuf v1.36.6 // indirect
|
||||
gopkg.in/yaml.v3 v3.0.1 // indirect
|
||||
)
|
||||
@@ -12,8 +12,6 @@ github.com/bytedance/sonic/loader v0.2.4/go.mod h1:N8A3vUdtUebEY2/VQC0MyhYeKUFos
|
||||
github.com/cloudwego/base64x v0.1.5 h1:XPciSp1xaq2VCSt6lF0phncD4koWyULpl5bUxbfCyP4=
|
||||
github.com/cloudwego/base64x v0.1.5/go.mod h1:0zlkT4Wn5C6NdauXdJRhSKRlJvmclQ1hhJgA0rcu/8w=
|
||||
github.com/cloudwego/iasm v0.2.0/go.mod h1:8rXZaNYT2n95jn+zTI1sDr+IgcD2GVs0nlbbQPiEFhY=
|
||||
github.com/coreos/go-oidc/v3 v3.15.0 h1:R6Oz8Z4bqWR7VFQ+sPSvZPQv4x8M+sJkDO5ojgwlyAg=
|
||||
github.com/coreos/go-oidc/v3 v3.15.0/go.mod h1:HaZ3szPaZ0e4r6ebqvsLWlk2Tn+aejfmrfah6hnSYEU=
|
||||
github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g=
|
||||
github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
|
||||
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
@@ -31,12 +29,6 @@ github.com/gin-contrib/sse v1.0.0 h1:y3bT1mUWUxDpW4JLQg/HnTqV4rozuW4tC9eFKTxYI9E
|
||||
github.com/gin-contrib/sse v1.0.0/go.mod h1:zNuFdwarAygJBht0NTKiSi3jRf6RbqeILZ9Sp6Slhe0=
|
||||
github.com/gin-gonic/gin v1.10.0 h1:nTuyha1TYqgedzytsKYqna+DfLos46nTv2ygFy86HFU=
|
||||
github.com/gin-gonic/gin v1.10.0/go.mod h1:4PMNQiOhvDRa013RKVbsiNwoyezlm2rm0uX/T7kzp5Y=
|
||||
github.com/go-jose/go-jose/v4 v4.1.1 h1:JYhSgy4mXXzAdF3nUx3ygx347LRXJRrpgyU3adRmkAI=
|
||||
github.com/go-jose/go-jose/v4 v4.1.1/go.mod h1:BdsZGqgdO3b6tTc6LSE56wcDbMMLuPsw5d4ZD5f94kA=
|
||||
github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI=
|
||||
github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
|
||||
github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag=
|
||||
github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE=
|
||||
github.com/go-playground/assert/v2 v2.2.0 h1:JvknZsQTYeFEAhQwI4qEt9cyV5ONwRHC+lYKSsYSR8s=
|
||||
github.com/go-playground/assert/v2 v2.2.0/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4=
|
||||
github.com/go-playground/locales v0.14.1 h1:EWaQ/wswjilfKLTECiXz7Rh+3BjFhfDFKv/oXslEjJA=
|
||||
@@ -51,8 +43,6 @@ github.com/go-sql-driver/mysql v1.10.0 h1:Q+1LV8DkHJvSYAdR83XzuhDaTykuDx0l6fkXxo
|
||||
github.com/go-sql-driver/mysql v1.10.0/go.mod h1:M+cqaI7+xxXGG9swrdeUIoPG3Y3KCkF0pZej+SK+nWk=
|
||||
github.com/goccy/go-json v0.10.5 h1:Fq85nIqj+gXn/S5ahsiTlK3TmC85qgirsdTP/+DeaC4=
|
||||
github.com/goccy/go-json v0.10.5/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M=
|
||||
github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek=
|
||||
github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps=
|
||||
github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=
|
||||
github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU=
|
||||
github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
|
||||
@@ -135,18 +125,6 @@ github.com/twitchyliquid64/golang-asm v0.15.1 h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS
|
||||
github.com/twitchyliquid64/golang-asm v0.15.1/go.mod h1:a1lVb/DtPvCB8fslRZhAngC2+aY1QWCk3Cedj/Gdt08=
|
||||
github.com/ugorji/go/codec v1.2.12 h1:9LC83zGrHhuUA9l16C9AHXAqEV/2wBQ4nkvumAE65EE=
|
||||
github.com/ugorji/go/codec v1.2.12/go.mod h1:UNopzCgEMSXjBc6AOMqYvWC1ktqTAfzJZUZgYf6w6lg=
|
||||
go.opentelemetry.io/auto/sdk v1.1.0 h1:cH53jehLUN6UFLY71z+NDOiNJqDdPRaXzTel0sJySYA=
|
||||
go.opentelemetry.io/auto/sdk v1.1.0/go.mod h1:3wSPjt5PWp2RhlCcmmOial7AvC4DQqZb7a7wCow3W8A=
|
||||
go.opentelemetry.io/otel v1.37.0 h1:9zhNfelUvx0KBfu/gb+ZgeAfAgtWrfHJZcAqFC228wQ=
|
||||
go.opentelemetry.io/otel v1.37.0/go.mod h1:ehE/umFRLnuLa/vSccNq9oS1ErUlkkK71gMcN34UG8I=
|
||||
go.opentelemetry.io/otel/metric v1.37.0 h1:mvwbQS5m0tbmqML4NqK+e3aDiO02vsf/WgbsdpcPoZE=
|
||||
go.opentelemetry.io/otel/metric v1.37.0/go.mod h1:04wGrZurHYKOc+RKeye86GwKiTb9FKm1WHtO+4EVr2E=
|
||||
go.opentelemetry.io/otel/sdk v1.37.0 h1:ItB0QUqnjesGRvNcmAcU0LyvkVyGJ2xftD29bWdDvKI=
|
||||
go.opentelemetry.io/otel/sdk v1.37.0/go.mod h1:VredYzxUvuo2q3WRcDnKDjbdvmO0sCzOvVAiY+yUkAg=
|
||||
go.opentelemetry.io/otel/sdk/metric v1.37.0 h1:90lI228XrB9jCMuSdA0673aubgRobVZFhbjxHHspCPc=
|
||||
go.opentelemetry.io/otel/sdk/metric v1.37.0/go.mod h1:cNen4ZWfiD37l5NhS+Keb5RXVWZWpRE+9WyVCpbo5ps=
|
||||
go.opentelemetry.io/otel/trace v1.37.0 h1:HLdcFNbRQBE2imdSEgm/kwqmQj1Or1l/7bW6mxVK7z4=
|
||||
go.opentelemetry.io/otel/trace v1.37.0/go.mod h1:TlgrlQ+PtQO5XFerSPUYG0JSgGyryXewPGyayAWSBS0=
|
||||
golang.org/x/arch v0.15.0 h1:QtOrQd0bTUnhNVNndMpLHNWrDmYzZ2KDqSrEymqInZw=
|
||||
golang.org/x/arch v0.15.0/go.mod h1:JmwW7aLIoRUKgaTzhkiEFxvcEiQGyOg9BMonBJUS7EE=
|
||||
golang.org/x/crypto v0.39.0 h1:SHs+kF4LP+f+p14esP5jAoDpHU8Gu/v9lFRK6IT5imM=
|
||||
@@ -155,8 +133,6 @@ golang.org/x/image v0.35.0 h1:LKjiHdgMtO8z7Fh18nGY6KDcoEtVfsgLDPeLyguqb7I=
|
||||
golang.org/x/image v0.35.0/go.mod h1:MwPLTVgvxSASsxdLzKrl8BRFuyqMyGhLwmC+TO1Sybk=
|
||||
golang.org/x/net v0.41.0 h1:vBTly1HeNPEn3wtREYfy4GZ/NECgw2Cnl+nK6Nz3uvw=
|
||||
golang.org/x/net v0.41.0/go.mod h1:B/K4NNqkfmg07DQYrbwvSluqCJOOXwUjeb/5lOisjbA=
|
||||
golang.org/x/oauth2 v0.30.0 h1:dnDm7JmhM45NNpd8FDDeLhK6FwqbOf4MLCM9zb1BOHI=
|
||||
golang.org/x/oauth2 v0.30.0/go.mod h1:B++QgG3ZKulg6sRPGD/mqlHQs5rB3Ml9erfeDY7xKlU=
|
||||
golang.org/x/sync v0.19.0 h1:vV+1eWNmZ5geRlYjzm2adRgW2/mcpevXNg50YZtPCE4=
|
||||
golang.org/x/sync v0.19.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI=
|
||||
golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
@@ -164,12 +140,6 @@ golang.org/x/sys v0.33.0 h1:q3i8TbbEz+JRD9ywIRlyRAQbM0qF7hu24q3teo2hbuw=
|
||||
golang.org/x/sys v0.33.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k=
|
||||
golang.org/x/text v0.33.0 h1:B3njUFyqtHDUI5jMn1YIr5B0IE2U0qck04r6d4KPAxE=
|
||||
golang.org/x/text v0.33.0/go.mod h1:LuMebE6+rBincTi9+xWTY8TztLzKHc/9C1uBCG27+q8=
|
||||
gonum.org/v1/gonum v0.16.0 h1:5+ul4Swaf3ESvrOnidPp4GZbzf0mxVQpDCYUQE7OJfk=
|
||||
gonum.org/v1/gonum v0.16.0/go.mod h1:fef3am4MQ93R2HHpKnLk4/Tbh/s0+wqD5nfa6Pnwy4E=
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20250707201910-8d1bb00bc6a7 h1:pFyd6EwwL2TqFf8emdthzeX+gZE1ElRq3iM8pui4KBY=
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20250707201910-8d1bb00bc6a7/go.mod h1:qQ0YXyHHx3XkvlzUtpXDkS29lDSafHMZBAZDc03LQ3A=
|
||||
google.golang.org/grpc v1.75.0 h1:+TW+dqTd2Biwe6KKfhE5JpiYIBWq865PhKGSXiivqt4=
|
||||
google.golang.org/grpc v1.75.0/go.mod h1:JtPAzKiq4v1xcAB2hydNlWI2RnF85XXcV0mhKXr2ecQ=
|
||||
google.golang.org/protobuf v1.36.6 h1:z1NpPI8ku2WgiWnf+t9wTPsn6eP1L7ksHUlkfLvd9xY=
|
||||
google.golang.org/protobuf v1.36.6/go.mod h1:jduwjTPXsFjZGTmRluh+L6NjiWu7pchiJ2/5YcXBHnY=
|
||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
|
||||
+6
-12
@@ -226,6 +226,9 @@ detect_arch() {
|
||||
riscv64)
|
||||
echo "riscv64"
|
||||
;;
|
||||
loongarch64|loong64)
|
||||
echo "loong64"
|
||||
;;
|
||||
*)
|
||||
log_error "不支持的架构: $arch"
|
||||
exit 1
|
||||
@@ -365,13 +368,7 @@ install_binary() {
|
||||
if systemctl is-active --quiet ${SERVICE_NAME}.service; then
|
||||
log_success "Komari 服务启动成功"
|
||||
|
||||
log_step "正在获取初始密码..."
|
||||
sleep 5
|
||||
local password=$(journalctl -u ${SERVICE_NAME} --since "1 minute ago" | grep "admin account created." | tail -n 1 | sed -e 's/.*admin account created.//')
|
||||
if [ -z "$password" ]; then
|
||||
log_error "未能获取初始密码,请检查日志"
|
||||
fi
|
||||
show_access_info "$password" "$LISTEN_PORT"
|
||||
show_access_info "$LISTEN_PORT"
|
||||
else
|
||||
ui_msgbox "错误" "Komari 服务启动失败。\n\n查看日志: journalctl -u ${SERVICE_NAME} -f"
|
||||
return 1
|
||||
@@ -405,16 +402,13 @@ EOF
|
||||
|
||||
# Show access information
|
||||
show_access_info() {
|
||||
local password=$1
|
||||
local port=${2:-$DEFAULT_PORT}
|
||||
local port=${1:-$DEFAULT_PORT}
|
||||
local ip=$(hostname -I | awk '{print $1}')
|
||||
|
||||
local content="安装完成!\n\n"
|
||||
content+="访问信息:\n"
|
||||
content+=" URL: http://${ip}:${port}\n"
|
||||
if [ -n "$password" ]; then
|
||||
content+=" 初始登录信息(仅显示一次): ${password}\n"
|
||||
fi
|
||||
content+="\n首次使用请访问上述地址,按安装向导创建管理员账号。\n"
|
||||
content+="\n服务管理命令:\n"
|
||||
content+=" 状态: systemctl status $SERVICE_NAME\n"
|
||||
content+=" 启动: systemctl start $SERVICE_NAME\n"
|
||||
|
||||
@@ -3,11 +3,11 @@ package config
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"reflect"
|
||||
"strings"
|
||||
"sync"
|
||||
|
||||
logger "github.com/komari-monitor/komari/utils/log"
|
||||
"gorm.io/gorm"
|
||||
"gorm.io/gorm/clause"
|
||||
)
|
||||
@@ -31,26 +31,6 @@ var (
|
||||
}
|
||||
)
|
||||
|
||||
// Get 获取原始值 (反序列化为 interface{})
|
||||
func Get(key string, defaul ...any) (any, error) {
|
||||
var item ConfigItem
|
||||
err := db.First(&item, "key = ?", key).Error
|
||||
if err != nil {
|
||||
if len(defaul) > 0 {
|
||||
v := defaul[0]
|
||||
err = Set(key, v)
|
||||
return v, err
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
|
||||
var val any
|
||||
if err := json.Unmarshal([]byte(item.Value), &val); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return val, nil
|
||||
}
|
||||
|
||||
// GetAs 获取并转换为指定类型 (泛型),支持数值类型自动转换
|
||||
func GetAs[T any](key string, defaul ...any) (T, error) {
|
||||
var t T
|
||||
@@ -125,7 +105,7 @@ func GetMany(keys map[string]any) (map[string]any, error) {
|
||||
// 序列化后加入待写入列表
|
||||
jsonBytes, err := json.Marshal(def)
|
||||
if err != nil {
|
||||
slog.Warn("marshal default value failed", "key", k, "error", err)
|
||||
logger.Warn("config", "marshal default value failed", "key", k, "error", err)
|
||||
continue
|
||||
}
|
||||
toInsert = append(toInsert, ConfigItem{
|
||||
@@ -142,7 +122,7 @@ func GetMany(keys map[string]any) (map[string]any, error) {
|
||||
Columns: []clause.Column{{Name: "key"}},
|
||||
DoUpdates: clause.AssignmentColumns([]string{"value"}),
|
||||
}).Create(&toInsert).Error; err != nil {
|
||||
slog.Warn("batch insert default config failed", "error", err)
|
||||
logger.Warn("config", "batch insert default config failed", "error", err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -180,9 +160,8 @@ func GetManyAs[T any]() (*T, error) {
|
||||
}
|
||||
|
||||
defaultTag := field.Tag.Get("default")
|
||||
hasDefault := defaultTag != "" || field.Tag.Get("default") == ""
|
||||
// 检查是否显式定义了 default tag (即使值为空)
|
||||
_, hasDefault = field.Tag.Lookup("default")
|
||||
_, hasDefault := field.Tag.Lookup("default")
|
||||
|
||||
fields = append(fields, fieldInfo{
|
||||
index: i,
|
||||
@@ -220,18 +199,18 @@ func GetManyAs[T any]() (*T, error) {
|
||||
if dbValue, found := foundItems[fi.key]; found {
|
||||
// 数据库中存在,使用数据库值
|
||||
if err := unmarshalToField(dbValue, fieldVal); err != nil {
|
||||
slog.Warn("unmarshal config failed", "key", fi.key, "error", err)
|
||||
logger.Warn("config", "unmarshal config failed", "key", fi.key, "error", err)
|
||||
}
|
||||
} else if fi.hasDefault {
|
||||
// 数据库中不存在,但有 default tag,解析默认值并写入数据库
|
||||
if err := parseDefaultToField(fi.defaultVal, fieldVal); err != nil {
|
||||
slog.Warn("parse default value failed", "key", fi.key, "error", err)
|
||||
logger.Warn("config", "parse default value failed", "key", fi.key, "error", err)
|
||||
continue
|
||||
}
|
||||
// 序列化后写入数据库
|
||||
jsonBytes, err := json.Marshal(fieldVal.Interface())
|
||||
if err != nil {
|
||||
slog.Warn("marshal default value failed", "key", fi.key, "error", err)
|
||||
logger.Warn("config", "marshal default value failed", "key", fi.key, "error", err)
|
||||
continue
|
||||
}
|
||||
toInsert = append(toInsert, ConfigItem{
|
||||
@@ -248,7 +227,7 @@ func GetManyAs[T any]() (*T, error) {
|
||||
Columns: []clause.Column{{Name: "key"}},
|
||||
DoUpdates: clause.AssignmentColumns([]string{"value"}),
|
||||
}).Create(&toInsert).Error; err != nil {
|
||||
slog.Warn("batch insert default config failed", "error", err)
|
||||
logger.Warn("config", "batch insert default config failed", "error", err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -427,76 +406,6 @@ func Set(key string, value any) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// SetMany 将结构体保存为多个配置项
|
||||
func SetManyAs[T any](config T) error {
|
||||
val := reflect.ValueOf(config)
|
||||
if val.Kind() == reflect.Ptr {
|
||||
val = val.Elem()
|
||||
}
|
||||
|
||||
typ := val.Type()
|
||||
var items []ConfigItem
|
||||
|
||||
for i := 0; i < val.NumField(); i++ {
|
||||
fieldType := typ.Field(i)
|
||||
tag := fieldType.Tag.Get("json")
|
||||
|
||||
if tag == "" || tag == "-" {
|
||||
continue
|
||||
}
|
||||
|
||||
fieldValue := val.Field(i).Interface()
|
||||
|
||||
bytes, err := json.Marshal(fieldValue)
|
||||
if err != nil {
|
||||
return fmt.Errorf("marshal field %s failed: %w", fieldType.Name, err)
|
||||
}
|
||||
|
||||
items = append(items, ConfigItem{
|
||||
Key: tag,
|
||||
Value: string(bytes),
|
||||
})
|
||||
}
|
||||
|
||||
if len(items) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
keys := make([]string, 0, len(items))
|
||||
newVal := make(map[string]any, len(items))
|
||||
for _, it := range items {
|
||||
keys = append(keys, it.Key)
|
||||
var parsed any
|
||||
if err := json.Unmarshal([]byte(it.Value), &parsed); err == nil {
|
||||
newVal[it.Key] = parsed
|
||||
}
|
||||
}
|
||||
|
||||
oldVal := map[string]any{}
|
||||
if len(keys) > 0 {
|
||||
var oldItems []ConfigItem
|
||||
if err := db.Where("key IN ?", keys).Find(&oldItems).Error; err == nil {
|
||||
for _, oi := range oldItems {
|
||||
var parsed any
|
||||
if err := json.Unmarshal([]byte(oi.Value), &parsed); err == nil {
|
||||
oldVal[oi.Key] = parsed
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
err := db.Clauses(clause.OnConflict{
|
||||
Columns: []clause.Column{{Name: "key"}},
|
||||
DoUpdates: clause.AssignmentColumns([]string{"value"}),
|
||||
}).Create(&items).Error
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
publishEvent(oldVal, newVal)
|
||||
return nil
|
||||
}
|
||||
|
||||
func SetMany(cst map[string]any) error {
|
||||
var items []ConfigItem
|
||||
for k, v := range cst {
|
||||
@@ -16,22 +16,20 @@ type Settings struct {
|
||||
AutoDiscoveryKey string `json:"auto_discovery_key" default:""` // 自动发现密钥
|
||||
ScriptDomain string `json:"script_domain" default:""` // 自定义脚本域名
|
||||
SendIpAddrToGuest bool `json:"send_ip_addr_to_guest" default:"false"` // 是否向访客页面发送 IP 地址,默认 false
|
||||
VisitorAuditEnabled bool `json:"visitor_audit_enabled" default:"false"` // 是否允许公开访客事件写入审计日志,默认 false
|
||||
EulaAccepted bool `json:"eula_accepted" default:"false"`
|
||||
BaseScriptsURLKey string `json:"base_scripts_url" default:""`
|
||||
// GeoIP 配置
|
||||
GeoIpEnabled bool `json:"geo_ip_enabled" default:"true"`
|
||||
GeoIpProvider string `json:"geo_ip_provider" default:"ipinfo"` // empty, mmdb, ip-api, geojs
|
||||
// Nezha 兼容(Agent gRPC)
|
||||
NezhaCompatEnabled bool `json:"nezha_compat_enabled" default:"false"`
|
||||
NezhaCompatListen string `json:"nezha_compat_listen" default:""` // 例如 0.0.0.0:5555
|
||||
// OAuth 配置
|
||||
OAuthEnabled bool `json:"o_auth_enabled" default:"false"`
|
||||
OAuthProvider string `json:"o_auth_provider" default:"github"`
|
||||
DisablePasswordLogin bool `json:"disable_password_login" default:"false"`
|
||||
CloudflareTunnelToken string `json:"cloudflare_tunnel_token" default:""`
|
||||
OAuthEnabled bool `json:"o_auth_enabled" default:"false"`
|
||||
OAuthProvider string `json:"o_auth_provider" default:"github"`
|
||||
DisablePasswordLogin bool `json:"disable_password_login" default:"false"`
|
||||
// 自定义美化
|
||||
CustomHead string `json:"custom_head" default:""`
|
||||
CustomBody string `json:"custom_body" default:""`
|
||||
|
||||
// 通知
|
||||
NotificationEnabled bool `json:"notification_enabled" default:"true"` // 通知总开关
|
||||
NotificationMethod string `json:"notification_method" default:"none"`
|
||||
@@ -40,38 +38,33 @@ type Settings struct {
|
||||
ExpireNotificationLeadDays int `json:"expire_notification_lead_days" default:"7"` // 过期前多少天通知,默认7天
|
||||
LoginNotification bool `json:"login_notification" default:"true"` // 登录通知
|
||||
TrafficLimitPercentage float64 `json:"traffic_limit_percentage" default:"80.00"` // 流量限制百分比,默认80.00%
|
||||
// Record
|
||||
RecordEnabled bool `json:"record_enabled" default:"true"` // 是否启用记录功能
|
||||
RecordPreserveTime int `json:"record_preserve_time" default:"720"` // 记录保留时间,单位小时,默认30天
|
||||
PingRecordPreserveTime int `json:"ping_record_preserve_time" default:"24"` // Ping 记录保留时间,单位小时,默认1天
|
||||
UpdatedAt time.Time
|
||||
UpdatedAt time.Time
|
||||
}
|
||||
|
||||
const (
|
||||
SitenameKey = "sitename"
|
||||
DescriptionKey = "description"
|
||||
CorsOriginCheckEnabledKey = "cors_origin_check_enabled"
|
||||
CorsAllowedOriginsKey = "cors_allowed_origins"
|
||||
WsOriginCheckEnabledKey = "ws_origin_check_enabled"
|
||||
WsAllowedOriginsKey = "ws_allowed_origins"
|
||||
ThemeKey = "theme"
|
||||
PrivateSiteKey = "private_site"
|
||||
ApiKeyKey = "api_key"
|
||||
AutoDiscoveryKeyKey = "auto_discovery_key"
|
||||
ScriptDomainKey = "script_domain"
|
||||
SendIpAddrToGuestKey = "send_ip_addr_to_guest"
|
||||
EulaAcceptedKey = "eula_accepted"
|
||||
BaseScriptsURLKey = "base_scripts_url"
|
||||
GeoIpEnabledKey = "geo_ip_enabled"
|
||||
GeoIpProviderKey = "geo_ip_provider"
|
||||
NezhaCompatEnabledKey = "nezha_compat_enabled"
|
||||
NezhaCompatListenKey = "nezha_compat_listen"
|
||||
OAuthEnabledKey = "o_auth_enabled"
|
||||
OAuthProviderKey = "o_auth_provider"
|
||||
DisablePasswordLoginKey = "disable_password_login"
|
||||
CloudflareTunnelTokenKey = "cloudflare_tunnel_token"
|
||||
CustomHeadKey = "custom_head"
|
||||
CustomBodyKey = "custom_body"
|
||||
SitenameKey = "sitename"
|
||||
DescriptionKey = "description"
|
||||
CorsOriginCheckEnabledKey = "cors_origin_check_enabled"
|
||||
CorsAllowedOriginsKey = "cors_allowed_origins"
|
||||
WsOriginCheckEnabledKey = "ws_origin_check_enabled"
|
||||
WsAllowedOriginsKey = "ws_allowed_origins"
|
||||
ThemeKey = "theme"
|
||||
PrivateSiteKey = "private_site"
|
||||
ApiKeyKey = "api_key"
|
||||
AutoDiscoveryKeyKey = "auto_discovery_key"
|
||||
ScriptDomainKey = "script_domain"
|
||||
SendIpAddrToGuestKey = "send_ip_addr_to_guest"
|
||||
VisitorAuditEnabledKey = "visitor_audit_enabled"
|
||||
EulaAcceptedKey = "eula_accepted"
|
||||
BaseScriptsURLKey = "base_scripts_url"
|
||||
GeoIpEnabledKey = "geo_ip_enabled"
|
||||
GeoIpProviderKey = "geo_ip_provider"
|
||||
OAuthEnabledKey = "o_auth_enabled"
|
||||
OAuthProviderKey = "o_auth_provider"
|
||||
DisablePasswordLoginKey = "disable_password_login"
|
||||
CustomHeadKey = "custom_head"
|
||||
CustomBodyKey = "custom_body"
|
||||
|
||||
NotificationEnabledKey = "notification_enabled"
|
||||
NotificationMethodKey = "notification_method"
|
||||
NotificationTemplateKey = "notification_template"
|
||||
@@ -79,9 +72,7 @@ const (
|
||||
ExpireNotificationLeadDaysKey = "expire_notification_lead_days"
|
||||
LoginNotificationKey = "login_notification"
|
||||
TrafficLimitPercentageKey = "traffic_limit_percentage"
|
||||
RecordEnabledKey = "record_enabled"
|
||||
RecordPreserveTimeKey = "record_preserve_time"
|
||||
PingRecordPreserveTimeKey = "ping_record_preserve_time"
|
||||
UpdatedAtKey = "updated_at"
|
||||
XtermjsSettingsKey = "xtermjs_settings"
|
||||
ThemeMarketSourcesKey = "theme_market_sources"
|
||||
)
|
||||
@@ -0,0 +1,24 @@
|
||||
// Package lifecycle carries process-level requests without coupling web RPC
|
||||
// handlers to the server runtime.
|
||||
package lifecycle
|
||||
|
||||
// RestartReason identifies why the current process must restart.
|
||||
type RestartReason string
|
||||
|
||||
const RestartForMetricStoreStructureUpgrade RestartReason = "metric-store-structure-upgrade"
|
||||
|
||||
var restartRequests = make(chan RestartReason, 1)
|
||||
|
||||
// RequestRestart asks the running server to shut down cleanly and exit. A
|
||||
// pending request is sufficient because the process will be replaced.
|
||||
func RequestRestart(reason RestartReason) {
|
||||
select {
|
||||
case restartRequests <- reason:
|
||||
default:
|
||||
}
|
||||
}
|
||||
|
||||
// RestartRequests returns process restart requests for the server runtime.
|
||||
func RestartRequests() <-chan RestartReason {
|
||||
return restartRequests
|
||||
}
|
||||
@@ -0,0 +1,30 @@
|
||||
package migrations
|
||||
|
||||
import "time"
|
||||
|
||||
// ClientInfo is the legacy table shape for migrating pre-Client model data.
|
||||
type ClientInfo struct {
|
||||
UUID string `json:"uuid,omitempty" gorm:"type:varchar(36);primaryKey;foreignKey:ClientUUID;references:UUID;constraint:OnDelete:CASCADE"`
|
||||
Name string `json:"name" gorm:"type:varchar(100);not null"`
|
||||
CpuName string `json:"cpu_name" gorm:"type:varchar(100)"`
|
||||
Virtualization string `json:"virtualization" gorm:"type:varchar(50)"`
|
||||
Arch string `json:"arch" gorm:"type:varchar(50)"`
|
||||
CpuCores int `json:"cpu_cores" gorm:"type:int"`
|
||||
OS string `json:"os" gorm:"type:varchar(100)"`
|
||||
GpuName string `json:"gpu_name" gorm:"type:varchar(100)"`
|
||||
IPv4 string `json:"ipv4,omitempty" gorm:"type:varchar(100)"`
|
||||
IPv6 string `json:"ipv6,omitempty" gorm:"type:varchar(100)"`
|
||||
Region string `json:"region" gorm:"type:varchar(100)"`
|
||||
Remark string `json:"remark,omitempty" gorm:"type:longtext"`
|
||||
PublicRemark string `json:"public_remark,omitempty" gorm:"type:longtext"`
|
||||
MemTotal int64 `json:"mem_total" gorm:"type:bigint"`
|
||||
SwapTotal int64 `json:"swap_total" gorm:"type:bigint"`
|
||||
DiskTotal int64 `json:"disk_total" gorm:"type:bigint"`
|
||||
Version string `json:"version,omitempty" gorm:"type:varchar(100)"`
|
||||
Weight int `json:"weight" gorm:"type:int"`
|
||||
Price float64 `json:"price"`
|
||||
BillingCycle int `json:"billing_cycle"`
|
||||
ExpiredAt *time.Time `json:"expired_at" gorm:"type:timestamp"`
|
||||
CreatedAt time.Time
|
||||
UpdatedAt time.Time
|
||||
}
|
||||
@@ -0,0 +1,677 @@
|
||||
package migrations
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
logger "github.com/komari-monitor/komari/utils/log"
|
||||
"math"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/komari-monitor/komari/database/metricstore"
|
||||
"github.com/komari-monitor/komari/database/models"
|
||||
appconfig "github.com/komari-monitor/komari/internal/config"
|
||||
"github.com/komari-monitor/komari/pkg/metric"
|
||||
"gorm.io/gorm"
|
||||
)
|
||||
|
||||
const (
|
||||
legacyMonitoringMigrationDoneKey = "migration_legacy_monitoring_to_metric_store_done"
|
||||
legacyMonitoringBatchSize = 500
|
||||
legacyMonitoringInterval = time.Hour
|
||||
)
|
||||
|
||||
var legacyMonitoringTables = []string{"records", "records_long_term", "gpu_records", "ping_records"}
|
||||
|
||||
type LegacyMonitoringStats struct {
|
||||
Records int64
|
||||
GPU int64
|
||||
Ping int64
|
||||
}
|
||||
|
||||
// LegacyMonitoringSummary describes the full legacy source data shown by the
|
||||
// pre-start migration guide. Every source row participates in the migration;
|
||||
// EstimatedPoints is the number of hourly P95 points expected after grouping.
|
||||
type LegacyMonitoringSummary struct {
|
||||
LoadRows int64 `json:"load_rows"`
|
||||
GPURows int64 `json:"gpu_rows"`
|
||||
LatencyRows int64 `json:"latency_rows"`
|
||||
MonitoringRows int64 `json:"monitoring_rows"`
|
||||
EstimatedPoints int64 `json:"estimated_points"`
|
||||
ServerCount int64 `json:"server_count"`
|
||||
RetentionDays int `json:"retention_days"`
|
||||
OldestAt *time.Time `json:"oldest_at,omitempty"`
|
||||
NewestAt *time.Time `json:"newest_at,omitempty"`
|
||||
}
|
||||
|
||||
type LegacyMonitoringProgress struct {
|
||||
Phase string `json:"phase"`
|
||||
Table string `json:"table,omitempty"`
|
||||
SourceRowsDone int64 `json:"source_rows_done"`
|
||||
SourceRowsTotal int64 `json:"source_rows_total"`
|
||||
WrittenPoints int64 `json:"written_points"`
|
||||
}
|
||||
|
||||
type legacyMonitoringBounds struct {
|
||||
Oldest string `gorm:"column:oldest"`
|
||||
Newest string `gorm:"column:newest"`
|
||||
}
|
||||
|
||||
type legacyBatchProgress func(table string, rows, points int64)
|
||||
|
||||
type legacyP95Group struct {
|
||||
metricName string
|
||||
entityID string
|
||||
tags map[string]string
|
||||
values []float64
|
||||
}
|
||||
|
||||
type legacyHourlyP95Aggregator struct {
|
||||
store *metric.Store
|
||||
partitionKey string
|
||||
bucket time.Time
|
||||
groups map[string]*legacyP95Group
|
||||
}
|
||||
|
||||
// LegacyMonitoringMigrationRequired reports whether startup must enter the
|
||||
// restricted 1.2.7 upgrade flow. The migration itself runs only after an
|
||||
// administrator explicitly starts it from that guide.
|
||||
func LegacyMonitoringMigrationRequired(db *gorm.DB) (bool, LegacyMonitoringSummary, error) {
|
||||
done, err := appconfig.GetAs[bool](legacyMonitoringMigrationDoneKey, false)
|
||||
if err != nil {
|
||||
return false, LegacyMonitoringSummary{}, fmt.Errorf("read legacy monitoring migration marker: %w", err)
|
||||
}
|
||||
summary, err := InspectLegacyMonitoring(db)
|
||||
if err != nil {
|
||||
return false, LegacyMonitoringSummary{}, err
|
||||
}
|
||||
return !done && summary.MonitoringRows > 0, summary, nil
|
||||
}
|
||||
|
||||
func InspectLegacyMonitoring(db *gorm.DB) (LegacyMonitoringSummary, error) {
|
||||
var summary LegacyMonitoringSummary
|
||||
if db == nil {
|
||||
return summary, fmt.Errorf("migration database is nil")
|
||||
}
|
||||
if db.Migrator().HasTable(&models.Client{}) {
|
||||
if err := db.Model(&models.Client{}).Count(&summary.ServerCount).Error; err != nil {
|
||||
return summary, fmt.Errorf("count monitored servers: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
counts := map[string]*int64{
|
||||
"records": &summary.LoadRows,
|
||||
"records_long_term": &summary.LoadRows,
|
||||
"gpu_records": &summary.GPURows,
|
||||
"ping_records": &summary.LatencyRows,
|
||||
}
|
||||
var oldest, newest time.Time
|
||||
for _, table := range legacyMonitoringTables {
|
||||
if !db.Migrator().HasTable(table) {
|
||||
continue
|
||||
}
|
||||
var count int64
|
||||
if err := db.Table(table).Count(&count).Error; err != nil {
|
||||
return summary, fmt.Errorf("count legacy %s: %w", table, err)
|
||||
}
|
||||
*counts[table] += count
|
||||
if count == 0 {
|
||||
continue
|
||||
}
|
||||
var bounds legacyMonitoringBounds
|
||||
if err := db.Table(table).Select("MIN(time) AS oldest, MAX(time) AS newest").Scan(&bounds).Error; err != nil {
|
||||
return summary, fmt.Errorf("inspect legacy %s time range: %w", table, err)
|
||||
}
|
||||
tableOldest, err := parseLegacyTimestamp(bounds.Oldest, time.UTC)
|
||||
if err != nil {
|
||||
return summary, fmt.Errorf("parse legacy %s oldest time: %w", table, err)
|
||||
}
|
||||
tableNewest, err := parseLegacyTimestamp(bounds.Newest, time.UTC)
|
||||
if err != nil {
|
||||
return summary, fmt.Errorf("parse legacy %s newest time: %w", table, err)
|
||||
}
|
||||
if oldest.IsZero() || (!tableOldest.IsZero() && tableOldest.Before(oldest)) {
|
||||
oldest = tableOldest
|
||||
}
|
||||
if newest.IsZero() || tableNewest.After(newest) {
|
||||
newest = tableNewest
|
||||
}
|
||||
}
|
||||
|
||||
summary.MonitoringRows = summary.LoadRows + summary.GPURows + summary.LatencyRows
|
||||
if !oldest.IsZero() {
|
||||
oldestCopy := oldest
|
||||
summary.OldestAt = &oldestCopy
|
||||
}
|
||||
if !newest.IsZero() {
|
||||
newestCopy := newest
|
||||
summary.NewestAt = &newestCopy
|
||||
}
|
||||
if !oldest.IsZero() && !newest.IsZero() {
|
||||
estimatedPoints, err := estimateLegacyHourlyP95Points(db, summary, oldest, newest)
|
||||
if err != nil {
|
||||
return summary, err
|
||||
}
|
||||
summary.EstimatedPoints = estimatedPoints
|
||||
retentionEnd := time.Now().UTC()
|
||||
if newest.After(retentionEnd) {
|
||||
retentionEnd = newest
|
||||
}
|
||||
summary.RetentionDays = int(math.Ceil(retentionEnd.Sub(oldest).Hours() / 24))
|
||||
if summary.RetentionDays == 0 && summary.MonitoringRows > 0 {
|
||||
summary.RetentionDays = 1
|
||||
}
|
||||
}
|
||||
return summary, nil
|
||||
}
|
||||
|
||||
func MigrateLegacyMonitoring(ctx context.Context, db *gorm.DB, s *metric.Store, progress func(LegacyMonitoringProgress)) (LegacyMonitoringStats, error) {
|
||||
if err := metricstore.EnsureBuiltinMetricDefinitions(ctx, s); err != nil {
|
||||
return LegacyMonitoringStats{}, fmt.Errorf("ensure built-in metric definitions: %w", err)
|
||||
}
|
||||
return migrateLegacyMonitoringTables(ctx, db, s, progress)
|
||||
}
|
||||
|
||||
// CompleteLegacyMonitoringMigration removes the legacy tables, runs any
|
||||
// caller-owned database maintenance, and only then persists the completion
|
||||
// marker. Keeping the marker last prevents a failed finalization from being
|
||||
// reported as a completed migration.
|
||||
func CompleteLegacyMonitoringMigration(db *gorm.DB, finalize func() error) error {
|
||||
if err := dropLegacyMonitoringTables(db); err != nil {
|
||||
return err
|
||||
}
|
||||
if finalize != nil {
|
||||
if err := finalize(); err != nil {
|
||||
return fmt.Errorf("finalize legacy monitoring migration: %w", err)
|
||||
}
|
||||
}
|
||||
if err := appconfig.Set(legacyMonitoringMigrationDoneKey, true); err != nil {
|
||||
return fmt.Errorf("mark legacy monitoring migration done: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func runLegacyMonitoringMigration(ctx context.Context, db *gorm.DB, s *metric.Store, done bool, markDone func() error) (LegacyMonitoringStats, error) {
|
||||
var stats LegacyMonitoringStats
|
||||
if done {
|
||||
return stats, nil
|
||||
}
|
||||
if db == nil {
|
||||
return stats, fmt.Errorf("migration database is nil")
|
||||
}
|
||||
if s == nil {
|
||||
return stats, fmt.Errorf("metric store is nil")
|
||||
}
|
||||
if markDone == nil {
|
||||
return stats, fmt.Errorf("migration marker writer is nil")
|
||||
}
|
||||
|
||||
stats, err := migrateLegacyMonitoringTables(ctx, db, s, nil)
|
||||
if err != nil {
|
||||
return stats, err
|
||||
}
|
||||
if err := dropLegacyMonitoringTables(db); err != nil {
|
||||
return stats, err
|
||||
}
|
||||
if err := markDone(); err != nil {
|
||||
return stats, fmt.Errorf("mark legacy monitoring migration done: %w", err)
|
||||
}
|
||||
return stats, nil
|
||||
}
|
||||
|
||||
func migrateLegacyMonitoringTables(ctx context.Context, db *gorm.DB, s *metric.Store, progress func(LegacyMonitoringProgress)) (LegacyMonitoringStats, error) {
|
||||
var stats LegacyMonitoringStats
|
||||
summary, err := InspectLegacyMonitoring(db)
|
||||
if err != nil {
|
||||
return stats, err
|
||||
}
|
||||
var rowsDone, writtenPoints int64
|
||||
emit := func(table string, rows, points int64) {
|
||||
rowsDone += rows
|
||||
writtenPoints += points
|
||||
if progress != nil {
|
||||
progress(LegacyMonitoringProgress{
|
||||
Phase: "migrating",
|
||||
Table: table,
|
||||
SourceRowsDone: rowsDone,
|
||||
SourceRowsTotal: summary.MonitoringRows,
|
||||
WrittenPoints: writtenPoints,
|
||||
})
|
||||
}
|
||||
}
|
||||
if progress != nil {
|
||||
emit("", 0, 0)
|
||||
}
|
||||
n, err := migrateLegacyRecordTables(ctx, s, db, []string{"records", "records_long_term"}, emit)
|
||||
if err != nil {
|
||||
return stats, err
|
||||
}
|
||||
stats.Records += n
|
||||
|
||||
n, err = migrateLegacyGPURecordTable(ctx, s, db, "gpu_records", emit)
|
||||
if err != nil {
|
||||
return stats, err
|
||||
}
|
||||
stats.GPU += n
|
||||
|
||||
n, err = migrateLegacyPingRecordTable(ctx, s, db, "ping_records", emit)
|
||||
if err != nil {
|
||||
return stats, err
|
||||
}
|
||||
stats.Ping += n
|
||||
|
||||
return stats, nil
|
||||
}
|
||||
|
||||
func migrateLegacyRecordTables(ctx context.Context, s *metric.Store, db *gorm.DB, tables []string, progress legacyBatchProgress) (int64, error) {
|
||||
var existing []string
|
||||
var total int64
|
||||
for _, table := range tables {
|
||||
if !db.Migrator().HasTable(table) {
|
||||
continue
|
||||
}
|
||||
var count int64
|
||||
if err := db.Table(table).Count(&count).Error; err != nil {
|
||||
return 0, fmt.Errorf("count legacy %s: %w", table, err)
|
||||
}
|
||||
if count > 0 {
|
||||
existing = append(existing, table)
|
||||
total += count
|
||||
}
|
||||
}
|
||||
if len(existing) == 0 {
|
||||
return 0, nil
|
||||
}
|
||||
|
||||
parts := make([]string, 0, len(existing))
|
||||
for _, table := range existing {
|
||||
projection, err := legacyRecordProjection(db, table)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
parts = append(parts, projection)
|
||||
}
|
||||
query := strings.Join(parts, " UNION ALL ") + " ORDER BY client ASC, time ASC"
|
||||
rows, err := db.Raw(query).Rows()
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("stream legacy record tables: %w", err)
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
logger.Infof("migration", "[legacy-migration] aggregating %d rows from %s into 1h P95 points", total, strings.Join(existing, ","))
|
||||
migrated, err := migrateLegacyStream(ctx, db, s, rows, "records", func() *models.Record { return &models.Record{} }, func(value models.Record) []metric.Point {
|
||||
return recordToPoints(value)
|
||||
}, progress)
|
||||
if err != nil {
|
||||
return migrated, fmt.Errorf("aggregate legacy record tables: %w", err)
|
||||
}
|
||||
logger.Infof("migration", "[legacy-migration] aggregated %d rows from %s", migrated, strings.Join(existing, ","))
|
||||
return migrated, nil
|
||||
}
|
||||
|
||||
func migrateLegacyGPURecordTable(ctx context.Context, s *metric.Store, db *gorm.DB, table string, progress legacyBatchProgress) (int64, error) {
|
||||
if !db.Migrator().HasTable(table) {
|
||||
return 0, nil
|
||||
}
|
||||
|
||||
var total int64
|
||||
if err := db.Table(table).Count(&total).Error; err != nil {
|
||||
return 0, fmt.Errorf("count legacy %s: %w", table, err)
|
||||
}
|
||||
if total == 0 {
|
||||
return 0, nil
|
||||
}
|
||||
|
||||
rows, err := db.Table(table).Select("*").Order("client ASC, device_index ASC, device_name ASC, time ASC").Rows()
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("stream legacy %s: %w", table, err)
|
||||
}
|
||||
defer rows.Close()
|
||||
logger.Infof("migration", "[legacy-migration] aggregating %d rows from %s into 1h P95 points", total, table)
|
||||
migrated, err := migrateLegacyStream(ctx, db, s, rows, table, func() *models.GPURecord { return &models.GPURecord{} }, func(value models.GPURecord) []metric.Point {
|
||||
return gpuRecordToPoints(value)
|
||||
}, progress)
|
||||
if err != nil {
|
||||
return migrated, fmt.Errorf("aggregate legacy %s: %w", table, err)
|
||||
}
|
||||
logger.Infof("migration", "[legacy-migration] aggregated %d rows from %s", migrated, table)
|
||||
return migrated, nil
|
||||
}
|
||||
|
||||
func migrateLegacyPingRecordTable(ctx context.Context, s *metric.Store, db *gorm.DB, table string, progress legacyBatchProgress) (int64, error) {
|
||||
if !db.Migrator().HasTable(table) {
|
||||
return 0, nil
|
||||
}
|
||||
|
||||
var total int64
|
||||
if err := db.Table(table).Count(&total).Error; err != nil {
|
||||
return 0, fmt.Errorf("count legacy %s: %w", table, err)
|
||||
}
|
||||
if total == 0 {
|
||||
return 0, nil
|
||||
}
|
||||
|
||||
rows, err := db.Table(table).Select("*").Order("client ASC, task_id ASC, time ASC").Rows()
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("stream legacy %s: %w", table, err)
|
||||
}
|
||||
defer rows.Close()
|
||||
logger.Infof("migration", "[legacy-migration] aggregating %d rows from %s into 1h P95 points", total, table)
|
||||
migrated, err := migrateLegacyStream(ctx, db, s, rows, table, func() *models.PingRecord { return &models.PingRecord{} }, func(value models.PingRecord) []metric.Point {
|
||||
return pingRecordToPoints(value)
|
||||
}, progress)
|
||||
if err != nil {
|
||||
return migrated, fmt.Errorf("aggregate legacy %s: %w", table, err)
|
||||
}
|
||||
logger.Infof("migration", "[legacy-migration] aggregated %d rows from %s", migrated, table)
|
||||
return migrated, nil
|
||||
}
|
||||
|
||||
func migrateLegacyStream[T any](ctx context.Context, db *gorm.DB, s *metric.Store, rows *sql.Rows, table string, newValue func() *T, toPoints func(T) []metric.Point, progress legacyBatchProgress) (int64, error) {
|
||||
aggregator := &legacyHourlyP95Aggregator{store: s}
|
||||
var migrated, batchRows, batchPoints int64
|
||||
emit := func(force bool) {
|
||||
if progress != nil && (batchRows > 0 || force && batchPoints > 0) {
|
||||
progress(table, batchRows, batchPoints)
|
||||
}
|
||||
batchRows = 0
|
||||
batchPoints = 0
|
||||
}
|
||||
for rows.Next() {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return migrated, ctx.Err()
|
||||
default:
|
||||
}
|
||||
value := newValue()
|
||||
if err := db.ScanRows(rows, value); err != nil {
|
||||
return migrated, err
|
||||
}
|
||||
written, err := aggregator.Add(ctx, toPoints(*value))
|
||||
if err != nil {
|
||||
return migrated, err
|
||||
}
|
||||
migrated++
|
||||
batchRows++
|
||||
batchPoints += written
|
||||
if batchRows >= legacyMonitoringBatchSize {
|
||||
emit(false)
|
||||
}
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return migrated, err
|
||||
}
|
||||
written, err := aggregator.Flush(ctx)
|
||||
if err != nil {
|
||||
return migrated, err
|
||||
}
|
||||
batchPoints += written
|
||||
emit(true)
|
||||
return migrated, nil
|
||||
}
|
||||
|
||||
func legacyRecordProjection(db *gorm.DB, table string) (string, error) {
|
||||
columnTypes, err := db.Migrator().ColumnTypes(table)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("inspect legacy %s columns: %w", table, err)
|
||||
}
|
||||
available := make(map[string]struct{}, len(columnTypes))
|
||||
for _, column := range columnTypes {
|
||||
available[strings.ToLower(column.Name())] = struct{}{}
|
||||
}
|
||||
for _, required := range []string{"client", "time"} {
|
||||
if _, ok := available[required]; !ok {
|
||||
return "", fmt.Errorf("legacy %s is missing required column %s", table, required)
|
||||
}
|
||||
}
|
||||
columns := []string{
|
||||
"client", "time", "cpu", "gpu", "ram", "swap", "load", "disk",
|
||||
"net_in", "net_out", "net_total_up", "net_total_down", "traffic_up",
|
||||
"traffic_down", "process", "connections", "connections_udp",
|
||||
}
|
||||
projection := make([]string, 0, len(columns))
|
||||
for _, column := range columns {
|
||||
if _, ok := available[column]; ok {
|
||||
projection = append(projection, column)
|
||||
} else {
|
||||
projection = append(projection, "0 AS "+column)
|
||||
}
|
||||
}
|
||||
return "SELECT " + strings.Join(projection, ", ") + " FROM " + table, nil
|
||||
}
|
||||
|
||||
func (a *legacyHourlyP95Aggregator) Add(ctx context.Context, points []metric.Point) (int64, error) {
|
||||
var written int64
|
||||
for _, point := range points {
|
||||
bucket := point.Timestamp.UTC().Truncate(legacyMonitoringInterval)
|
||||
partition, err := legacyPointPartition(point)
|
||||
if err != nil {
|
||||
return written, err
|
||||
}
|
||||
if a.groups != nil && (partition != a.partitionKey || !bucket.Equal(a.bucket)) {
|
||||
if partition == a.partitionKey && bucket.Before(a.bucket) {
|
||||
return written, fmt.Errorf("legacy points are not ordered within series: %s before %s", bucket, a.bucket)
|
||||
}
|
||||
n, err := a.Flush(ctx)
|
||||
written += n
|
||||
if err != nil {
|
||||
return written, err
|
||||
}
|
||||
}
|
||||
if a.groups == nil {
|
||||
a.partitionKey = partition
|
||||
a.bucket = bucket
|
||||
a.groups = make(map[string]*legacyP95Group)
|
||||
}
|
||||
group := a.groups[point.MetricName]
|
||||
if group == nil {
|
||||
group = &legacyP95Group{
|
||||
metricName: point.MetricName,
|
||||
entityID: point.EntityID,
|
||||
tags: cloneLegacyTags(point.Tags),
|
||||
}
|
||||
a.groups[point.MetricName] = group
|
||||
}
|
||||
group.values = append(group.values, point.Value)
|
||||
}
|
||||
return written, nil
|
||||
}
|
||||
|
||||
func (a *legacyHourlyP95Aggregator) Flush(ctx context.Context) (int64, error) {
|
||||
if len(a.groups) == 0 {
|
||||
return 0, nil
|
||||
}
|
||||
names := make([]string, 0, len(a.groups))
|
||||
for name := range a.groups {
|
||||
names = append(names, name)
|
||||
}
|
||||
sort.Strings(names)
|
||||
points := make([]metric.Point, 0, len(names))
|
||||
for _, name := range names {
|
||||
group := a.groups[name]
|
||||
points = append(points, metric.Point{
|
||||
MetricName: group.metricName,
|
||||
EntityID: group.entityID,
|
||||
Timestamp: a.bucket,
|
||||
Value: legacyP95(group.values),
|
||||
Tags: group.tags,
|
||||
})
|
||||
}
|
||||
if err := a.store.ReplaceRollupPoints(ctx, legacyMonitoringInterval, points); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
a.partitionKey = ""
|
||||
a.bucket = time.Time{}
|
||||
a.groups = nil
|
||||
return int64(len(points)), nil
|
||||
}
|
||||
|
||||
func legacyP95(values []float64) float64 {
|
||||
if len(values) == 0 {
|
||||
return 0
|
||||
}
|
||||
sort.Float64s(values)
|
||||
if len(values) == 1 {
|
||||
return values[0]
|
||||
}
|
||||
position := 0.95 * float64(len(values)-1)
|
||||
lower := int(math.Floor(position))
|
||||
upper := int(math.Ceil(position))
|
||||
if lower == upper {
|
||||
return values[lower]
|
||||
}
|
||||
weight := position - float64(lower)
|
||||
return values[lower]*(1-weight) + values[upper]*weight
|
||||
}
|
||||
|
||||
func legacyPointPartition(point metric.Point) (string, error) {
|
||||
tags, err := json.Marshal(point.Tags)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("encode legacy point tags: %w", err)
|
||||
}
|
||||
return point.EntityID + "\x00" + string(tags), nil
|
||||
}
|
||||
|
||||
func cloneLegacyTags(tags map[string]string) map[string]string {
|
||||
if len(tags) == 0 {
|
||||
return nil
|
||||
}
|
||||
cloned := make(map[string]string, len(tags))
|
||||
for key, value := range tags {
|
||||
cloned[key] = value
|
||||
}
|
||||
return cloned
|
||||
}
|
||||
|
||||
func estimateLegacyHourlyP95Points(db *gorm.DB, summary LegacyMonitoringSummary, oldest, newest time.Time) (int64, error) {
|
||||
type estimate struct {
|
||||
Count int64 `gorm:"column:count"`
|
||||
}
|
||||
countSeries := func(query string) (int64, error) {
|
||||
var result estimate
|
||||
if err := db.Raw("SELECT COUNT(*) AS count FROM (" + query + ")").Scan(&result).Error; err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return result.Count, nil
|
||||
}
|
||||
firstHour := oldest.UTC().Truncate(legacyMonitoringInterval)
|
||||
lastHour := newest.UTC().Truncate(legacyMonitoringInterval)
|
||||
hours := int64(lastHour.Sub(firstHour)/legacyMonitoringInterval) + 1
|
||||
if hours < 1 {
|
||||
hours = 1
|
||||
}
|
||||
|
||||
var recordParts []string
|
||||
for _, table := range []string{"records", "records_long_term"} {
|
||||
if db.Migrator().HasTable(table) {
|
||||
recordParts = append(recordParts, "SELECT client FROM "+table)
|
||||
}
|
||||
}
|
||||
var total int64
|
||||
if len(recordParts) > 0 {
|
||||
series, err := countSeries(strings.Join(recordParts, " UNION "))
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("estimate load series: %w", err)
|
||||
}
|
||||
total += minLegacyBuckets(summary.LoadRows, series*hours) * 15
|
||||
}
|
||||
if db.Migrator().HasTable("gpu_records") {
|
||||
series, err := countSeries("SELECT client, device_index, device_name FROM gpu_records GROUP BY client, device_index, device_name")
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("estimate GPU series: %w", err)
|
||||
}
|
||||
total += minLegacyBuckets(summary.GPURows, series*hours) * 4
|
||||
}
|
||||
if db.Migrator().HasTable("ping_records") {
|
||||
series, err := countSeries("SELECT client, task_id FROM ping_records GROUP BY client, task_id")
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("estimate ping series: %w", err)
|
||||
}
|
||||
total += minLegacyBuckets(summary.LatencyRows, series*hours) * 2
|
||||
}
|
||||
return total, nil
|
||||
}
|
||||
|
||||
func minLegacyBuckets(sourceRows, estimatedBuckets int64) int64 {
|
||||
if sourceRows < estimatedBuckets {
|
||||
return sourceRows
|
||||
}
|
||||
return estimatedBuckets
|
||||
}
|
||||
|
||||
func recordToPoints(rec models.Record) []metric.Point {
|
||||
ts := rec.Time
|
||||
entityID := rec.Client
|
||||
return []metric.Point{
|
||||
{MetricName: metricstore.MetricCPU, EntityID: entityID, Timestamp: ts, Value: float64(rec.Cpu)},
|
||||
{MetricName: metricstore.MetricGPU, EntityID: entityID, Timestamp: ts, Value: float64(rec.Gpu)},
|
||||
{MetricName: metricstore.MetricRAM, EntityID: entityID, Timestamp: ts, Value: float64(rec.Ram)},
|
||||
{MetricName: metricstore.MetricSwap, EntityID: entityID, Timestamp: ts, Value: float64(rec.Swap)},
|
||||
{MetricName: metricstore.MetricLoad, EntityID: entityID, Timestamp: ts, Value: float64(rec.Load)},
|
||||
{MetricName: metricstore.MetricDisk, EntityID: entityID, Timestamp: ts, Value: float64(rec.Disk)},
|
||||
{MetricName: metricstore.MetricNetIn, EntityID: entityID, Timestamp: ts, Value: float64(rec.NetIn)},
|
||||
{MetricName: metricstore.MetricNetOut, EntityID: entityID, Timestamp: ts, Value: float64(rec.NetOut)},
|
||||
{MetricName: metricstore.MetricNetTotalUp, EntityID: entityID, Timestamp: ts, Value: float64(rec.NetTotalUp)},
|
||||
{MetricName: metricstore.MetricNetTotalDown, EntityID: entityID, Timestamp: ts, Value: float64(rec.NetTotalDown)},
|
||||
{MetricName: metricstore.MetricTrafficUp, EntityID: entityID, Timestamp: ts, Value: float64(rec.TrafficUp)},
|
||||
{MetricName: metricstore.MetricTrafficDown, EntityID: entityID, Timestamp: ts, Value: float64(rec.TrafficDown)},
|
||||
{MetricName: metricstore.MetricProcess, EntityID: entityID, Timestamp: ts, Value: float64(rec.Process)},
|
||||
{MetricName: metricstore.MetricConnections, EntityID: entityID, Timestamp: ts, Value: float64(rec.Connections)},
|
||||
{MetricName: metricstore.MetricConnectionsUDP, EntityID: entityID, Timestamp: ts, Value: float64(rec.ConnectionsUdp)},
|
||||
}
|
||||
}
|
||||
|
||||
func gpuRecordToPoints(rec models.GPURecord) []metric.Point {
|
||||
ts := rec.Time
|
||||
tags := map[string]string{
|
||||
"device_index": fmt.Sprintf("%d", rec.DeviceIndex),
|
||||
"device_name": rec.DeviceName,
|
||||
}
|
||||
return []metric.Point{
|
||||
{MetricName: metricstore.MetricGPUMem, EntityID: rec.Client, Timestamp: ts, Value: float64(rec.MemUsed), Tags: tags},
|
||||
{MetricName: metricstore.MetricGPUMemTotal, EntityID: rec.Client, Timestamp: ts, Value: float64(rec.MemTotal), Tags: tags},
|
||||
{MetricName: metricstore.MetricGPUDeviceUsage, EntityID: rec.Client, Timestamp: ts, Value: float64(rec.Utilization), Tags: tags},
|
||||
{MetricName: metricstore.MetricGPUTemp, EntityID: rec.Client, Timestamp: ts, Value: float64(rec.Temperature), Tags: tags},
|
||||
}
|
||||
}
|
||||
|
||||
func pingRecordToPoints(rec models.PingRecord) []metric.Point {
|
||||
ts := rec.Time
|
||||
tags := map[string]string{"task_id": fmt.Sprintf("%d", rec.TaskId)}
|
||||
loss := 0.0
|
||||
if rec.Value < 0 {
|
||||
loss = 1
|
||||
}
|
||||
return []metric.Point{
|
||||
{
|
||||
MetricName: metricstore.MetricPingLatency,
|
||||
EntityID: rec.Client,
|
||||
Timestamp: ts,
|
||||
Value: float64(rec.Value),
|
||||
Tags: tags,
|
||||
},
|
||||
{
|
||||
MetricName: metricstore.MetricPingLoss,
|
||||
EntityID: rec.Client,
|
||||
Timestamp: ts,
|
||||
Value: loss,
|
||||
Tags: tags,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func dropLegacyMonitoringTables(db *gorm.DB) error {
|
||||
for _, table := range legacyMonitoringTables {
|
||||
if !db.Migrator().HasTable(table) {
|
||||
continue
|
||||
}
|
||||
logger.Infof("migration", "[legacy-migration] dropping legacy table %s", table)
|
||||
if err := db.Migrator().DropTable(table); err != nil {
|
||||
return fmt.Errorf("drop legacy table %s: %w", table, err)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,303 @@
|
||||
package migrations
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"errors"
|
||||
"math"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/komari-monitor/komari/database/metricstore"
|
||||
"github.com/komari-monitor/komari/database/models"
|
||||
appconfig "github.com/komari-monitor/komari/internal/config"
|
||||
"github.com/komari-monitor/komari/pkg/metric"
|
||||
"gorm.io/driver/sqlite"
|
||||
"gorm.io/gorm"
|
||||
)
|
||||
|
||||
func TestLegacyMonitoringTablesMigratedByOneShotMigration(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
mainDB, err := gorm.Open(sqlite.Open("file:"+filepath.ToSlash(filepath.Join(t.TempDir(), "komari.db"))+"?mode=rwc"), &gorm.Config{})
|
||||
if err != nil {
|
||||
t.Fatalf("open legacy db: %v", err)
|
||||
}
|
||||
if sqlDB, err := mainDB.DB(); err == nil {
|
||||
t.Cleanup(func() { _ = sqlDB.Close() })
|
||||
} else {
|
||||
t.Fatalf("legacy sql db: %v", err)
|
||||
}
|
||||
if err := mainDB.AutoMigrate(&models.Record{}, &models.GPURecord{}, &models.PingRecord{}); err != nil {
|
||||
t.Fatalf("migrate legacy tables: %v", err)
|
||||
}
|
||||
if err := mainDB.Table("records_long_term").AutoMigrate(&models.Record{}); err != nil {
|
||||
t.Fatalf("migrate legacy long-term table: %v", err)
|
||||
}
|
||||
|
||||
base := time.Date(2026, 7, 8, 23, 42, 0, 0, time.UTC)
|
||||
if err := mainDB.Create(&models.Record{Client: "client-a", Time: base, Cpu: 12.5, Ram: 2048}).Error; err != nil {
|
||||
t.Fatalf("seed records: %v", err)
|
||||
}
|
||||
if err := mainDB.Table("records_long_term").Create(&models.Record{Client: "client-a", Time: base.Add(time.Minute), Cpu: 22.5, Ram: 4096}).Error; err != nil {
|
||||
t.Fatalf("seed records_long_term: %v", err)
|
||||
}
|
||||
if err := mainDB.Create(&models.GPURecord{Client: "client-a", Time: base, DeviceIndex: 0, DeviceName: "GPU 0", MemUsed: 1024, MemTotal: 2048, Utilization: 67, Temperature: 55}).Error; err != nil {
|
||||
t.Fatalf("seed gpu_records: %v", err)
|
||||
}
|
||||
if err := mainDB.Create(&models.PingRecord{Client: "client-a", TaskId: 7, Time: base, Value: 36}).Error; err != nil {
|
||||
t.Fatalf("seed ping_records: %v", err)
|
||||
}
|
||||
if err := mainDB.Create(&models.PingRecord{Client: "client-a", TaskId: 7, Time: base.Add(30 * time.Second), Value: -1}).Error; err != nil {
|
||||
t.Fatalf("seed loss ping_records: %v", err)
|
||||
}
|
||||
summary, err := InspectLegacyMonitoring(mainDB)
|
||||
if err != nil {
|
||||
t.Fatalf("inspect legacy monitoring summary: %v", err)
|
||||
}
|
||||
if summary.LoadRows != 2 || summary.GPURows != 1 || summary.LatencyRows != 2 || summary.MonitoringRows != 5 {
|
||||
t.Fatalf("unexpected legacy monitoring summary: %#v", summary)
|
||||
}
|
||||
if summary.EstimatedPoints != 21 || summary.RetentionDays < 1 {
|
||||
t.Fatalf("unexpected legacy point estimate or retention: %#v", summary)
|
||||
}
|
||||
|
||||
metricDB, err := sql.Open("sqlite3", "file:"+filepath.ToSlash(filepath.Join(t.TempDir(), "metrics.db"))+"?mode=rwc")
|
||||
if err != nil {
|
||||
t.Fatalf("open metric db: %v", err)
|
||||
}
|
||||
metricStore, err := metric.Open(ctx, metric.SQLite("", metric.WithDB(metricDB)))
|
||||
if err != nil {
|
||||
t.Fatalf("open metric store: %v", err)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
_ = metricStore.Close()
|
||||
_ = metricDB.Close()
|
||||
})
|
||||
|
||||
markDoneCalls := 0
|
||||
var lastProgress LegacyMonitoringProgress
|
||||
if _, err := MigrateLegacyMonitoring(ctx, mainDB, metricStore, func(progress LegacyMonitoringProgress) {
|
||||
lastProgress = progress
|
||||
}); err != nil {
|
||||
t.Fatalf("migrate legacy monitoring with progress: %v", err)
|
||||
}
|
||||
if lastProgress.SourceRowsDone != 5 || lastProgress.SourceRowsTotal != 5 || lastProgress.WrittenPoints != 21 {
|
||||
t.Fatalf("unexpected final migration progress: %#v", lastProgress)
|
||||
}
|
||||
|
||||
stats, err := runLegacyMonitoringMigration(ctx, mainDB, metricStore, false, func() error {
|
||||
markDoneCalls++
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("run legacy monitoring migration: %v", err)
|
||||
}
|
||||
if markDoneCalls != 1 {
|
||||
t.Fatalf("expected migration marker to be written once, got %d", markDoneCalls)
|
||||
}
|
||||
if stats.Records != 2 || stats.GPU != 1 || stats.Ping != 2 {
|
||||
t.Fatalf("unexpected stats: %#v", stats)
|
||||
}
|
||||
|
||||
hour := base.Truncate(time.Hour)
|
||||
cpuPoints := queryLegacyRollups(t, ctx, metricStore, metric.Query{MetricName: metricstore.MetricCPU, EntityID: "client-a", Start: hour.Add(-time.Second), End: hour.Add(time.Hour)})
|
||||
if len(cpuPoints) != 1 || math.Abs(cpuPoints[0].Value-22) > 1e-9 || !cpuPoints[0].Bucket.Equal(hour) {
|
||||
t.Fatalf("unexpected cpu points: %#v", cpuPoints)
|
||||
}
|
||||
|
||||
gpuPoints := queryLegacyRollups(t, ctx, metricStore, metric.Query{MetricName: metricstore.MetricGPUDeviceUsage, EntityID: "client-a", Start: hour.Add(-time.Second), End: hour.Add(time.Hour), Tags: map[string]string{"device_index": "0"}})
|
||||
if len(gpuPoints) != 1 || gpuPoints[0].Value != 67 {
|
||||
t.Fatalf("unexpected gpu points: %#v", gpuPoints)
|
||||
}
|
||||
|
||||
pingPoints := queryLegacyRollups(t, ctx, metricStore, metric.Query{MetricName: metricstore.MetricPingLatency, EntityID: "client-a", Start: hour.Add(-time.Second), End: hour.Add(time.Hour), Tags: map[string]string{"task_id": "7"}})
|
||||
if len(pingPoints) != 1 || math.Abs(pingPoints[0].Value-34.15) > 1e-9 || !pingPoints[0].Bucket.Equal(hour) {
|
||||
t.Fatalf("unexpected ping points: %#v", pingPoints)
|
||||
}
|
||||
|
||||
pingLossPoints := queryLegacyRollups(t, ctx, metricStore, metric.Query{MetricName: metricstore.MetricPingLoss, EntityID: "client-a", Start: hour.Add(-time.Second), End: hour.Add(time.Hour), Tags: map[string]string{"task_id": "7"}})
|
||||
if len(pingLossPoints) != 1 || math.Abs(pingLossPoints[0].Value-0.95) > 1e-9 || !pingLossPoints[0].Bucket.Equal(hour) {
|
||||
t.Fatalf("unexpected ping loss points: %#v", pingLossPoints)
|
||||
}
|
||||
|
||||
for _, table := range legacyMonitoringTables {
|
||||
if mainDB.Migrator().HasTable(table) {
|
||||
t.Fatalf("legacy table %s still exists", table)
|
||||
}
|
||||
}
|
||||
|
||||
stats, err = runLegacyMonitoringMigration(ctx, mainDB, metricStore, true, func() error {
|
||||
markDoneCalls++
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("rerun completed legacy monitoring migration: %v", err)
|
||||
}
|
||||
if stats != (LegacyMonitoringStats{}) {
|
||||
t.Fatalf("completed migration should not scan legacy tables, got %#v", stats)
|
||||
}
|
||||
if markDoneCalls != 1 {
|
||||
t.Fatalf("completed migration rewrote marker, calls=%d", markDoneCalls)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLegacyHourlyP95PreservesHoursAndTaggedSeries(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
metricDB, err := sql.Open("sqlite3", "file:"+filepath.ToSlash(filepath.Join(t.TempDir(), "hourly-p95.db"))+"?mode=rwc")
|
||||
if err != nil {
|
||||
t.Fatalf("open metric db: %v", err)
|
||||
}
|
||||
store, err := metric.Open(ctx, metric.SQLite("", metric.WithDB(metricDB)))
|
||||
if err != nil {
|
||||
t.Fatalf("open metric store: %v", err)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
_ = store.Close()
|
||||
_ = metricDB.Close()
|
||||
})
|
||||
if err := store.UpsertMetric(ctx, metric.Definition{Name: "test.p95", Type: metric.TypeGauge, RetentionDays: 1}); err != nil {
|
||||
t.Fatalf("create test metric: %v", err)
|
||||
}
|
||||
|
||||
base := time.Date(2026, 7, 8, 10, 0, 0, 0, time.UTC)
|
||||
aggregator := &legacyHourlyP95Aggregator{store: store}
|
||||
inputs := [][]metric.Point{
|
||||
{{MetricName: "test.p95", EntityID: "node-a", Timestamp: base.Add(time.Minute), Value: 10, Tags: map[string]string{"task_id": "1"}}},
|
||||
{{MetricName: "test.p95", EntityID: "node-a", Timestamp: base.Add(2 * time.Minute), Value: 20, Tags: map[string]string{"task_id": "1"}}},
|
||||
{{MetricName: "test.p95", EntityID: "node-a", Timestamp: base.Add(time.Hour), Value: 30, Tags: map[string]string{"task_id": "1"}}},
|
||||
{{MetricName: "test.p95", EntityID: "node-a", Timestamp: base.Add(time.Minute), Value: 100, Tags: map[string]string{"task_id": "2"}}},
|
||||
}
|
||||
for _, points := range inputs {
|
||||
if _, err := aggregator.Add(ctx, points); err != nil {
|
||||
t.Fatalf("add aggregate input: %v", err)
|
||||
}
|
||||
}
|
||||
if _, err := aggregator.Flush(ctx); err != nil {
|
||||
t.Fatalf("flush aggregate input: %v", err)
|
||||
}
|
||||
|
||||
taskOne := queryLegacyRollups(t, ctx, store, metric.Query{
|
||||
MetricName: "test.p95", EntityID: "node-a", Start: base.Add(-time.Second), End: base.Add(2 * time.Hour),
|
||||
Tags: map[string]string{"task_id": "1"}, Order: metric.OrderAsc,
|
||||
})
|
||||
if len(taskOne) != 2 || math.Abs(taskOne[0].Value-19.5) > 1e-9 || taskOne[1].Value != 30 {
|
||||
t.Fatalf("unexpected task one aggregates: %#v", taskOne)
|
||||
}
|
||||
taskTwo := queryLegacyRollups(t, ctx, store, metric.Query{
|
||||
MetricName: "test.p95", EntityID: "node-a", Start: base.Add(-time.Second), End: base.Add(2 * time.Hour),
|
||||
Tags: map[string]string{"task_id": "2"}, Order: metric.OrderAsc,
|
||||
})
|
||||
if len(taskTwo) != 1 || taskTwo[0].Value != 100 || !taskTwo[0].Bucket.Equal(base) {
|
||||
t.Fatalf("unexpected task two aggregates: %#v", taskTwo)
|
||||
}
|
||||
}
|
||||
|
||||
func queryLegacyRollups(t *testing.T, ctx context.Context, store *metric.Store, query metric.Query) []metric.AggregatePoint {
|
||||
t.Helper()
|
||||
points, err := store.Series(ctx, metric.AggregateQuery{
|
||||
Query: query, Aggregation: metric.AggLast, Interval: time.Hour, PreserveSeries: true,
|
||||
}, query.End)
|
||||
if err != nil {
|
||||
t.Fatalf("query %s rollups: %v", query.MetricName, err)
|
||||
}
|
||||
return points
|
||||
}
|
||||
|
||||
func TestLegacyRecordProjectionFillsMissingMetricColumns(t *testing.T) {
|
||||
db, err := gorm.Open(sqlite.Open("file:"+filepath.ToSlash(filepath.Join(t.TempDir(), "minimal-legacy.db"))+"?mode=rwc"), &gorm.Config{})
|
||||
if err != nil {
|
||||
t.Fatalf("open minimal legacy db: %v", err)
|
||||
}
|
||||
if sqlDB, err := db.DB(); err == nil {
|
||||
t.Cleanup(func() { _ = sqlDB.Close() })
|
||||
} else {
|
||||
t.Fatalf("minimal legacy sql db: %v", err)
|
||||
}
|
||||
if err := db.Exec("CREATE TABLE records_minimal (client TEXT NOT NULL, time DATETIME NOT NULL, cpu REAL NOT NULL)").Error; err != nil {
|
||||
t.Fatalf("create minimal legacy table: %v", err)
|
||||
}
|
||||
base := time.Date(2026, 7, 8, 10, 0, 0, 0, time.UTC)
|
||||
if err := db.Exec("INSERT INTO records_minimal (client, time, cpu) VALUES (?, ?, ?)", "node-a", base, 42.5).Error; err != nil {
|
||||
t.Fatalf("insert minimal legacy row: %v", err)
|
||||
}
|
||||
projection, err := legacyRecordProjection(db, "records_minimal")
|
||||
if err != nil {
|
||||
t.Fatalf("build legacy projection: %v", err)
|
||||
}
|
||||
var record models.Record
|
||||
if err := db.Raw(projection).Scan(&record).Error; err != nil {
|
||||
t.Fatalf("scan projected legacy row: %v", err)
|
||||
}
|
||||
if record.Client != "node-a" || !record.Time.Equal(base) || record.Cpu != 42.5 {
|
||||
t.Fatalf("unexpected projected record: %#v", record)
|
||||
}
|
||||
if record.ConnectionsUdp != 0 || record.NetTotalDown != 0 {
|
||||
t.Fatalf("missing metric columns were not zero-filled: %#v", record)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCompleteLegacyMonitoringMigrationFinalizesBeforeMarkingDone(t *testing.T) {
|
||||
db, err := gorm.Open(sqlite.Open("file:"+filepath.ToSlash(filepath.Join(t.TempDir(), "finalize.db"))+"?mode=rwc"), &gorm.Config{})
|
||||
if err != nil {
|
||||
t.Fatalf("open finalization database: %v", err)
|
||||
}
|
||||
if sqlDB, err := db.DB(); err == nil {
|
||||
t.Cleanup(func() { _ = sqlDB.Close() })
|
||||
} else {
|
||||
t.Fatalf("finalization sql db: %v", err)
|
||||
}
|
||||
appconfig.SetDb(db)
|
||||
if err := db.AutoMigrate(&models.Record{}); err != nil {
|
||||
t.Fatalf("migrate legacy table: %v", err)
|
||||
}
|
||||
if err := appconfig.Set(legacyMonitoringMigrationDoneKey, false); err != nil {
|
||||
t.Fatalf("reset completion marker: %v", err)
|
||||
}
|
||||
|
||||
finalized := false
|
||||
if err := CompleteLegacyMonitoringMigration(db, func() error {
|
||||
finalized = true
|
||||
if db.Migrator().HasTable(&models.Record{}) {
|
||||
t.Fatal("legacy table still exists during finalization")
|
||||
}
|
||||
done, err := appconfig.GetAs[bool](legacyMonitoringMigrationDoneKey, false)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if done {
|
||||
t.Fatal("completion marker was written before finalization")
|
||||
}
|
||||
return nil
|
||||
}); err != nil {
|
||||
t.Fatalf("complete legacy migration: %v", err)
|
||||
}
|
||||
if !finalized {
|
||||
t.Fatal("finalizer was not called")
|
||||
}
|
||||
done, err := appconfig.GetAs[bool](legacyMonitoringMigrationDoneKey, false)
|
||||
if err != nil {
|
||||
t.Fatalf("read completion marker: %v", err)
|
||||
}
|
||||
if !done {
|
||||
t.Fatal("completion marker was not written after finalization")
|
||||
}
|
||||
|
||||
if err := db.AutoMigrate(&models.Record{}); err != nil {
|
||||
t.Fatalf("restore legacy table: %v", err)
|
||||
}
|
||||
if err := appconfig.Set(legacyMonitoringMigrationDoneKey, false); err != nil {
|
||||
t.Fatalf("reset completion marker after success: %v", err)
|
||||
}
|
||||
finalizeErr := errors.New("vacuum failed")
|
||||
if err := CompleteLegacyMonitoringMigration(db, func() error { return finalizeErr }); !errors.Is(err, finalizeErr) {
|
||||
t.Fatalf("finalization error = %v, want %v", err, finalizeErr)
|
||||
}
|
||||
done, err = appconfig.GetAs[bool](legacyMonitoringMigrationDoneKey, false)
|
||||
if err != nil {
|
||||
t.Fatalf("read completion marker after failure: %v", err)
|
||||
}
|
||||
if done {
|
||||
t.Fatal("failed finalization wrote the completion marker")
|
||||
}
|
||||
}
|
||||
@@ -3,13 +3,13 @@ package migrations
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"log"
|
||||
logger "github.com/komari-monitor/komari/utils/log"
|
||||
"reflect"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/komari-monitor/komari/database/models"
|
||||
appconfig "github.com/komari-monitor/komari/pkg/config"
|
||||
appconfig "github.com/komari-monitor/komari/internal/config"
|
||||
"gorm.io/gorm"
|
||||
"gorm.io/gorm/clause"
|
||||
)
|
||||
@@ -27,8 +27,6 @@ type legacyModelConfig struct {
|
||||
EulaAccepted bool `json:"eula_accepted" gorm:"default:false"`
|
||||
GeoIpEnabled bool `json:"geo_ip_enabled" gorm:"default:true"`
|
||||
GeoIpProvider string `json:"geo_ip_provider" gorm:"type:varchar(20);default:'ip-api'"`
|
||||
NezhaCompatEnabled bool `json:"nezha_compat_enabled" gorm:"default:false"`
|
||||
NezhaCompatListen string `json:"nezha_compat_listen" gorm:"type:varchar(100);default:''"`
|
||||
OAuthEnabled bool `json:"o_auth_enabled" gorm:"default:false"`
|
||||
OAuthProvider string `json:"o_auth_provider" gorm:"type:varchar(50);default:'github'"`
|
||||
DisablePasswordLogin bool `json:"disable_password_login" gorm:"default:false"`
|
||||
@@ -41,11 +39,8 @@ type legacyModelConfig struct {
|
||||
ExpireNotificationLeadDays int `json:"expire_notification_lead_days" gorm:"default:7"`
|
||||
LoginNotification bool `json:"login_notification" gorm:"default:false"`
|
||||
TrafficLimitPercentage float64 `json:"traffic_limit_percentage" gorm:"default:80.00"`
|
||||
RecordEnabled bool `json:"record_enabled" gorm:"default:true"`
|
||||
RecordPreserveTime int `json:"record_preserve_time" gorm:"default:720"`
|
||||
PingRecordPreserveTime int `json:"ping_record_preserve_time" gorm:"default:24"`
|
||||
CreatedAt models.LocalTime
|
||||
UpdatedAt models.LocalTime
|
||||
CreatedAt time.Time
|
||||
UpdatedAt time.Time
|
||||
}
|
||||
|
||||
func (legacyModelConfig) TableName() string {
|
||||
@@ -66,12 +61,9 @@ type legacyConfig struct {
|
||||
BaseScriptsURLKey string `json:"base_scripts_url"`
|
||||
GeoIpEnabled bool `json:"geo_ip_enabled"`
|
||||
GeoIpProvider string `json:"geo_ip_provider"`
|
||||
NezhaCompatEnabled bool `json:"nezha_compat_enabled"`
|
||||
NezhaCompatListen string `json:"nezha_compat_listen"`
|
||||
OAuthEnabled bool `json:"o_auth_enabled"`
|
||||
OAuthProvider string `json:"o_auth_provider"`
|
||||
DisablePasswordLogin bool `json:"disable_password_login"`
|
||||
CloudflareTunnelToken string `json:"cloudflare_tunnel_token"`
|
||||
CustomHead string `json:"custom_head"`
|
||||
CustomBody string `json:"custom_body"`
|
||||
NotificationEnabled bool `json:"notification_enabled"`
|
||||
@@ -81,9 +73,6 @@ type legacyConfig struct {
|
||||
ExpireNotificationLeadDays int `json:"expire_notification_lead_days"`
|
||||
LoginNotification bool `json:"login_notification"`
|
||||
TrafficLimitPercentage float64 `json:"traffic_limit_percentage"`
|
||||
RecordEnabled bool `json:"record_enabled"`
|
||||
RecordPreserveTime int `json:"record_preserve_time"`
|
||||
PingRecordPreserveTime int `json:"ping_record_preserve_time"`
|
||||
UpdatedAt time.Time `json:"updated_at"`
|
||||
}
|
||||
|
||||
@@ -108,6 +97,9 @@ func Run(ctx Context) error {
|
||||
}
|
||||
|
||||
legacyConfigTable := hasLegacyConfigTable(db)
|
||||
if err := migrateLegacyTimestampColumns(db); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if legacyConfigTable {
|
||||
if err := migrateLegacyOidcConfig(db); err != nil {
|
||||
@@ -117,7 +109,6 @@ func Run(ctx Context) error {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
if err := migrateLegacyClientInfo(db); err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -132,10 +123,37 @@ func Run(ctx Context) error {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if err := migrateDeprecatedMetricRetentionConfig(db); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := migrateRemovedCompatibilityConfig(db); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := markTimestampMigrationDone(db); err != nil {
|
||||
return fmt.Errorf("mark UTC timestamp migration done: %w", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func migrateDeprecatedMetricRetentionConfig(db *gorm.DB) error {
|
||||
if !db.Migrator().HasTable(&appconfig.ConfigItem{}) {
|
||||
return nil
|
||||
}
|
||||
return db.Delete(&appconfig.ConfigItem{}, "key = ?", "metric_retention_days").Error
|
||||
}
|
||||
|
||||
func migrateRemovedCompatibilityConfig(db *gorm.DB) error {
|
||||
if !db.Migrator().HasTable(&appconfig.ConfigItem{}) {
|
||||
return nil
|
||||
}
|
||||
return db.Delete(&appconfig.ConfigItem{}, "key IN ?", []string{
|
||||
"nezha_compat_enabled",
|
||||
"nezha_compat_listen",
|
||||
"low_resource_mode",
|
||||
}).Error
|
||||
}
|
||||
|
||||
func hasLegacyConfigTable(db *gorm.DB) bool {
|
||||
if !db.Migrator().HasTable("configs") {
|
||||
return false
|
||||
@@ -159,7 +177,7 @@ func hasTableColumn(db *gorm.DB, tableName, columnName string) bool {
|
||||
|
||||
func migrateLegacyLoadNotification(db *gorm.DB) error {
|
||||
if db.Migrator().HasColumn(&models.LoadNotification{}, "client") {
|
||||
log.Println("[>0.1.4] Rebuilding LoadNotification table....")
|
||||
logger.InfoArgs("migration", "[>0.1.4] Rebuilding LoadNotification table....")
|
||||
return db.Migrator().DropTable(&models.LoadNotification{})
|
||||
}
|
||||
return nil
|
||||
@@ -170,7 +188,7 @@ func migrateLegacyOidcConfig(db *gorm.DB) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
log.Println("[>1.0.2] Merge OidcProvider table....")
|
||||
logger.InfoArgs("migration", "[>1.0.2] Merge OidcProvider table....")
|
||||
var oldData struct {
|
||||
OAuthClientID string `gorm:"column:o_auth_client_id"`
|
||||
OAuthClientSecret string `gorm:"column:o_auth_client_secret"`
|
||||
@@ -207,7 +225,7 @@ func migrateLegacyMessageSenderConfig(db *gorm.DB) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
log.Println("[>1.0.2] Migrate MessageSender configuration....")
|
||||
logger.InfoArgs("migration", "[>1.0.2] Migrate MessageSender configuration....")
|
||||
var oldData struct {
|
||||
TelegramBotToken string `gorm:"column:telegram_bot_token"`
|
||||
TelegramChatID string `gorm:"column:telegram_chat_id"`
|
||||
@@ -291,7 +309,7 @@ func saveLegacyMessageSenderConfig(db *gorm.DB, name string, config map[string]i
|
||||
}
|
||||
|
||||
func migrateLegacyConfigToItems(db *gorm.DB) error {
|
||||
log.Println("[>1.1.4] Moving legacy config data...")
|
||||
logger.InfoArgs("migration", "[>1.1.4] Moving legacy config data...")
|
||||
|
||||
var oldData legacyConfig
|
||||
if err := db.Order("id desc").First(&oldData).Error; err != nil {
|
||||
@@ -354,7 +372,7 @@ func migrateLegacyClientInfo(db *gorm.DB) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
log.Println("[>0.0.5] Legacy ClientInfo table detected, starting data migration...")
|
||||
logger.InfoArgs("migration", "[>0.0.5] Legacy ClientInfo table detected, starting data migration...")
|
||||
if err := db.AutoMigrate(&models.Client{}); err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -367,7 +385,7 @@ func migrateLegacyClientInfo(db *gorm.DB) error {
|
||||
for _, info := range clientInfos {
|
||||
var client models.Client
|
||||
if err := db.Where("uuid = ?", info.UUID).First(&client).Error; err != nil {
|
||||
log.Printf("Could not find Client record with UUID %s: %v", info.UUID, err)
|
||||
logger.Errorf("migration", "Could not find Client record with UUID %s: %v", info.UUID, err)
|
||||
continue
|
||||
}
|
||||
|
||||
@@ -390,7 +408,7 @@ func migrateLegacyClientInfo(db *gorm.DB) error {
|
||||
client.Weight = info.Weight
|
||||
client.Price = info.Price
|
||||
client.BillingCycle = info.BillingCycle
|
||||
client.ExpiredAt = models.FromTime(info.ExpiredAt)
|
||||
client.ExpiredAt = info.ExpiredAt
|
||||
if err := db.Save(&client).Error; err != nil {
|
||||
return fmt.Errorf("update Client record %s: %w", info.UUID, err)
|
||||
}
|
||||
@@ -399,7 +417,7 @@ func migrateLegacyClientInfo(db *gorm.DB) error {
|
||||
if err := db.Migrator().RenameTable("client_infos", "client_infos_backup"); err != nil {
|
||||
return fmt.Errorf("backup legacy ClientInfo table: %w", err)
|
||||
}
|
||||
log.Println("Data migration completed, old table has been backed up as client_infos_backup")
|
||||
logger.InfoArgs("migration", "Data migration completed, old table has been backed up as client_infos_backup")
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -7,7 +7,7 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/komari-monitor/komari/database/models"
|
||||
appconfig "github.com/komari-monitor/komari/pkg/config"
|
||||
appconfig "github.com/komari-monitor/komari/internal/config"
|
||||
"gorm.io/driver/sqlite"
|
||||
"gorm.io/gorm"
|
||||
)
|
||||
@@ -79,6 +79,60 @@ func TestRunSkipsLegacyConfigMigrationForCurrentConfigItemTable(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunRemovesDeprecatedMetricRetentionConfig(t *testing.T) {
|
||||
db := openTestDB(t, "migrations_remove_metric_retention")
|
||||
if err := db.AutoMigrate(&appconfig.ConfigItem{}); err != nil {
|
||||
t.Fatalf("migrate config item table: %v", err)
|
||||
}
|
||||
if err := db.Create(&appconfig.ConfigItem{Key: "metric_retention_days", Value: "90"}).Error; err != nil {
|
||||
t.Fatalf("seed deprecated config: %v", err)
|
||||
}
|
||||
if err := Run(Context{DB: db}); err != nil {
|
||||
t.Fatalf("run migrations: %v", err)
|
||||
}
|
||||
var count int64
|
||||
if err := db.Model(&appconfig.ConfigItem{}).Where("key = ?", "metric_retention_days").Count(&count).Error; err != nil {
|
||||
t.Fatalf("count deprecated config: %v", err)
|
||||
}
|
||||
if count != 0 {
|
||||
t.Fatalf("deprecated metric retention config was not removed: %d", count)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunRemovesCompatibilityConfig(t *testing.T) {
|
||||
db := openTestDB(t, "migrations_remove_compatibility_config")
|
||||
if err := db.AutoMigrate(&appconfig.ConfigItem{}); err != nil {
|
||||
t.Fatalf("migrate config item table: %v", err)
|
||||
}
|
||||
for _, key := range []string{"nezha_compat_enabled", "nezha_compat_listen", "low_resource_mode"} {
|
||||
if err := db.Create(&appconfig.ConfigItem{Key: key, Value: "true"}).Error; err != nil {
|
||||
t.Fatalf("seed removed config %q: %v", key, err)
|
||||
}
|
||||
}
|
||||
if err := db.Create(&appconfig.ConfigItem{Key: "sitename", Value: `"Komari"`}).Error; err != nil {
|
||||
t.Fatalf("seed retained config: %v", err)
|
||||
}
|
||||
|
||||
if err := Run(Context{DB: db}); err != nil {
|
||||
t.Fatalf("run migrations: %v", err)
|
||||
}
|
||||
|
||||
var count int64
|
||||
if err := db.Model(&appconfig.ConfigItem{}).Where("key IN ?", []string{
|
||||
"nezha_compat_enabled",
|
||||
"nezha_compat_listen",
|
||||
"low_resource_mode",
|
||||
}).Count(&count).Error; err != nil {
|
||||
t.Fatalf("count removed config: %v", err)
|
||||
}
|
||||
if count != 0 {
|
||||
t.Fatalf("removed compatibility config remains: %d", count)
|
||||
}
|
||||
if err := db.First(&appconfig.ConfigItem{}, "key = ?", "sitename").Error; err != nil {
|
||||
t.Fatalf("unrelated config was removed: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunPreservesVersion120RuntimeShape(t *testing.T) {
|
||||
db := openTestDB(t, "migrations_v120_runtime_shape")
|
||||
if err := db.AutoMigrate(
|
||||
@@ -90,7 +144,7 @@ func TestRunPreservesVersion120RuntimeShape(t *testing.T) {
|
||||
); err != nil {
|
||||
t.Fatalf("migrate 1.2.0 runtime shape: %v", err)
|
||||
}
|
||||
now := models.FromTime(time.Now())
|
||||
now := time.Now().UTC()
|
||||
if err := db.Create(&models.Client{UUID: "client-a", Token: "token-a", CreatedAt: now, UpdatedAt: now}).Error; err != nil {
|
||||
t.Fatalf("seed client: %v", err)
|
||||
}
|
||||
@@ -169,9 +223,6 @@ func TestRunMigratesLegacyConfigTableToConfigItems(t *testing.T) {
|
||||
OAuthProvider: "github",
|
||||
NotificationMethod: "none",
|
||||
TrafficLimitPercentage: 66.5,
|
||||
RecordEnabled: true,
|
||||
RecordPreserveTime: 48,
|
||||
PingRecordPreserveTime: 12,
|
||||
ExpireNotificationLeadDays: 3,
|
||||
}
|
||||
if err := db.Create(&legacy).Error; err != nil {
|
||||
@@ -218,7 +269,7 @@ func TestRunExpandsLegacyPingAllClientsTasks(t *testing.T) {
|
||||
`).Error; err != nil {
|
||||
t.Fatalf("create legacy ping_tasks: %v", err)
|
||||
}
|
||||
now := models.FromTime(time.Now())
|
||||
now := time.Now().UTC()
|
||||
clients := []models.Client{
|
||||
{UUID: "client-a", Token: "token-a", CreatedAt: now, UpdatedAt: now},
|
||||
{UUID: "client-b", Token: "token-b", CreatedAt: now, UpdatedAt: now},
|
||||
@@ -0,0 +1,242 @@
|
||||
package migrations
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"fmt"
|
||||
logger "github.com/komari-monitor/komari/utils/log"
|
||||
"math"
|
||||
"os"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
appconfig "github.com/komari-monitor/komari/internal/config"
|
||||
"gorm.io/gorm"
|
||||
"gorm.io/gorm/clause"
|
||||
)
|
||||
|
||||
const timestampUTCMigrationKey = "internal_timestamp_utc_migrated"
|
||||
|
||||
type timestampColumn struct {
|
||||
table string
|
||||
column string
|
||||
}
|
||||
|
||||
var legacyTimestampColumns = [...]timestampColumn{
|
||||
{table: "clients", column: "expired_at"},
|
||||
{table: "clients", column: "created_at"},
|
||||
{table: "clients", column: "updated_at"},
|
||||
{table: "client_infos", column: "expired_at"},
|
||||
{table: "client_infos", column: "created_at"},
|
||||
{table: "client_infos", column: "updated_at"},
|
||||
{table: "users", column: "created_at"},
|
||||
{table: "users", column: "updated_at"},
|
||||
{table: "sessions", column: "latest_online"},
|
||||
{table: "sessions", column: "expires"},
|
||||
{table: "sessions", column: "created_at"},
|
||||
{table: "logs", column: "time"},
|
||||
{table: "clipboards", column: "created_at"},
|
||||
{table: "clipboards", column: "updated_at"},
|
||||
{table: "offline_notifications", column: "last_notified"},
|
||||
{table: "load_notifications", column: "last_notified"},
|
||||
{table: "task_results", column: "finished_at"},
|
||||
{table: "task_results", column: "created_at"},
|
||||
{table: "records", column: "time"},
|
||||
{table: "records_long_term", column: "time"},
|
||||
{table: "gpu_records", column: "time"},
|
||||
{table: "ping_records", column: "time"},
|
||||
{table: "configs", column: "created_at"},
|
||||
{table: "configs", column: "updated_at"},
|
||||
}
|
||||
|
||||
var legacyTimestampLayouts = [...]string{
|
||||
"2006-01-02 15:04:05.999999999-07:00",
|
||||
"2006-01-02 15:04:05-07:00",
|
||||
"2006-01-02 15:04:05.999999999",
|
||||
"2006-01-02 15:04:05",
|
||||
"2006-01-02T15:04:05.999999999",
|
||||
"2006-01-02T15:04:05",
|
||||
"2006-01-02",
|
||||
}
|
||||
|
||||
type timestampRow struct {
|
||||
rowID int64
|
||||
value sql.NullString
|
||||
}
|
||||
|
||||
// migrateLegacyTimestampColumns makes old offset-free SQLite values
|
||||
// unambiguous before any current model scans them as time.Time.
|
||||
func migrateLegacyTimestampColumns(db *gorm.DB) error {
|
||||
if timestampMigrationDone(db) {
|
||||
return nil
|
||||
}
|
||||
|
||||
location := legacyTimestampLocation()
|
||||
var converted int64
|
||||
err := db.Transaction(func(tx *gorm.DB) error {
|
||||
for _, target := range legacyTimestampColumns {
|
||||
if !tx.Migrator().HasTable(target.table) || !tx.Migrator().HasColumn(target.table, target.column) {
|
||||
continue
|
||||
}
|
||||
count, err := migrateTimestampColumn(tx, target, location)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
converted += count
|
||||
}
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if converted > 0 {
|
||||
logger.Infof("migration", "Converted %d legacy timestamp values to explicit UTC", converted)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func migrateTimestampColumn(db *gorm.DB, target timestampColumn, location *time.Location) (int64, error) {
|
||||
table := quoteSQLiteIdentifier(target.table)
|
||||
column := quoteSQLiteIdentifier(target.column)
|
||||
var converted int64
|
||||
var lastRowID int64
|
||||
|
||||
for {
|
||||
query := fmt.Sprintf(
|
||||
"SELECT rowid, CAST(%s AS TEXT) FROM %s WHERE rowid > ? AND %s IS NOT NULL ORDER BY rowid LIMIT 1000",
|
||||
column, table, column,
|
||||
)
|
||||
rows, err := db.Raw(query, lastRowID).Rows()
|
||||
if err != nil {
|
||||
return converted, fmt.Errorf("read legacy timestamp %s.%s: %w", target.table, target.column, err)
|
||||
}
|
||||
|
||||
batch := make([]timestampRow, 0, 1000)
|
||||
for rows.Next() {
|
||||
var row timestampRow
|
||||
if err := rows.Scan(&row.rowID, &row.value); err != nil {
|
||||
rows.Close()
|
||||
return converted, fmt.Errorf("scan legacy timestamp %s.%s: %w", target.table, target.column, err)
|
||||
}
|
||||
batch = append(batch, row)
|
||||
lastRowID = row.rowID
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
rows.Close()
|
||||
return converted, fmt.Errorf("iterate legacy timestamp %s.%s: %w", target.table, target.column, err)
|
||||
}
|
||||
rows.Close()
|
||||
if len(batch) == 0 {
|
||||
break
|
||||
}
|
||||
|
||||
for _, row := range batch {
|
||||
raw := strings.TrimSpace(row.value.String)
|
||||
if !row.value.Valid || raw == "" {
|
||||
continue
|
||||
}
|
||||
stamp, err := parseLegacyTimestamp(raw, location)
|
||||
if err != nil {
|
||||
return converted, fmt.Errorf("convert legacy timestamp %s.%s rowid=%d: %w", target.table, target.column, row.rowID, err)
|
||||
}
|
||||
update := fmt.Sprintf("UPDATE %s SET %s = ? WHERE rowid = ?", table, column)
|
||||
if err := db.Exec(update, stamp.UTC(), row.rowID).Error; err != nil {
|
||||
return converted, fmt.Errorf("write UTC timestamp %s.%s rowid=%d: %w", target.table, target.column, row.rowID, err)
|
||||
}
|
||||
converted++
|
||||
}
|
||||
if len(batch) < 1000 {
|
||||
break
|
||||
}
|
||||
}
|
||||
return converted, nil
|
||||
}
|
||||
|
||||
func parseLegacyTimestamp(value string, location *time.Location) (time.Time, error) {
|
||||
raw := strings.TrimSpace(value)
|
||||
if raw == "" {
|
||||
return time.Time{}, fmt.Errorf("timestamp is empty")
|
||||
}
|
||||
if location == nil {
|
||||
location = time.UTC
|
||||
}
|
||||
if stamp, err := time.Parse(time.RFC3339Nano, raw); err == nil {
|
||||
return stamp.UTC(), nil
|
||||
}
|
||||
for _, layout := range legacyTimestampLayouts {
|
||||
if stamp, err := time.ParseInLocation(layout, raw, location); err == nil {
|
||||
return stamp.UTC(), nil
|
||||
}
|
||||
}
|
||||
epoch, err := strconv.ParseInt(raw, 10, 64)
|
||||
if err != nil {
|
||||
return time.Time{}, fmt.Errorf("unsupported legacy timestamp %q", value)
|
||||
}
|
||||
return legacyEpochTime(epoch), nil
|
||||
}
|
||||
|
||||
func legacyEpochTime(value int64) time.Time {
|
||||
abs := value
|
||||
if abs < 0 {
|
||||
if abs == math.MinInt64 {
|
||||
return time.Unix(0, value).UTC()
|
||||
}
|
||||
abs = -abs
|
||||
}
|
||||
switch {
|
||||
case abs >= 1e17:
|
||||
return time.Unix(0, value).UTC()
|
||||
case abs >= 1e14:
|
||||
return time.Unix(0, value*int64(time.Microsecond)).UTC()
|
||||
case abs >= 1e11:
|
||||
return time.UnixMilli(value).UTC()
|
||||
default:
|
||||
return time.Unix(value, 0).UTC()
|
||||
}
|
||||
}
|
||||
|
||||
// legacyTimestampLocation is intentionally migration-only. The old custom
|
||||
// type used TZ and defaulted to UTC; current runtime code uses system time.Local.
|
||||
func legacyTimestampLocation() *time.Location {
|
||||
name := strings.TrimSpace(os.Getenv("TZ"))
|
||||
if name == "" {
|
||||
return time.UTC
|
||||
}
|
||||
location, err := time.LoadLocation(name)
|
||||
if err != nil {
|
||||
logger.Infof("migration", "Legacy timezone %q cannot be loaded; interpreting old timestamps as UTC: %v", name, err)
|
||||
return time.UTC
|
||||
}
|
||||
return location
|
||||
}
|
||||
|
||||
func timestampMigrationDone(db *gorm.DB) bool {
|
||||
if !db.Migrator().HasTable(&appconfig.ConfigItem{}) || hasLegacyConfigTable(db) {
|
||||
return false
|
||||
}
|
||||
var item appconfig.ConfigItem
|
||||
if err := db.Where("key = ?", timestampUTCMigrationKey).First(&item).Error; err != nil {
|
||||
return false
|
||||
}
|
||||
return item.Value == "true"
|
||||
}
|
||||
|
||||
func markTimestampMigrationDone(db *gorm.DB) error {
|
||||
if hasLegacyConfigTable(db) {
|
||||
return fmt.Errorf("new config item table is unavailable")
|
||||
}
|
||||
if !db.Migrator().HasTable(&appconfig.ConfigItem{}) {
|
||||
if err := db.AutoMigrate(&appconfig.ConfigItem{}); err != nil {
|
||||
return fmt.Errorf("create config item table: %w", err)
|
||||
}
|
||||
}
|
||||
item := appconfig.ConfigItem{Key: timestampUTCMigrationKey, Value: "true"}
|
||||
return db.Clauses(clause.OnConflict{
|
||||
Columns: []clause.Column{{Name: "key"}},
|
||||
DoUpdates: clause.AssignmentColumns([]string{"value"}),
|
||||
}).Create(&item).Error
|
||||
}
|
||||
|
||||
func quoteSQLiteIdentifier(value string) string {
|
||||
return `"` + strings.ReplaceAll(value, `"`, `""`) + `"`
|
||||
}
|
||||
@@ -0,0 +1,99 @@
|
||||
package migrations
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"gorm.io/driver/sqlite"
|
||||
"gorm.io/gorm"
|
||||
"gorm.io/gorm/logger"
|
||||
)
|
||||
|
||||
func TestLegacyTimestampMigrationDefaultsToUTC(t *testing.T) {
|
||||
t.Setenv("TZ", "")
|
||||
db := openTimestampMigrationDB(t)
|
||||
insertLegacyLogTime(t, db, "2026-07-17 09:30:00.1234567")
|
||||
|
||||
if err := migrateLegacyTimestampColumns(db); err != nil {
|
||||
t.Fatalf("migrate timestamps: %v", err)
|
||||
}
|
||||
want := time.Date(2026, 7, 17, 9, 30, 0, 123456700, time.UTC)
|
||||
assertMigratedLogTime(t, db, want)
|
||||
}
|
||||
|
||||
func TestLegacyTimestampMigrationUsesLegacyTZ(t *testing.T) {
|
||||
t.Setenv("TZ", "Asia/Shanghai")
|
||||
db := openTimestampMigrationDB(t)
|
||||
insertLegacyLogTime(t, db, "2026-07-17 09:30:00.123456789")
|
||||
|
||||
if err := migrateLegacyTimestampColumns(db); err != nil {
|
||||
t.Fatalf("migrate timestamps: %v", err)
|
||||
}
|
||||
want := time.Date(2026, 7, 17, 1, 30, 0, 123456789, time.UTC)
|
||||
assertMigratedLogTime(t, db, want)
|
||||
}
|
||||
|
||||
func TestParseLegacyTimestampAcceptsHistoricalEpochUnits(t *testing.T) {
|
||||
stamp := time.Date(2026, 7, 17, 1, 30, 0, 123456789, time.UTC)
|
||||
tests := map[string]struct {
|
||||
value string
|
||||
want time.Time
|
||||
}{
|
||||
"seconds": {value: fmt.Sprint(stamp.Unix()), want: stamp.Truncate(time.Second)},
|
||||
"milliseconds": {value: fmt.Sprint(stamp.UnixMilli()), want: stamp.Truncate(time.Millisecond)},
|
||||
"microseconds": {value: fmt.Sprint(stamp.UnixMicro()), want: stamp.Truncate(time.Microsecond)},
|
||||
"nanoseconds": {value: fmt.Sprint(stamp.UnixNano()), want: stamp},
|
||||
}
|
||||
for name, test := range tests {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
got, err := parseLegacyTimestamp(test.value, time.UTC)
|
||||
if err != nil {
|
||||
t.Fatalf("parse legacy timestamp: %v", err)
|
||||
}
|
||||
if !got.Equal(test.want) {
|
||||
t.Fatalf("parsed timestamp = %s, want %s", got, test.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func openTimestampMigrationDB(t *testing.T) *gorm.DB {
|
||||
t.Helper()
|
||||
dsn := fmt.Sprintf("file:%s?mode=memory&cache=shared", strings.ReplaceAll(t.Name(), "/", "_"))
|
||||
db, err := gorm.Open(sqlite.Open(dsn), &gorm.Config{Logger: logger.Default.LogMode(logger.Silent)})
|
||||
if err != nil {
|
||||
t.Fatalf("open database: %v", err)
|
||||
}
|
||||
if err := db.Exec(`CREATE TABLE logs (id INTEGER PRIMARY KEY, time TIMESTAMP)`).Error; err != nil {
|
||||
t.Fatalf("create logs table: %v", err)
|
||||
}
|
||||
return db
|
||||
}
|
||||
|
||||
func insertLegacyLogTime(t *testing.T, db *gorm.DB, value string) {
|
||||
t.Helper()
|
||||
if err := db.Exec(`INSERT INTO logs (time) VALUES (?)`, value).Error; err != nil {
|
||||
t.Fatalf("insert legacy timestamp: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func assertMigratedLogTime(t *testing.T, db *gorm.DB, want time.Time) {
|
||||
t.Helper()
|
||||
var got time.Time
|
||||
if err := db.Raw(`SELECT time FROM logs LIMIT 1`).Scan(&got).Error; err != nil {
|
||||
t.Fatalf("read migrated timestamp: %v", err)
|
||||
}
|
||||
if !got.Equal(want) || got.Nanosecond() != want.Nanosecond() {
|
||||
t.Fatalf("migrated time = %s, want %s", got, want)
|
||||
}
|
||||
|
||||
var raw string
|
||||
if err := db.Raw(`SELECT CAST(time AS TEXT) FROM logs LIMIT 1`).Scan(&raw).Error; err != nil {
|
||||
t.Fatalf("read raw migrated timestamp: %v", err)
|
||||
}
|
||||
if !strings.HasSuffix(raw, "+00:00") {
|
||||
t.Fatalf("stored timestamp %q has no explicit UTC offset", raw)
|
||||
}
|
||||
}
|
||||
@@ -1,9 +1,9 @@
|
||||
package corn
|
||||
package scheduler
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"log"
|
||||
logger "github.com/komari-monitor/komari/utils/log"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
@@ -40,7 +40,7 @@ type cronSchedule struct {
|
||||
}
|
||||
|
||||
func (s cronSchedule) Next(t time.Time) time.Time {
|
||||
next := t.Truncate(time.Second).Add(time.Second)
|
||||
next := t.UTC().Truncate(time.Second).Add(time.Second)
|
||||
limit := next.Add(366 * 24 * time.Hour)
|
||||
for next.Before(limit) {
|
||||
if s.match(next) {
|
||||
@@ -52,6 +52,7 @@ func (s cronSchedule) Next(t time.Time) time.Time {
|
||||
}
|
||||
|
||||
func (s cronSchedule) match(t time.Time) bool {
|
||||
t = t.In(time.Local)
|
||||
_, okSecond := s.seconds[t.Second()]
|
||||
_, okMinute := s.minutes[t.Minute()]
|
||||
_, okHour := s.hours[t.Hour()]
|
||||
@@ -171,7 +172,7 @@ func (m *Manager) run(ctx context.Context, name string, s schedule, runImmediate
|
||||
|
||||
nextTick := s.Next(time.Now())
|
||||
if nextTick.IsZero() {
|
||||
log.Printf("corn job %s has no next run time", name)
|
||||
logger.Warnf("scheduler", "corn job %s has no next run time", name)
|
||||
return
|
||||
}
|
||||
timer := time.NewTimer(time.Until(nextTick))
|
||||
@@ -202,7 +203,7 @@ func resetTimer(timer *time.Timer, duration time.Duration) {
|
||||
func safeRun(ctx context.Context, name string, fn Func) {
|
||||
defer func() {
|
||||
if r := recover(); r != nil {
|
||||
log.Printf("corn job %s panic: %v", name, r)
|
||||
logger.Errorf("scheduler", "corn job %s panic: %v", name, r)
|
||||
}
|
||||
}()
|
||||
|
||||
@@ -0,0 +1,35 @@
|
||||
package scheduler
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestCronScheduleUsesSystemLocalWallClock(t *testing.T) {
|
||||
originalLocal := time.Local
|
||||
time.Local = time.FixedZone("UTC+8", 8*60*60)
|
||||
t.Cleanup(func() { time.Local = originalLocal })
|
||||
|
||||
schedule, err := Parse("0 0 9 * * *")
|
||||
if err != nil {
|
||||
t.Fatalf("parse schedule: %v", err)
|
||||
}
|
||||
after := time.Date(2026, 7, 17, 0, 30, 0, 0, time.UTC)
|
||||
want := time.Date(2026, 7, 17, 1, 0, 0, 0, time.UTC)
|
||||
if got := schedule.Next(after); !got.Equal(want) {
|
||||
t.Fatalf("next run = %s, want %s", got, want)
|
||||
} else if got.Location() != time.UTC {
|
||||
t.Fatalf("next run location = %s, want UTC", got.Location())
|
||||
}
|
||||
}
|
||||
|
||||
func TestEverySchedulePreservesElapsedDuration(t *testing.T) {
|
||||
schedule, err := Parse("@every 90s")
|
||||
if err != nil {
|
||||
t.Fatalf("parse schedule: %v", err)
|
||||
}
|
||||
after := time.Now()
|
||||
if got := schedule.Next(after); got.Sub(after) != 90*time.Second {
|
||||
t.Fatalf("interval = %s, want 90s", got.Sub(after))
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,47 @@
|
||||
package server
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net/http"
|
||||
|
||||
"github.com/gin-gonic/gin"
|
||||
"github.com/komari-monitor/komari/internal/config"
|
||||
)
|
||||
|
||||
// cleanupFunc is a cleanup action run during shutdown.
|
||||
type cleanupFunc struct {
|
||||
name string
|
||||
fn func(ctx context.Context) error
|
||||
}
|
||||
|
||||
// App owns the server lifecycle. Individual phases live in focused files:
|
||||
// bootstrap, metric store, providers, guides, router, and runtime.
|
||||
type App struct {
|
||||
listenAddr string
|
||||
settings *config.Settings
|
||||
engine *gin.Engine
|
||||
server *http.Server
|
||||
reload *ReloadManager
|
||||
dbReady bool
|
||||
oauthReady bool
|
||||
metricStoreCleanupAdded bool
|
||||
|
||||
cleanups []cleanupFunc
|
||||
}
|
||||
|
||||
// Options configures the process-wide application runtime.
|
||||
type Options struct {
|
||||
ListenAddr string
|
||||
}
|
||||
|
||||
// New constructs an empty application. Initialization happens in the
|
||||
// explicit lifecycle phases called by the command entrypoint.
|
||||
func New(options Options) *App {
|
||||
return &App{listenAddr: options.ListenAddr, reload: NewReloadManager()}
|
||||
}
|
||||
|
||||
// addCleanup registers a cleanup action. Shutdown executes actions in LIFO
|
||||
// order, mirroring resource initialization order.
|
||||
func (a *App) addCleanup(name string, fn func(ctx context.Context) error) {
|
||||
a.cleanups = append(a.cleanups, cleanupFunc{name: name, fn: fn})
|
||||
}
|
||||
@@ -0,0 +1,82 @@
|
||||
package server
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
|
||||
"github.com/gin-gonic/gin"
|
||||
)
|
||||
|
||||
func TestRetryMetricStoreConnectionStopsAfterRecovery(t *testing.T) {
|
||||
wantErr := errors.New("temporary connection failure")
|
||||
attempts := 0
|
||||
err := retryMetricStoreConnection(3, 0, func() error {
|
||||
attempts++
|
||||
if attempts < 3 {
|
||||
return wantErr
|
||||
}
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("retry returned error after recovery: %v", err)
|
||||
}
|
||||
if attempts != 3 {
|
||||
t.Fatalf("attempts = %d, want 3", attempts)
|
||||
}
|
||||
}
|
||||
|
||||
func TestGuideNoRouteRejectsUnknownAPIAndRedirectsPages(t *testing.T) {
|
||||
gin.SetMode(gin.TestMode)
|
||||
router := gin.New()
|
||||
router.Use(noStoreAPIResponses())
|
||||
router.NoRoute(guideNoRoute("/install", "Not found in install mode", nil))
|
||||
|
||||
apiResponse := httptest.NewRecorder()
|
||||
router.ServeHTTP(apiResponse, httptest.NewRequest(http.MethodGet, "/api/unknown", nil))
|
||||
if apiResponse.Code != http.StatusNotFound {
|
||||
t.Fatalf("API status = %d, want %d", apiResponse.Code, http.StatusNotFound)
|
||||
}
|
||||
if got := apiResponse.Header().Get("Cache-Control"); got != "no-store" {
|
||||
t.Fatalf("API cache control = %q, want no-store", got)
|
||||
}
|
||||
|
||||
pageResponse := httptest.NewRecorder()
|
||||
router.ServeHTTP(pageResponse, httptest.NewRequest(http.MethodGet, "/stale-guide", nil))
|
||||
if pageResponse.Code != http.StatusTemporaryRedirect {
|
||||
t.Fatalf("page status = %d, want %d", pageResponse.Code, http.StatusTemporaryRedirect)
|
||||
}
|
||||
if got := pageResponse.Header().Get("Location"); got != "/install" {
|
||||
t.Fatalf("redirect location = %q, want /install", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRetryMetricStoreConnectionReturnsLastError(t *testing.T) {
|
||||
wantErr := errors.New("connection unavailable")
|
||||
attempts := 0
|
||||
err := retryMetricStoreConnection(3, 0, func() error {
|
||||
attempts++
|
||||
return wantErr
|
||||
})
|
||||
if !errors.Is(err, wantErr) {
|
||||
t.Fatalf("error = %v, want %v", err, wantErr)
|
||||
}
|
||||
if attempts != 3 {
|
||||
t.Fatalf("attempts = %d, want 3", attempts)
|
||||
}
|
||||
}
|
||||
|
||||
func TestShutdownReturnsCleanupError(t *testing.T) {
|
||||
wantErr := errors.New("queued reports were not flushed")
|
||||
app := New(Options{})
|
||||
app.addCleanup("metric-report-batcher", func(_ context.Context) error {
|
||||
return wantErr
|
||||
})
|
||||
|
||||
err := app.Shutdown()
|
||||
if !errors.Is(err, wantErr) {
|
||||
t.Fatalf("shutdown error = %v, want %v", err, wantErr)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,34 @@
|
||||
package server
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
|
||||
"github.com/gin-gonic/gin"
|
||||
"github.com/komari-monitor/komari/database/dbcore"
|
||||
"github.com/komari-monitor/komari/internal/config"
|
||||
"github.com/komari-monitor/komari/utils"
|
||||
)
|
||||
|
||||
// Bootstrap initializes the data directory, primary database, and settings.
|
||||
func (a *App) Bootstrap() error {
|
||||
if err := os.MkdirAll("./data/theme", os.ModePerm); err != nil {
|
||||
return fmt.Errorf("failed to create theme directory: %w", err)
|
||||
}
|
||||
|
||||
dbcore.SetVersionID(utils.CurrentVersion + "-" + utils.VersionHash)
|
||||
if err := dbcore.Initialize(); err != nil {
|
||||
return fmt.Errorf("failed to initialize database: %w", err)
|
||||
}
|
||||
a.dbReady = true
|
||||
a.addCleanup("database", func(context.Context) error { return dbcore.Close() })
|
||||
|
||||
gin.SetMode(gin.ReleaseMode)
|
||||
settings, err := config.GetManyAs[config.Settings]()
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to load settings: %w", err)
|
||||
}
|
||||
a.settings = settings
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,118 @@
|
||||
package server
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"os"
|
||||
"os/signal"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
"github.com/gin-gonic/gin"
|
||||
logger "github.com/komari-monitor/komari/utils/log"
|
||||
"github.com/komari-monitor/komari/web/api"
|
||||
frontendpublic "github.com/komari-monitor/komari/web/public"
|
||||
"github.com/komari-monitor/komari/web/security"
|
||||
)
|
||||
|
||||
type guideController interface {
|
||||
Activate()
|
||||
Deactivate()
|
||||
Register(*gin.Engine)
|
||||
Done() <-chan struct{}
|
||||
}
|
||||
|
||||
type guideServerConfig struct {
|
||||
pagePath string
|
||||
missingAPI string
|
||||
logMessage string
|
||||
requireIdentity bool
|
||||
restrictedStatic bool
|
||||
}
|
||||
|
||||
// runGuideServer hosts one temporary, narrowly scoped guide. It deliberately
|
||||
// constructs a new Gin engine for each mode so normal routes never leak into
|
||||
// installation, upgrade, or recovery flows.
|
||||
func (a *App) runGuideServer(controller guideController, cfg guideServerConfig) (bool, error) {
|
||||
controller.Activate()
|
||||
defer controller.Deactivate()
|
||||
|
||||
r := gin.New()
|
||||
r.Use(logger.GinLogger(), logger.GinRecovery(), noStoreAPIResponses())
|
||||
if cfg.requireIdentity {
|
||||
cors := security.NewCorsController(a.settings.CorsOriginCheckEnabled, a.settings.CorsAllowedOrigins)
|
||||
r.Use(cors.Middleware(), api.IdentityMiddleware())
|
||||
}
|
||||
controller.Register(r)
|
||||
|
||||
serveStatic := frontendpublic.Static
|
||||
if cfg.restrictedStatic {
|
||||
serveStatic = frontendpublic.StaticRestricted
|
||||
}
|
||||
serveStatic(r.Group("/"), func(handlers ...gin.HandlerFunc) {
|
||||
r.NoRoute(guideNoRoute(cfg.pagePath, cfg.missingAPI, handlers))
|
||||
})
|
||||
|
||||
server := &http.Server{Addr: a.listenAddr, Handler: r}
|
||||
a.engine = r
|
||||
a.server = server
|
||||
serverErr := make(chan error, 1)
|
||||
logger.Infof("server", cfg.logMessage, a.listenAddr)
|
||||
go func() {
|
||||
if err := server.ListenAndServe(); err != nil && err != http.ErrServerClosed {
|
||||
serverErr <- err
|
||||
}
|
||||
}()
|
||||
|
||||
quit := make(chan os.Signal, 1)
|
||||
signal.Notify(quit, os.Interrupt, syscall.SIGTERM)
|
||||
defer signal.Stop(quit)
|
||||
|
||||
select {
|
||||
case err := <-serverErr:
|
||||
return false, fmt.Errorf("listen in guide mode: %w", err)
|
||||
case <-quit:
|
||||
return false, a.Shutdown()
|
||||
case <-controller.Done():
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||
defer cancel()
|
||||
if err := server.Shutdown(ctx); err != nil {
|
||||
return false, fmt.Errorf("stop guide server: %w", err)
|
||||
}
|
||||
a.server = nil
|
||||
a.engine = nil
|
||||
return true, nil
|
||||
}
|
||||
}
|
||||
|
||||
func noStoreAPIResponses() gin.HandlerFunc {
|
||||
return func(c *gin.Context) {
|
||||
if strings.HasPrefix(c.Request.URL.Path, "/api") {
|
||||
c.Header("Cache-Control", "no-store")
|
||||
}
|
||||
c.Next()
|
||||
}
|
||||
}
|
||||
|
||||
func guideNoRoute(pagePath, missingAPI string, handlers []gin.HandlerFunc) gin.HandlerFunc {
|
||||
return func(c *gin.Context) {
|
||||
requestPath := c.Request.URL.Path
|
||||
if strings.HasPrefix(requestPath, "/api") {
|
||||
api.RespondError(c, http.StatusNotFound, missingAPI)
|
||||
return
|
||||
}
|
||||
if c.Request.Method == http.MethodGet && requestPath != pagePath && filepath.Ext(requestPath) == "" {
|
||||
c.Redirect(http.StatusTemporaryRedirect, pagePath)
|
||||
return
|
||||
}
|
||||
for _, handler := range handlers {
|
||||
handler(c)
|
||||
if c.IsAborted() {
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,96 @@
|
||||
package server
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"github.com/komari-monitor/komari/database/dbcore"
|
||||
"github.com/komari-monitor/komari/database/metricstore"
|
||||
"github.com/komari-monitor/komari/database/models"
|
||||
"github.com/komari-monitor/komari/internal/migrations"
|
||||
installweb "github.com/komari-monitor/komari/web/install"
|
||||
migrationweb "github.com/komari-monitor/komari/web/migration"
|
||||
recoveryweb "github.com/komari-monitor/komari/web/recovery"
|
||||
)
|
||||
|
||||
// InstallRequired reports whether the instance still needs the first-run guide.
|
||||
func (a *App) InstallRequired() (bool, error) {
|
||||
var count int64
|
||||
if err := dbcore.GetDBInstance().Model(&models.User{}).Count(&count).Error; err != nil {
|
||||
return false, err
|
||||
}
|
||||
return count == 0, nil
|
||||
}
|
||||
|
||||
type DatabaseMigrationRequirement struct {
|
||||
mode migrationweb.Mode
|
||||
summary migrations.LegacyMonitoringSummary
|
||||
}
|
||||
|
||||
func (r DatabaseMigrationRequirement) Required() bool { return r.mode != "" }
|
||||
|
||||
// DatabaseMigrationRequired checks both supported migration inputs before the
|
||||
// normal Metric Store connection is initialized. Structure upgrades run first
|
||||
// when both inputs exist, then the startup loop detects the legacy tables on
|
||||
// its next pass.
|
||||
func (a *App) DatabaseMigrationRequired() (DatabaseMigrationRequirement, error) {
|
||||
structureRequired, err := metricstore.StructureUpgradeRequired(context.Background())
|
||||
if err != nil {
|
||||
return DatabaseMigrationRequirement{}, err
|
||||
}
|
||||
if structureRequired {
|
||||
return DatabaseMigrationRequirement{mode: migrationweb.ModeMetricStructure}, nil
|
||||
}
|
||||
legacyRequired, summary, err := migrations.LegacyMonitoringMigrationRequired(dbcore.GetDBInstance())
|
||||
if err != nil {
|
||||
return DatabaseMigrationRequirement{}, err
|
||||
}
|
||||
if legacyRequired {
|
||||
return DatabaseMigrationRequirement{mode: migrationweb.ModeLegacyMonitoring, summary: summary}, nil
|
||||
}
|
||||
return DatabaseMigrationRequirement{}, nil
|
||||
}
|
||||
|
||||
// RunInstallGuide exposes only first-run installation APIs. It intentionally
|
||||
// does not mount authentication or normal application routes.
|
||||
func (a *App) RunInstallGuide() (bool, error) {
|
||||
return a.runGuideServer(installweb.NewController(dbcore.GetDBInstance()), guideServerConfig{
|
||||
pagePath: installweb.PagePath,
|
||||
missingAPI: "Not found in install mode",
|
||||
logMessage: "First-run installation guide is available on %s",
|
||||
})
|
||||
}
|
||||
|
||||
// RunMetricStoreRecovery keeps login available while exposing only the
|
||||
// administrator-protected metric-store recovery API.
|
||||
func (a *App) RunMetricStoreRecovery(initialErr error) (bool, error) {
|
||||
a.initOAuth()
|
||||
return a.runGuideServer(recoveryweb.NewController(initialErr, metricStoreReconnectAttempts), guideServerConfig{
|
||||
pagePath: recoveryweb.PagePath,
|
||||
missingAPI: "Not found in database recovery mode",
|
||||
logMessage: "Metric store recovery is available on %s",
|
||||
requireIdentity: true,
|
||||
restrictedStatic: true,
|
||||
})
|
||||
}
|
||||
|
||||
// RunDatabaseMigration serves the same authenticated guide and status model
|
||||
// for either migration input while leaving each conversion engine independent.
|
||||
func (a *App) RunDatabaseMigration(requirement DatabaseMigrationRequirement) (bool, error) {
|
||||
a.initOAuth()
|
||||
var controller guideController
|
||||
switch requirement.mode {
|
||||
case migrationweb.ModeMetricStructure:
|
||||
controller = migrationweb.NewStructureController()
|
||||
case migrationweb.ModeLegacyMonitoring:
|
||||
controller = migrationweb.NewLegacyController(dbcore.GetDBInstance(), requirement.summary)
|
||||
default:
|
||||
return false, nil
|
||||
}
|
||||
return a.runGuideServer(controller, guideServerConfig{
|
||||
pagePath: migrationweb.PagePath,
|
||||
missingAPI: "Not found in database migration mode",
|
||||
logMessage: "Database migration guide is available on %s",
|
||||
requireIdentity: true,
|
||||
restrictedStatic: true,
|
||||
})
|
||||
}
|
||||
@@ -0,0 +1,90 @@
|
||||
package server
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"github.com/komari-monitor/komari/database/auditlog"
|
||||
"github.com/komari-monitor/komari/database/metricstore"
|
||||
"github.com/komari-monitor/komari/internal/config"
|
||||
logger "github.com/komari-monitor/komari/utils/log"
|
||||
)
|
||||
|
||||
const (
|
||||
metricStoreReconnectAttempts = 3
|
||||
metricStoreReconnectInterval = 5 * time.Second
|
||||
)
|
||||
|
||||
// ConnectMetricStore performs one connection attempt and registers cleanup
|
||||
// only after a store has actually opened.
|
||||
func (a *App) ConnectMetricStore() error {
|
||||
if err := metricstore.InitializeStore(); err != nil {
|
||||
return fmt.Errorf("failed to initialize metric store: %s", redactMetricStoreError(err))
|
||||
}
|
||||
if !a.metricStoreCleanupAdded {
|
||||
a.addCleanup("metric-store", metricstore.CloseStoreContext)
|
||||
a.metricStoreCleanupAdded = true
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func redactMetricStoreError(err error) string {
|
||||
if err == nil {
|
||||
return ""
|
||||
}
|
||||
dsn := ""
|
||||
if cfg, cfgErr := config.GetManyAs[metricstore.MetricStoreConfig](); cfgErr == nil {
|
||||
dsn = cfg.DSN
|
||||
}
|
||||
return metricstore.RedactConnectionError(err.Error(), dsn)
|
||||
}
|
||||
|
||||
// ConnectMetricStoreWithRetry retries the monitoring database connection.
|
||||
func (a *App) ConnectMetricStoreWithRetry() error {
|
||||
attempt := 0
|
||||
err := retryMetricStoreConnection(metricStoreReconnectAttempts, metricStoreReconnectInterval, func() error {
|
||||
attempt++
|
||||
err := a.ConnectMetricStore()
|
||||
if err != nil {
|
||||
logger.Warn("server", "Metric store connection attempt failed", "attempt", attempt, "max_attempts", metricStoreReconnectAttempts, "error", err)
|
||||
}
|
||||
return err
|
||||
})
|
||||
if err == nil && attempt > 1 {
|
||||
logger.Infof("server", "Metric store connection recovered on attempt %d/%d", attempt, metricStoreReconnectAttempts)
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
func retryMetricStoreConnection(attempts int, interval time.Duration, connect func() error) error {
|
||||
if attempts < 1 {
|
||||
return fmt.Errorf("metric store retry attempts must be positive")
|
||||
}
|
||||
var lastErr error
|
||||
for attempt := 1; attempt <= attempts; attempt++ {
|
||||
if attempt > 1 {
|
||||
time.Sleep(interval)
|
||||
}
|
||||
if err := connect(); err == nil {
|
||||
return nil
|
||||
} else {
|
||||
lastErr = err
|
||||
}
|
||||
}
|
||||
return lastErr
|
||||
}
|
||||
|
||||
// InitStores opens the metric store and starts its report batcher. Historical
|
||||
// data migrations are started only through an explicit administrator action.
|
||||
func (a *App) InitStores() error {
|
||||
if err := a.ConnectMetricStore(); err != nil {
|
||||
auditlog.EventLog("error", fmt.Sprintf("Failed to initialize metric store: %v", err))
|
||||
return err
|
||||
}
|
||||
metricstore.StartReportBatcher()
|
||||
a.addCleanup("metric-report-batcher", metricstore.StopReportBatcher)
|
||||
// A store-to-store migration holds the exclusive operation lease. Stop it
|
||||
// before flushing queued reports, which need the shared lease to write.
|
||||
a.addCleanup("metric-store-migration", metricstore.StopStoreMigrationForShutdown)
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,38 @@
|
||||
package server
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
|
||||
"github.com/komari-monitor/komari/database/auditlog"
|
||||
"github.com/komari-monitor/komari/utils/geoip"
|
||||
logger "github.com/komari-monitor/komari/utils/log"
|
||||
"github.com/komari-monitor/komari/utils/messageSender"
|
||||
"github.com/komari-monitor/komari/web/oauth"
|
||||
)
|
||||
|
||||
// InitProviders initializes providers needed by the normal application.
|
||||
func (a *App) InitProviders() error {
|
||||
a.initOAuth()
|
||||
|
||||
go geoip.InitGeoIp()
|
||||
a.addCleanup("geoip", func(context.Context) error { return geoip.Shutdown() })
|
||||
|
||||
messageSender.Initialize()
|
||||
a.addCleanup("message-sender", func(context.Context) error { return messageSender.Shutdown() })
|
||||
return nil
|
||||
}
|
||||
|
||||
// initOAuth initializes OAuth once. Restricted authenticated guides need it
|
||||
// before the normal provider initialization phase.
|
||||
func (a *App) initOAuth() {
|
||||
if a.oauthReady {
|
||||
return
|
||||
}
|
||||
if err := oauth.Initialize(); err != nil {
|
||||
logger.Errorf("server", "Failed to initialize OAuth provider: %v", err)
|
||||
auditlog.EventLog("error", fmt.Sprintf("Failed to initialize OAuth provider: %v", err))
|
||||
}
|
||||
a.oauthReady = true
|
||||
a.addCleanup("oauth", func(context.Context) error { return oauth.Shutdown() })
|
||||
}
|
||||
@@ -0,0 +1,67 @@
|
||||
package server
|
||||
|
||||
import (
|
||||
logger "github.com/komari-monitor/komari/utils/log"
|
||||
|
||||
"github.com/komari-monitor/komari/internal/config"
|
||||
)
|
||||
|
||||
// reloadHandler 是单个配置热重载处理器。
|
||||
//
|
||||
// name 仅用于日志定位,handler 接收配置变更事件,自行判断关心的 key 是否变化。
|
||||
// 每个 handler 相互独立:一个 panic 不应影响其它 handler。
|
||||
type reloadHandler struct {
|
||||
name string
|
||||
handler func(config.ConfigEvent)
|
||||
}
|
||||
|
||||
// ReloadManager 统一收敛所有配置热重载逻辑。
|
||||
//
|
||||
// 过去 config.Subscribe 分散在 cmd/server.go、cors.go 等多处,导致:
|
||||
// - 谁监听哪些 key 不清晰;
|
||||
// - 某个 handler panic 会连带影响同一订阅回调里的其它逻辑;
|
||||
// - 关闭时无法统一停止。
|
||||
//
|
||||
// ReloadManager 把这些 handler 注册在一处,用单个 config.Subscribe 分发,
|
||||
// 并对每个 handler 做 panic 隔离。
|
||||
type ReloadManager struct {
|
||||
handlers []reloadHandler
|
||||
started bool
|
||||
}
|
||||
|
||||
// NewReloadManager 创建一个空的热重载管理器。
|
||||
func NewReloadManager() *ReloadManager {
|
||||
return &ReloadManager{}
|
||||
}
|
||||
|
||||
// Register 注册一个命名的热重载处理器。必须在 Start 之前调用。
|
||||
func (m *ReloadManager) Register(name string, handler func(config.ConfigEvent)) {
|
||||
if handler == nil {
|
||||
return
|
||||
}
|
||||
m.handlers = append(m.handlers, reloadHandler{name: name, handler: handler})
|
||||
}
|
||||
|
||||
// Start 向 config 订阅一次,之后每个事件会分发给全部已注册 handler。
|
||||
// 重复调用无副作用。
|
||||
func (m *ReloadManager) Start() {
|
||||
if m.started {
|
||||
return
|
||||
}
|
||||
m.started = true
|
||||
config.Subscribe(func(event config.ConfigEvent) {
|
||||
for _, h := range m.handlers {
|
||||
m.dispatch(h, event)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// dispatch 执行单个 handler,并隔离 panic,避免一个 handler 影响其它。
|
||||
func (m *ReloadManager) dispatch(h reloadHandler, event config.ConfigEvent) {
|
||||
defer func() {
|
||||
if r := recover(); r != nil {
|
||||
logger.Errorf("reload", "config reload handler %q panicked: %v", h.name, r)
|
||||
}
|
||||
}()
|
||||
h.handler(event)
|
||||
}
|
||||
@@ -0,0 +1,221 @@
|
||||
package server
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"os"
|
||||
"os/signal"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
"github.com/gin-gonic/gin"
|
||||
"github.com/komari-monitor/komari/database"
|
||||
"github.com/komari-monitor/komari/database/accounts"
|
||||
"github.com/komari-monitor/komari/database/auditlog"
|
||||
"github.com/komari-monitor/komari/database/metricstore"
|
||||
d_notification "github.com/komari-monitor/komari/database/notification"
|
||||
"github.com/komari-monitor/komari/database/tasks"
|
||||
"github.com/komari-monitor/komari/internal/config"
|
||||
"github.com/komari-monitor/komari/internal/lifecycle"
|
||||
"github.com/komari-monitor/komari/internal/scheduler"
|
||||
"github.com/komari-monitor/komari/utils/geoip"
|
||||
logger "github.com/komari-monitor/komari/utils/log"
|
||||
"github.com/komari-monitor/komari/utils/messageSender"
|
||||
"github.com/komari-monitor/komari/utils/notifier"
|
||||
"github.com/komari-monitor/komari/web/api"
|
||||
"github.com/komari-monitor/komari/web/oauth"
|
||||
recoveryweb "github.com/komari-monitor/komari/web/recovery"
|
||||
"github.com/komari-monitor/komari/web/router"
|
||||
"github.com/komari-monitor/komari/web/security"
|
||||
)
|
||||
|
||||
// ErrRestartRequested is returned after a clean shutdown when a configuration
|
||||
// change requires the next startup to enter a restricted guide.
|
||||
var ErrRestartRequested = errors.New("server restart requested")
|
||||
|
||||
const (
|
||||
// Give in-flight HTTP requests time to finish before the listener closes.
|
||||
httpShutdownTimeout = 10 * time.Second
|
||||
// Keep an independent budget for report flushing and store teardown. Reusing
|
||||
// the HTTP deadline here can skip queued metric writes after a slow request.
|
||||
resourceCleanupTimeout = 30 * time.Second
|
||||
)
|
||||
|
||||
// StartBackground starts scheduled work after all stores are ready.
|
||||
func (a *App) StartBackground() error {
|
||||
registerScheduledWork()
|
||||
a.addCleanup("scheduler", func(context.Context) error {
|
||||
scheduler.StopAll()
|
||||
return nil
|
||||
})
|
||||
return nil
|
||||
}
|
||||
|
||||
func (a *App) registerReloadHandlers(cors *security.CorsController) {
|
||||
a.reload.Register("oauth-provider", func(event config.ConfigEvent) {
|
||||
if ok, providerName := config.IsChangedT[string](event, config.OAuthProviderKey); ok {
|
||||
if providerName == "" || providerName == "none" {
|
||||
providerName = "github"
|
||||
}
|
||||
oidcProvider, err := database.GetOidcConfigByName(providerName)
|
||||
if err != nil {
|
||||
logger.Errorf("server", "Failed to get OIDC provider config: %v", err)
|
||||
return
|
||||
}
|
||||
logger.Infof("server", "Using %s as OIDC provider", oidcProvider.Name)
|
||||
if err := oauth.LoadProvider(oidcProvider.Name, oidcProvider.Addition); err != nil {
|
||||
auditlog.EventLog("error", fmt.Sprintf("Failed to load OIDC provider: %v", err))
|
||||
}
|
||||
}
|
||||
})
|
||||
a.reload.Register("geoip-provider", func(event config.ConfigEvent) {
|
||||
if event.IsChanged(config.GeoIpProviderKey) {
|
||||
go geoip.InitGeoIp()
|
||||
}
|
||||
})
|
||||
a.reload.Register("message-sender", func(event config.ConfigEvent) {
|
||||
if event.IsChanged(config.NotificationMethodKey) {
|
||||
go messageSender.Initialize()
|
||||
}
|
||||
})
|
||||
a.reload.Register("cors", func(event config.ConfigEvent) { cors.Update(event) })
|
||||
}
|
||||
|
||||
// BuildRouter constructs the normal application router and starts reloads.
|
||||
func (a *App) BuildRouter() error {
|
||||
r := gin.New()
|
||||
r.Use(logger.GinLogger(), logger.GinRecovery())
|
||||
cors := security.NewCorsController(a.settings.CorsOriginCheckEnabled, a.settings.CorsAllowedOrigins)
|
||||
r.Use(cors.Middleware(), api.IdentityMiddleware(), api.PrivateSiteMiddleware(), noStoreAPIResponses())
|
||||
|
||||
// The recovery UI belongs only to its temporary restricted listener.
|
||||
r.GET(recoveryweb.PagePath, func(c *gin.Context) {
|
||||
c.Redirect(http.StatusTemporaryRedirect, "/")
|
||||
})
|
||||
router.Register(r)
|
||||
a.registerReloadHandlers(cors)
|
||||
a.reload.Start()
|
||||
a.engine = r
|
||||
return nil
|
||||
}
|
||||
|
||||
// Run starts the normal HTTP server and blocks until shutdown or fatal error.
|
||||
func (a *App) Run() error {
|
||||
a.server = &http.Server{Addr: a.listenAddr, Handler: a.engine}
|
||||
serverErr := make(chan error, 1)
|
||||
logger.Infof("server", "Starting server on %s ...", a.listenAddr)
|
||||
go func() {
|
||||
if err := a.server.ListenAndServe(); err != nil && err != http.ErrServerClosed {
|
||||
serverErr <- err
|
||||
}
|
||||
}()
|
||||
|
||||
quit := make(chan os.Signal, 1)
|
||||
signal.Notify(quit, os.Interrupt, syscall.SIGTERM)
|
||||
defer signal.Stop(quit)
|
||||
select {
|
||||
case err := <-serverErr:
|
||||
a.onFatal(err)
|
||||
return fmt.Errorf("listen: %w", err)
|
||||
case reason := <-lifecycle.RestartRequests():
|
||||
logger.Infof("server", "Restarting service for %s", reason)
|
||||
if err := a.Shutdown(); err != nil {
|
||||
logger.Errorf("server", "Cleanup before restart failed: %v", err)
|
||||
}
|
||||
return fmt.Errorf("%w: %s", ErrRestartRequested, reason)
|
||||
case <-quit:
|
||||
return a.Shutdown()
|
||||
}
|
||||
}
|
||||
|
||||
// Shutdown stops HTTP first, then releases registered resources in LIFO order.
|
||||
func (a *App) Shutdown() error {
|
||||
if a.dbReady {
|
||||
auditlog.Log("", "", "server is shutting down", "info")
|
||||
}
|
||||
httpCtx, cancelHTTP := context.WithTimeout(context.Background(), httpShutdownTimeout)
|
||||
defer cancelHTTP()
|
||||
if a.server != nil {
|
||||
if err := a.server.Shutdown(httpCtx); err != nil {
|
||||
logger.Infof("server", "HTTP server forced to shutdown: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
cleanupCtx, cancelCleanup := context.WithTimeout(context.Background(), resourceCleanupTimeout)
|
||||
defer cancelCleanup()
|
||||
return a.runCleanups(cleanupCtx)
|
||||
}
|
||||
|
||||
func (a *App) onFatal(err error) {
|
||||
if a.dbReady {
|
||||
auditlog.Log("", "", "server encountered a fatal error: "+err.Error(), "error")
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), resourceCleanupTimeout)
|
||||
defer cancel()
|
||||
if cleanupErr := a.runCleanups(ctx); cleanupErr != nil {
|
||||
logger.Errorf("server", "Cleanup after fatal server error failed: %v", cleanupErr)
|
||||
}
|
||||
}
|
||||
|
||||
func (a *App) runCleanups(ctx context.Context) error {
|
||||
var cleanupErrors []error
|
||||
for i := len(a.cleanups) - 1; i >= 0; i-- {
|
||||
cleanup := a.cleanups[i]
|
||||
if err := cleanup.fn(ctx); err != nil {
|
||||
logger.Errorf("server", "cleanup %q failed: %v", cleanup.name, err)
|
||||
cleanupErrors = append(cleanupErrors, fmt.Errorf("cleanup %q: %w", cleanup.name, err))
|
||||
}
|
||||
}
|
||||
return errors.Join(cleanupErrors...)
|
||||
}
|
||||
|
||||
func registerScheduledWork() {
|
||||
if err := tasks.ReloadPingSchedule(); err != nil {
|
||||
logger.ErrorArgs("server", "Failed to reload ping schedule:", err)
|
||||
}
|
||||
if err := d_notification.ReloadLoadNotificationSchedule(); err != nil {
|
||||
logger.ErrorArgs("server", "Failed to reload load notification schedule:", err)
|
||||
}
|
||||
if err := scheduler.AddFunc("records:cleanup", "@every 30m", cleanupScheduledData); err != nil {
|
||||
logger.ErrorArgs("server", "Failed to add cleanup scheduled task:", err)
|
||||
}
|
||||
if err := scheduler.AddContextFunc("metrics:compact", "@every 5m", true, compactMetricStore); err != nil {
|
||||
logger.ErrorArgs("server", "Failed to add metric compact scheduled task:", err)
|
||||
}
|
||||
if err := scheduler.AddFunc("notifier:traffic", "@every 1m", notifier.CheckTraffic); err != nil {
|
||||
logger.ErrorArgs("server", "Failed to add traffic notification task:", err)
|
||||
}
|
||||
if err := scheduler.AddFunc("notifier:expire", "0 0 9 * * *", notifier.CheckExpireScheduledWork); err != nil {
|
||||
logger.ErrorArgs("server", "Failed to add expire notification task:", err)
|
||||
}
|
||||
notifier.InitTrafficReportSchedule()
|
||||
}
|
||||
|
||||
const taskResultRetentionDays = 30
|
||||
|
||||
func cleanupScheduledData() {
|
||||
before := time.Now().UTC().Add(-24 * time.Hour * taskResultRetentionDays)
|
||||
if err := tasks.ClearTaskResultsByTimeBefore(before); err != nil {
|
||||
logger.Errorf("server", "Failed to clean expired task results: %v", err)
|
||||
}
|
||||
auditlog.RemoveOldLogs()
|
||||
accounts.RemoveExpiredSessions()
|
||||
}
|
||||
|
||||
func compactMetricStore(ctx context.Context) {
|
||||
compactCtx, cancel := context.WithTimeout(ctx, 15*time.Minute)
|
||||
defer cancel()
|
||||
written, err := metricstore.Compact(compactCtx, time.Now().UTC())
|
||||
if errors.Is(err, metricstore.ErrCompactInProgress) {
|
||||
return
|
||||
}
|
||||
if err != nil {
|
||||
logger.Errorf("server", "Failed to compact metric store after writing %d rollup buckets: %v", written, err)
|
||||
return
|
||||
}
|
||||
if written > 0 {
|
||||
logger.Infof("server", "Metric store compacted %d rollup buckets", written)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,181 @@
|
||||
// Package sqlitetune applies a consistent SQLite baseline to every physical
|
||||
// connection opened through database/sql.
|
||||
package sqlitetune
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"database/sql/driver"
|
||||
"fmt"
|
||||
"math"
|
||||
"time"
|
||||
|
||||
sqlite3 "github.com/mattn/go-sqlite3"
|
||||
)
|
||||
|
||||
// SynchronousMode selects SQLite's synchronous durability mode.
|
||||
type SynchronousMode string
|
||||
|
||||
const (
|
||||
// SynchronousNormal balances write throughput and durability in WAL mode.
|
||||
SynchronousNormal SynchronousMode = "NORMAL"
|
||||
// SynchronousOff favors throughput over durability.
|
||||
SynchronousOff SynchronousMode = "OFF"
|
||||
// SynchronousFull favors durability over write throughput.
|
||||
SynchronousFull SynchronousMode = "FULL"
|
||||
)
|
||||
|
||||
// Options configures the SQLite PRAGMAs applied to every physical connection.
|
||||
// WAL mode is always enabled. Zero mmap size explicitly disables memory
|
||||
// mapping; it is not treated as an unset value.
|
||||
type Options struct {
|
||||
PageSize int
|
||||
ForeignKeys bool
|
||||
BusyTimeout time.Duration
|
||||
CacheSizeKB int
|
||||
MMapSizeBytes int64
|
||||
TempStoreMemory bool
|
||||
CacheSpill bool
|
||||
WALAutoCheckpoint int
|
||||
JournalSizeLimitBytes int64
|
||||
Synchronous SynchronousMode
|
||||
}
|
||||
|
||||
// Open returns a database/sql pool whose new SQLite connections are configured
|
||||
// before they are handed to callers.
|
||||
func Open(dsn string, options Options) (*sql.DB, error) {
|
||||
options, err := normalize(options)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
driver := &sqlite3.SQLiteDriver{
|
||||
ConnectHook: func(conn *sqlite3.SQLiteConn) error {
|
||||
return applyDriverConn(conn, options)
|
||||
},
|
||||
}
|
||||
return sql.OpenDB(&connector{driver: driver, dsn: dsn}), nil
|
||||
}
|
||||
|
||||
// Apply configures one physical connection from a caller-owned pool. Future
|
||||
// connections in that pool remain the caller's responsibility; use Open when
|
||||
// this package owns the pool and every physical connection must be configured.
|
||||
func Apply(ctx context.Context, db *sql.DB, options Options) error {
|
||||
options, err := normalize(options)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
conn, err := db.Conn(ctx)
|
||||
if err != nil {
|
||||
return fmt.Errorf("sqlite tune: acquire connection: %w", err)
|
||||
}
|
||||
defer conn.Close()
|
||||
|
||||
for _, pragma := range pragmas(options) {
|
||||
if _, err := conn.ExecContext(ctx, pragma); err != nil {
|
||||
return fmt.Errorf("sqlite tune: apply %q: %w", pragma, err)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
type connector struct {
|
||||
driver *sqlite3.SQLiteDriver
|
||||
dsn string
|
||||
}
|
||||
|
||||
func (c *connector) Connect(ctx context.Context) (driver.Conn, error) {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
conn, err := c.driver.Open(c.dsn)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := ctx.Err(); err != nil {
|
||||
_ = conn.Close()
|
||||
return nil, err
|
||||
}
|
||||
return conn, nil
|
||||
}
|
||||
|
||||
func (c *connector) Driver() driver.Driver {
|
||||
return c.driver
|
||||
}
|
||||
|
||||
func applyDriverConn(conn *sqlite3.SQLiteConn, options Options) error {
|
||||
for _, pragma := range pragmas(options) {
|
||||
if _, err := conn.Exec(pragma, nil); err != nil {
|
||||
return fmt.Errorf("sqlite tune: apply %q: %w", pragma, err)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func normalize(options Options) (Options, error) {
|
||||
if options.Synchronous == "" {
|
||||
options.Synchronous = SynchronousNormal
|
||||
}
|
||||
switch options.Synchronous {
|
||||
case SynchronousNormal, SynchronousOff, SynchronousFull:
|
||||
default:
|
||||
return options, fmt.Errorf("sqlite tune: unsupported synchronous mode %q", options.Synchronous)
|
||||
}
|
||||
if options.PageSize < 0 {
|
||||
return options, fmt.Errorf("sqlite tune: page size cannot be negative")
|
||||
}
|
||||
if options.BusyTimeout < 0 {
|
||||
return options, fmt.Errorf("sqlite tune: busy timeout cannot be negative")
|
||||
}
|
||||
if options.CacheSizeKB <= 0 {
|
||||
return options, fmt.Errorf("sqlite tune: cache size must be positive")
|
||||
}
|
||||
if options.MMapSizeBytes < 0 {
|
||||
return options, fmt.Errorf("sqlite tune: mmap size cannot be negative")
|
||||
}
|
||||
if options.WALAutoCheckpoint <= 0 {
|
||||
return options, fmt.Errorf("sqlite tune: WAL auto-checkpoint must be positive")
|
||||
}
|
||||
// SQLite uses -1 to disable the post-checkpoint WAL size limit. Preserve
|
||||
// that documented value for callers that explicitly opt out of the cap.
|
||||
if options.JournalSizeLimitBytes < -1 {
|
||||
return options, fmt.Errorf("sqlite tune: journal size limit cannot be less than -1")
|
||||
}
|
||||
return options, nil
|
||||
}
|
||||
|
||||
func pragmas(options Options) []string {
|
||||
pragmas := make([]string, 0, 11)
|
||||
if options.PageSize > 0 {
|
||||
pragmas = append(pragmas, fmt.Sprintf("PRAGMA page_size = %d", options.PageSize))
|
||||
}
|
||||
pragmas = append(pragmas, "PRAGMA journal_mode = WAL")
|
||||
if options.ForeignKeys {
|
||||
pragmas = append(pragmas, "PRAGMA foreign_keys = ON")
|
||||
}
|
||||
pragmas = append(pragmas,
|
||||
fmt.Sprintf("PRAGMA synchronous = %s", options.Synchronous),
|
||||
fmt.Sprintf("PRAGMA busy_timeout = %d", durationMillis(options.BusyTimeout)),
|
||||
fmt.Sprintf("PRAGMA cache_size = -%d", options.CacheSizeKB),
|
||||
fmt.Sprintf("PRAGMA mmap_size = %d", options.MMapSizeBytes),
|
||||
)
|
||||
if options.TempStoreMemory {
|
||||
pragmas = append(pragmas, "PRAGMA temp_store = MEMORY")
|
||||
} else {
|
||||
pragmas = append(pragmas, "PRAGMA temp_store = FILE")
|
||||
}
|
||||
if options.CacheSpill {
|
||||
pragmas = append(pragmas, "PRAGMA cache_spill = ON")
|
||||
} else {
|
||||
pragmas = append(pragmas, "PRAGMA cache_spill = OFF")
|
||||
}
|
||||
return append(pragmas,
|
||||
fmt.Sprintf("PRAGMA wal_autocheckpoint = %d", options.WALAutoCheckpoint),
|
||||
fmt.Sprintf("PRAGMA journal_size_limit = %d", options.JournalSizeLimitBytes),
|
||||
)
|
||||
}
|
||||
|
||||
func durationMillis(d time.Duration) int {
|
||||
return int(math.Ceil(float64(d) / float64(time.Millisecond)))
|
||||
}
|
||||
@@ -0,0 +1,21 @@
|
||||
package sqlitetune
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestNormalizeAllowsUnlimitedJournalSize(t *testing.T) {
|
||||
options, err := normalize(Options{
|
||||
BusyTimeout: 5 * time.Second,
|
||||
CacheSizeKB: 8,
|
||||
WALAutoCheckpoint: 1,
|
||||
JournalSizeLimitBytes: -1,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("normalize unlimited journal size: %v", err)
|
||||
}
|
||||
if options.JournalSizeLimitBytes != -1 {
|
||||
t.Fatalf("journal size limit = %d, want -1", options.JournalSizeLimitBytes)
|
||||
}
|
||||
}
|
||||
@@ -1,22 +1,21 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"log"
|
||||
"log/slog"
|
||||
|
||||
"github.com/komari-monitor/komari/cmd"
|
||||
"github.com/komari-monitor/komari/utils"
|
||||
logutil "github.com/komari-monitor/komari/utils/log"
|
||||
logger "github.com/komari-monitor/komari/utils/log"
|
||||
)
|
||||
|
||||
func main() {
|
||||
if utils.VersionHash == "unknown" {
|
||||
logutil.SetupGlobalLogger(slog.LevelDebug)
|
||||
logger.Setup(slog.LevelDebug)
|
||||
} else {
|
||||
logutil.SetupGlobalLogger(slog.LevelInfo)
|
||||
logger.Setup(slog.LevelInfo)
|
||||
}
|
||||
|
||||
log.Printf("Komari Monitor %s (hash: %s)", utils.CurrentVersion, utils.VersionHash)
|
||||
logger.Infof("server", "Komari Monitor %s (hash: %s)", utils.CurrentVersion, utils.VersionHash)
|
||||
|
||||
cmd.Execute()
|
||||
}
|
||||
+71
-36
@@ -7,6 +7,17 @@ import (
|
||||
"time"
|
||||
)
|
||||
|
||||
type aggregateSeriesKey struct {
|
||||
metricName string
|
||||
entityID string
|
||||
tagsHash string
|
||||
}
|
||||
|
||||
type aggregateGroupKey struct {
|
||||
aggregateSeriesKey
|
||||
bucket int64
|
||||
}
|
||||
|
||||
// AggregatePoints groups raw points into time buckets and computes aggregate values.
|
||||
//
|
||||
// AggregatePoints 在内存中按查询配置将原始点分桶并聚合。
|
||||
@@ -15,49 +26,88 @@ func AggregatePoints(points []Point, query AggregateQuery) ([]AggregatePoint, er
|
||||
return nil, err
|
||||
}
|
||||
if len(points) == 0 {
|
||||
if query.FillEmpty {
|
||||
return emptyBuckets(query), nil
|
||||
}
|
||||
return []AggregatePoint{}, nil
|
||||
}
|
||||
|
||||
points = sortedPoints(points)
|
||||
groups := make(map[int64][]Point)
|
||||
groups := make(map[aggregateGroupKey][]Point)
|
||||
seriesTags := make(map[aggregateSeriesKey]map[string]string)
|
||||
for _, point := range points {
|
||||
point = point.normalized()
|
||||
if point.MetricName == "" {
|
||||
point.MetricName = query.MetricName
|
||||
}
|
||||
if point.EntityID == "" {
|
||||
point.EntityID = query.EntityID
|
||||
}
|
||||
tagsHash, _, err := tagsFingerprint(point.Tags)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
bucket := alignTime(point.Timestamp, query.Interval).UnixNano()
|
||||
groups[bucket] = append(groups[bucket], point)
|
||||
series := aggregateSeriesKey{
|
||||
metricName: point.MetricName,
|
||||
entityID: point.EntityID,
|
||||
tagsHash: tagsHash,
|
||||
}
|
||||
key := aggregateGroupKey{aggregateSeriesKey: series, bucket: bucket}
|
||||
groups[key] = append(groups[key], point)
|
||||
if _, ok := seriesTags[series]; !ok {
|
||||
seriesTags[series] = cloneStringMap(point.Tags)
|
||||
}
|
||||
}
|
||||
|
||||
var buckets []int64
|
||||
if query.FillEmpty {
|
||||
for t := alignTime(query.Start, query.Interval); !t.After(query.End); t = t.Add(query.Interval) {
|
||||
buckets = append(buckets, t.UnixNano())
|
||||
}
|
||||
} else {
|
||||
for bucket := range groups {
|
||||
buckets = append(buckets, bucket)
|
||||
}
|
||||
sort.Slice(buckets, func(i, j int) bool { return buckets[i] < buckets[j] })
|
||||
keys := make([]aggregateGroupKey, 0, len(groups))
|
||||
for key := range groups {
|
||||
keys = append(keys, key)
|
||||
}
|
||||
sortAggregateGroupKeys(keys)
|
||||
|
||||
out := make([]AggregatePoint, 0, len(buckets))
|
||||
for _, bucket := range buckets {
|
||||
group := groups[bucket]
|
||||
out := make([]AggregatePoint, 0, len(keys))
|
||||
for _, key := range keys {
|
||||
group := groups[key]
|
||||
value, err := aggregateValue(group, query.Aggregation)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out = append(out, AggregatePoint{
|
||||
MetricName: query.MetricName,
|
||||
EntityID: query.EntityID,
|
||||
Bucket: time.Unix(0, bucket).UTC(),
|
||||
MetricName: key.metricName,
|
||||
EntityID: key.entityID,
|
||||
Bucket: time.Unix(0, key.bucket).UTC(),
|
||||
Value: value,
|
||||
Count: len(group),
|
||||
Tags: cloneStringMap(seriesTags[key.aggregateSeriesKey]),
|
||||
})
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func sortAggregateGroupKeys(keys []aggregateGroupKey) {
|
||||
sort.Slice(keys, func(i, j int) bool {
|
||||
if keys[i].bucket != keys[j].bucket {
|
||||
return keys[i].bucket < keys[j].bucket
|
||||
}
|
||||
if keys[i].metricName != keys[j].metricName {
|
||||
return keys[i].metricName < keys[j].metricName
|
||||
}
|
||||
if keys[i].entityID != keys[j].entityID {
|
||||
return keys[i].entityID < keys[j].entityID
|
||||
}
|
||||
return keys[i].tagsHash < keys[j].tagsHash
|
||||
})
|
||||
}
|
||||
|
||||
func cloneStringMap(in map[string]string) map[string]string {
|
||||
if len(in) == 0 {
|
||||
return map[string]string{}
|
||||
}
|
||||
out := make(map[string]string, len(in))
|
||||
for k, v := range in {
|
||||
out[k] = v
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// CalculateStats computes summary statistics for a point series.
|
||||
//
|
||||
// CalculateStats 基于一组点计算统计摘要,包括均值、百分位、首尾值和标准差。
|
||||
@@ -253,18 +303,3 @@ func alignTime(t time.Time, interval time.Duration) time.Time {
|
||||
rem := ((nano % size) + size) % size
|
||||
return time.Unix(0, nano-rem).UTC()
|
||||
}
|
||||
|
||||
// emptyBuckets builds zero-count aggregate buckets for an empty range.
|
||||
//
|
||||
// emptyBuckets 根据查询范围生成空聚合桶,用于 FillEmpty 场景。
|
||||
func emptyBuckets(query AggregateQuery) []AggregatePoint {
|
||||
var out []AggregatePoint
|
||||
for t := alignTime(query.Start, query.Interval); !t.After(query.End); t = t.Add(query.Interval) {
|
||||
out = append(out, AggregatePoint{
|
||||
MetricName: query.MetricName,
|
||||
EntityID: query.EntityID,
|
||||
Bucket: t,
|
||||
})
|
||||
}
|
||||
return out
|
||||
}
|
||||
+54
-37
@@ -33,10 +33,6 @@ type Config struct {
|
||||
//
|
||||
// TablePrefix 是 Store 管理的所有表名前缀。
|
||||
TablePrefix string
|
||||
// DefaultRetentionDays is used when a metric definition omits retention.
|
||||
//
|
||||
// DefaultRetentionDays 在指标定义未设置保留时间时使用。
|
||||
DefaultRetentionDays int
|
||||
// AutoMigrate controls whether Open creates or updates the schema.
|
||||
//
|
||||
// AutoMigrate 控制 Open 是否创建或更新表结构。
|
||||
@@ -62,12 +58,10 @@ type Config struct {
|
||||
// SQLite 保存 SQLite 专用选项。
|
||||
SQLite SQLiteOptions
|
||||
|
||||
// RollupPolicy configures downsampling tiers and tiered retention. When it
|
||||
// defines tiers, Store.Compact materializes them and enforces every
|
||||
// retention window. The zero value disables rollups (raw points only).
|
||||
// RollupPolicy configures downsampling tiers and tiered retention. Exact raw
|
||||
// samples use the Store's fixed ten-minute in-memory window.
|
||||
//
|
||||
// RollupPolicy 配置降采样层级和分层保留时间;定义层级后,
|
||||
// Store.Compact 会生成 rollup 并执行保留策略。零值表示禁用 rollup。
|
||||
// RollupPolicy 配置降采样层级和分层保留时间。
|
||||
RollupPolicy RollupPolicy
|
||||
}
|
||||
|
||||
@@ -103,6 +97,8 @@ type SQLiteOptions struct {
|
||||
//
|
||||
// WALAutoCheckpoint 设置 SQLite WAL 自动 checkpoint 页数。
|
||||
WALAutoCheckpoint int
|
||||
// JournalSizeLimitBytes caps the WAL retained after checkpoints.
|
||||
JournalSizeLimitBytes int64
|
||||
// ReadPoolSize, when > 0, opens a second read-only connection pool with
|
||||
// this many connections for SELECT-style calls. SQLite serializes writes,
|
||||
// but WAL lets readers run concurrently, so a read pool lifts read
|
||||
@@ -151,22 +147,32 @@ type Option func(*Config)
|
||||
// DefaultConfig 返回指定后端的默认配置。
|
||||
func DefaultConfig(driver Driver, dsn string) Config {
|
||||
return Config{
|
||||
Driver: driver,
|
||||
DSN: dsn,
|
||||
TablePrefix: "metric_",
|
||||
DefaultRetentionDays: 90,
|
||||
AutoMigrate: true,
|
||||
MaxOpenConns: 25,
|
||||
MaxIdleConns: 5,
|
||||
ConnMaxLifetime: time.Hour,
|
||||
ConnectTimeout: 10 * time.Second,
|
||||
Driver: driver,
|
||||
DSN: dsn,
|
||||
TablePrefix: "metric_",
|
||||
AutoMigrate: true,
|
||||
MaxOpenConns: 25,
|
||||
MaxIdleConns: 5,
|
||||
ConnMaxLifetime: time.Hour,
|
||||
ConnectTimeout: 10 * time.Second,
|
||||
SQLite: SQLiteOptions{
|
||||
PerformanceProfile: SQLiteProfileBalanced,
|
||||
BusyTimeout: 5 * time.Second,
|
||||
CacheSizeKB: 64 * 1024,
|
||||
TempStoreMemory: true,
|
||||
MMapSizeBytes: 256 * 1024 * 1024,
|
||||
WALAutoCheckpoint: 1000,
|
||||
PerformanceProfile: SQLiteProfileBalanced,
|
||||
BusyTimeout: 5 * time.Second,
|
||||
CacheSizeKB: 16 * 1024,
|
||||
TempStoreMemory: false,
|
||||
MMapSizeBytes: 32 * 1024 * 1024,
|
||||
WALAutoCheckpoint: 1000,
|
||||
JournalSizeLimitBytes: 4 * 1024 * 1024,
|
||||
},
|
||||
RollupPolicy: RollupPolicy{
|
||||
RawRetention: 10 * time.Minute,
|
||||
Tiers: []RollupTier{
|
||||
{Interval: time.Minute, Retention: 600 * time.Minute},
|
||||
{Interval: 5 * time.Minute, Retention: 600 * 5 * time.Minute},
|
||||
{Interval: time.Hour, Retention: 600 * time.Hour},
|
||||
{Interval: 24 * time.Hour, Retention: 100 * 365 * 24 * time.Hour},
|
||||
},
|
||||
Compression: 30,
|
||||
},
|
||||
}
|
||||
}
|
||||
@@ -233,6 +239,10 @@ func SQLiteConfig(dsn string) Config {
|
||||
// is honored instead of being mistaken for "unset".
|
||||
cfg.MaxOpenConns = 1
|
||||
cfg.MaxIdleConns = 1
|
||||
// SQLite benefits from warm page caches. Unlike client/server backends,
|
||||
// rotating its local file connections only adds churn, so the default keeps
|
||||
// them until the Store closes. Callers can still set a lifetime explicitly.
|
||||
cfg.ConnMaxLifetime = 0
|
||||
return cfg
|
||||
}
|
||||
|
||||
@@ -268,15 +278,6 @@ func WithTablePrefix(prefix string) Option {
|
||||
}
|
||||
}
|
||||
|
||||
// WithDefaultRetention sets the default metric retention in days.
|
||||
//
|
||||
// WithDefaultRetention 设置指标默认保留天数。
|
||||
func WithDefaultRetention(days int) Option {
|
||||
return func(c *Config) {
|
||||
c.DefaultRetentionDays = days
|
||||
}
|
||||
}
|
||||
|
||||
// WithAutoMigrate controls whether Open creates the schema automatically.
|
||||
//
|
||||
// WithAutoMigrate 控制 Open 时是否自动创建表结构。
|
||||
@@ -385,6 +386,13 @@ func WithSQLiteWALAutoCheckpoint(pages int) Option {
|
||||
}
|
||||
}
|
||||
|
||||
// WithSQLiteJournalSizeLimit sets the WAL bytes retained after checkpoints.
|
||||
func WithSQLiteJournalSizeLimit(bytes int64) Option {
|
||||
return func(c *Config) {
|
||||
c.SQLite.JournalSizeLimitBytes = bytes
|
||||
}
|
||||
}
|
||||
|
||||
// WithSQLiteReadPool enables a dedicated read-only connection pool of n
|
||||
// connections for SQLite. Writes stay on the single primary connection
|
||||
// (SQLite serializes them); reads fan out across the pool, which WAL mode
|
||||
@@ -401,10 +409,10 @@ func WithSQLiteReadPool(n int) Option {
|
||||
|
||||
// WithRollupPolicy sets the downsampling/retention ladder. Compact uses it to
|
||||
// build rollup tiers (each progressively coarser and longer-lived) and to age
|
||||
// out raw points and expired tiers.
|
||||
// out expired summary tiers.
|
||||
//
|
||||
// WithRollupPolicy 设置降采样与保留时间阶梯;Compact 会据此构建逐级
|
||||
// 更粗、保留更久的 rollup,并清理过期原始点和过期层级。
|
||||
// 更粗、保留更久的 rollup,并清理过期层级。
|
||||
func WithRollupPolicy(p RollupPolicy) Option {
|
||||
return func(c *Config) {
|
||||
c.RollupPolicy = p
|
||||
@@ -442,8 +450,17 @@ func (c Config) Validate() error {
|
||||
return fmt.Errorf("%w: table prefix must contain only letters, digits, and underscores", ErrInvalidArgument)
|
||||
}
|
||||
}
|
||||
if c.DefaultRetentionDays <= 0 {
|
||||
return fmt.Errorf("%w: default retention days must be positive", ErrInvalidArgument)
|
||||
// Bound generated index names to the backend identifier limit. Restructure
|
||||
// performs the stricter check required by its temporary "rebuild_" prefix.
|
||||
switch c.Driver {
|
||||
case DriverMySQL:
|
||||
if len(c.TablePrefix) > 35 {
|
||||
return fmt.Errorf("%w: table prefix is too long for MySQL identifiers", ErrInvalidArgument)
|
||||
}
|
||||
case DriverPostgreSQL:
|
||||
if len(c.TablePrefix) > 34 {
|
||||
return fmt.Errorf("%w: table prefix is too long for PostgreSQL identifiers", ErrInvalidArgument)
|
||||
}
|
||||
}
|
||||
if err := c.RollupPolicy.Validate(); err != nil {
|
||||
return err
|
||||
|
||||
@@ -12,7 +12,6 @@ func TestBackendBuildersApplyOptions(t *testing.T) {
|
||||
cfg := SQLite(
|
||||
"file:metrics.db?cache=shared",
|
||||
WithTablePrefix("x_metric_"),
|
||||
WithDefaultRetention(180),
|
||||
WithAutoMigrate(false),
|
||||
WithMaxOpenConns(10),
|
||||
WithMaxIdleConns(2),
|
||||
@@ -26,9 +25,6 @@ func TestBackendBuildersApplyOptions(t *testing.T) {
|
||||
if cfg.TablePrefix != "x_metric_" {
|
||||
t.Fatalf("table prefix option was not applied: %q", cfg.TablePrefix)
|
||||
}
|
||||
if cfg.DefaultRetentionDays != 180 {
|
||||
t.Fatalf("default retention option was not applied: %d", cfg.DefaultRetentionDays)
|
||||
}
|
||||
if cfg.AutoMigrate {
|
||||
t.Fatalf("auto migrate option was not applied")
|
||||
}
|
||||
|
||||
+17
-95
@@ -1,7 +1,6 @@
|
||||
package metric
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"fmt"
|
||||
"strings"
|
||||
)
|
||||
@@ -24,7 +23,6 @@ type dialect interface {
|
||||
autoIncrementPrimaryKey() string
|
||||
nowExpr() string
|
||||
insertDefinitionSQL(t tables) string
|
||||
upsertPointSQL(t tables, rowCount int) string
|
||||
// jsonExtractEquals renders a boolean predicate comparing the text value at
|
||||
// key inside a JSON column to a bind placeholder. The key is interpolated
|
||||
// into the SQL (callers must restrict it to a safe charset); the compared
|
||||
@@ -38,21 +36,6 @@ type dialect interface {
|
||||
//
|
||||
// blobType 是 rollup t-digest sketch 的列类型。
|
||||
blobType() string
|
||||
// upsertRollupSQL builds a single-row upsert for one rollup bucket cell.
|
||||
//
|
||||
// upsertRollupSQL 为单个 rollup 桶单元构造单行 upsert。
|
||||
upsertRollupSQL(t tables) string
|
||||
// compactTxOptions returns the transaction options used for the compaction
|
||||
// transaction. PostgreSQL/MySQL escalate to SERIALIZABLE so the raw scan and
|
||||
// the raw deletion share one snapshot and a point inserted between them
|
||||
// cannot be deleted without first being rolled up. SQLite returns nil
|
||||
// (its default) because a single connection already serializes writes.
|
||||
//
|
||||
// compactTxOptions 返回 compaction 事务使用的事务选项。PostgreSQL/MySQL 会
|
||||
// 提升到 SERIALIZABLE,使 raw 扫描和 raw 删除共享同一个快照,让两者之间写入
|
||||
// 的点不会在尚未进入 rollup 前被删除。SQLite 返回 nil(默认),因为单连接
|
||||
// 已经串行化写入。
|
||||
compactTxOptions() *sql.TxOptions
|
||||
}
|
||||
|
||||
// tables stores the physical table names used by a Store.
|
||||
@@ -63,14 +46,20 @@ type tables struct {
|
||||
//
|
||||
// definitions 是指标定义表。
|
||||
definitions string
|
||||
// points is the raw points table.
|
||||
//
|
||||
// points 是原始采样点表。
|
||||
// points and watermarks name legacy tables read only by the explicit
|
||||
// structure rebuild. The normalized schema never creates either table.
|
||||
points string
|
||||
// series interns the name/entity/tag/tag-hash tuple.
|
||||
series string
|
||||
// resolutions interns rollup intervals in milliseconds.
|
||||
resolutions string
|
||||
// labels interns labels independently from series tags.
|
||||
labels string
|
||||
// rollups is the downsampled rollups table.
|
||||
//
|
||||
// rollups 是降采样 rollup 表。
|
||||
rollups string
|
||||
rollups string
|
||||
watermarks string
|
||||
}
|
||||
|
||||
// newDialect returns the SQL dialect implementation for a backend.
|
||||
@@ -122,7 +111,7 @@ func (sqliteDialect) nowExpr() string { return "CURRENT_TIMESTAMP" }
|
||||
// insertDefinitionSQL 构造 SQLite 指标定义 upsert SQL。
|
||||
func (sqliteDialect) insertDefinitionSQL(t tables) string {
|
||||
return fmt.Sprintf(`INSERT INTO %s
|
||||
(name, type, unit, description, retention_days, metadata, created_at, updated_at)
|
||||
(name, type, unit, description, retention_days, metadata, created_at_milli, updated_at_milli)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?)
|
||||
ON CONFLICT(name) DO UPDATE SET
|
||||
type = excluded.type,
|
||||
@@ -130,18 +119,7 @@ func (sqliteDialect) insertDefinitionSQL(t tables) string {
|
||||
description = excluded.description,
|
||||
retention_days = excluded.retention_days,
|
||||
metadata = excluded.metadata,
|
||||
updated_at = excluded.updated_at`, t.definitions)
|
||||
}
|
||||
|
||||
// upsertPointSQL builds SQL for inserting or updating metric points.
|
||||
//
|
||||
// upsertPointSQL 构造 SQLite 采样点批量 upsert SQL。
|
||||
func (sqliteDialect) upsertPointSQL(t tables, rowCount int) string {
|
||||
return buildInsertPointsSQL(t.points, sqliteDialect{}, rowCount, `ON CONFLICT(metric_name, entity_id, tags_hash, ts_nano) DO UPDATE SET
|
||||
value = excluded.value,
|
||||
tags = excluded.tags,
|
||||
labels = excluded.labels,
|
||||
created_at = excluded.created_at`)
|
||||
updated_at_milli = excluded.updated_at_milli`, t.definitions)
|
||||
}
|
||||
|
||||
// mysqlDialect implements SQL rendering for MySQL.
|
||||
@@ -179,7 +157,7 @@ func (mysqlDialect) nowExpr() string { return "CURRENT_TIMESTAMP" }
|
||||
// insertDefinitionSQL 构造 MySQL 指标定义 upsert SQL。
|
||||
func (mysqlDialect) insertDefinitionSQL(t tables) string {
|
||||
return fmt.Sprintf(`INSERT INTO %s
|
||||
(name, type, unit, description, retention_days, metadata, created_at, updated_at)
|
||||
(name, type, unit, description, retention_days, metadata, created_at_milli, updated_at_milli)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?)
|
||||
ON DUPLICATE KEY UPDATE
|
||||
type = VALUES(type),
|
||||
@@ -187,18 +165,7 @@ func (mysqlDialect) insertDefinitionSQL(t tables) string {
|
||||
description = VALUES(description),
|
||||
retention_days = VALUES(retention_days),
|
||||
metadata = VALUES(metadata),
|
||||
updated_at = VALUES(updated_at)`, t.definitions)
|
||||
}
|
||||
|
||||
// upsertPointSQL builds SQL for inserting or updating metric points.
|
||||
//
|
||||
// upsertPointSQL 构造 MySQL 采样点批量 upsert SQL。
|
||||
func (mysqlDialect) upsertPointSQL(t tables, rowCount int) string {
|
||||
return buildInsertPointsSQL(t.points, mysqlDialect{}, rowCount, `ON DUPLICATE KEY UPDATE
|
||||
value = VALUES(value),
|
||||
tags = VALUES(tags),
|
||||
labels = VALUES(labels),
|
||||
created_at = VALUES(created_at)`)
|
||||
updated_at_milli = VALUES(updated_at_milli)`, t.definitions)
|
||||
}
|
||||
|
||||
// postgresDialect implements SQL rendering for PostgreSQL.
|
||||
@@ -236,7 +203,7 @@ func (postgresDialect) nowExpr() string { return "CURRENT_TIMESTAMP" }
|
||||
// insertDefinitionSQL 构造 PostgreSQL 指标定义 upsert SQL。
|
||||
func (postgresDialect) insertDefinitionSQL(t tables) string {
|
||||
return fmt.Sprintf(`INSERT INTO %s
|
||||
(name, type, unit, description, retention_days, metadata, created_at, updated_at)
|
||||
(name, type, unit, description, retention_days, metadata, created_at_milli, updated_at_milli)
|
||||
VALUES ($1, $2, $3, $4, $5, $6::jsonb, $7, $8)
|
||||
ON CONFLICT(name) DO UPDATE SET
|
||||
type = EXCLUDED.type,
|
||||
@@ -244,52 +211,7 @@ func (postgresDialect) insertDefinitionSQL(t tables) string {
|
||||
description = EXCLUDED.description,
|
||||
retention_days = EXCLUDED.retention_days,
|
||||
metadata = EXCLUDED.metadata,
|
||||
updated_at = EXCLUDED.updated_at`, t.definitions)
|
||||
}
|
||||
|
||||
// upsertPointSQL builds SQL for inserting or updating metric points.
|
||||
//
|
||||
// upsertPointSQL 构造 PostgreSQL 采样点批量 upsert SQL。
|
||||
func (postgresDialect) upsertPointSQL(t tables, rowCount int) string {
|
||||
return buildInsertPointsSQL(t.points, postgresDialect{}, rowCount, `ON CONFLICT(metric_name, entity_id, tags_hash, ts_nano) DO UPDATE SET
|
||||
value = EXCLUDED.value,
|
||||
tags = EXCLUDED.tags,
|
||||
labels = EXCLUDED.labels,
|
||||
created_at = EXCLUDED.created_at`)
|
||||
}
|
||||
|
||||
// buildInsertPointsSQL builds a multi-row insert statement for metric points.
|
||||
//
|
||||
// buildInsertPointsSQL 构造多行采样点 INSERT/UPSERT SQL。
|
||||
func buildInsertPointsSQL(table string, d dialect, rowCount int, suffix string) string {
|
||||
var b strings.Builder
|
||||
b.WriteString("INSERT INTO ")
|
||||
b.WriteString(table)
|
||||
b.WriteString(" (metric_name, entity_id, tags_hash, ts_nano, value, tags, labels, created_at) VALUES ")
|
||||
arg := 1
|
||||
rows := make([]string, rowCount)
|
||||
for i := 0; i < rowCount; i++ {
|
||||
parts := make([]string, 8)
|
||||
for j := range parts {
|
||||
// Columns are (metric_name, entity_id, tags_hash, ts_nano, value, tags,
|
||||
// labels, created_at); positions 5 and 6 (0-indexed) are JSON values.
|
||||
// tags_hash makes the tag set part of a point's identity, so points
|
||||
// that share entity+timestamp but differ in tags no longer collide.
|
||||
if j == 5 || j == 6 {
|
||||
parts[j] = d.jsonPlaceholder(arg)
|
||||
} else {
|
||||
parts[j] = d.placeholder(arg)
|
||||
}
|
||||
arg++
|
||||
}
|
||||
rows[i] = "(" + strings.Join(parts, ", ") + ")"
|
||||
}
|
||||
b.WriteString(strings.Join(rows, ", "))
|
||||
if suffix != "" {
|
||||
b.WriteByte(' ')
|
||||
b.WriteString(suffix)
|
||||
}
|
||||
return b.String()
|
||||
updated_at_milli = EXCLUDED.updated_at_milli`, t.definitions)
|
||||
}
|
||||
|
||||
// tableName combines a table prefix and logical table name.
|
||||
@@ -306,7 +228,7 @@ func tableName(prefix, name string) string {
|
||||
// insertDefinitionOnlySQL 构造普通 INSERT(不带 upsert),供
|
||||
// CreateMetric 保持“只创建”语义;重复名称会暴露为唯一约束错误。
|
||||
func insertDefinitionOnlySQL(d dialect, t tables) string {
|
||||
cols := "(name, type, unit, description, retention_days, metadata, created_at, updated_at)"
|
||||
cols := "(name, type, unit, description, retention_days, metadata, created_at_milli, updated_at_milli)"
|
||||
ph := []string{
|
||||
d.placeholder(1),
|
||||
d.placeholder(2),
|
||||
|
||||
@@ -1,160 +1,6 @@
|
||||
package metric
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"fmt"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// rollupColumns is the fixed column list for the rollups table, shared by the
|
||||
// upsert builder and the row scanner so they cannot drift apart. tags_hash is a
|
||||
// stable fingerprint of the (canonical) tag set; tags carries the tag map itself
|
||||
// so reads can return it and tag filters can be pushed down. Including the tag
|
||||
// dimension in the key is what keeps each distinct tag combination (e.g. a GPU
|
||||
// device_index) as its own rollup series instead of being merged together.
|
||||
//
|
||||
// rollupColumns 是 rollups 表固定列清单,由 upsert 构造器和行扫描器共用,
|
||||
// 避免二者漂移。tags_hash 是(规范化)标签集合的稳定指纹;tags 携带标签 map
|
||||
// 本身,因此读取时可以返回它,也能下推标签过滤。把标签维度纳入 key,正是让
|
||||
// 每个不同标签组合(例如 GPU 的 device_index)保留为自己的 rollup 序列、
|
||||
// 而不是被合并在一起的原因。
|
||||
const rollupColumns = "metric_name, entity_id, tags_hash, tags, resolution_nano, bucket_nano, " +
|
||||
"count, sum, sum_sq, min_val, max_val, first_val, first_ts, last_val, last_ts, digest, created_at"
|
||||
|
||||
// rollupColumnCount is the number of columns in rollupColumns.
|
||||
//
|
||||
// rollupColumnCount 是 rollupColumns 中的列数量。
|
||||
const rollupColumnCount = 17
|
||||
|
||||
// rollupTagsArgIndex is the 1-based position of the `tags` JSON column in
|
||||
// rollupColumns. PostgreSQL needs a ::jsonb cast on that bind placeholder.
|
||||
//
|
||||
// rollupTagsArgIndex 是 tags JSON 列在 rollupColumns 中的 1 基位置;
|
||||
// PostgreSQL 需要在对应占位符上追加 ::jsonb 转换。
|
||||
const rollupTagsArgIndex = 4
|
||||
|
||||
// blobType returns the column type used to store the t-digest sketch.
|
||||
//
|
||||
// blobType 返回用于保存 t-digest 摘要的二进制列类型。
|
||||
func (sqliteDialect) blobType() string { return "BLOB" }
|
||||
|
||||
// blobType returns the binary column type used for t-digest blobs.
|
||||
//
|
||||
// blobType 返回 MySQL 用于保存 t-digest 摘要的二进制列类型。
|
||||
func (mysqlDialect) blobType() string { return "LONGBLOB" }
|
||||
|
||||
// blobType returns the binary column type used for t-digest blobs.
|
||||
//
|
||||
// blobType 返回 PostgreSQL 用于保存 t-digest 摘要的二进制列类型。
|
||||
// blobType returns the backend-specific column type for t-digest sketches.
|
||||
func (sqliteDialect) blobType() string { return "BLOB" }
|
||||
func (mysqlDialect) blobType() string { return "LONGBLOB" }
|
||||
func (postgresDialect) blobType() string { return "BYTEA" }
|
||||
|
||||
// compactTxOptions returns nil so SQLite uses its default isolation. A single
|
||||
// write connection already serializes the compaction transaction against
|
||||
// concurrent writers, so the raw scan and the raw deletion observe a stable
|
||||
// view without escalating the isolation level.
|
||||
//
|
||||
// compactTxOptions 返回 nil,让 SQLite 使用默认隔离级别。单个写连接已经把
|
||||
// compaction 事务与并发写入串行化,因此 raw 扫描和 raw 删除无需提升隔离级别
|
||||
// 即可观察到稳定视图。
|
||||
func (sqliteDialect) compactTxOptions() *sql.TxOptions { return nil }
|
||||
|
||||
// compactTxOptions escalates to SERIALIZABLE so a point inserted between the raw
|
||||
// scan and the raw deletion cannot be deleted without first being rolled up.
|
||||
// InnoDB's SERIALIZABLE turns the scan into a locking read that blocks such a
|
||||
// concurrent insert in the scanned range until the compaction commits.
|
||||
//
|
||||
// compactTxOptions 提升到 SERIALIZABLE,使在 raw 扫描和 raw 删除之间写入的点
|
||||
// 不会在尚未进入 rollup 前被删除。InnoDB 的 SERIALIZABLE 会把扫描变为加锁读,
|
||||
// 在 compaction 提交前阻塞扫描范围内的此类并发插入。
|
||||
func (mysqlDialect) compactTxOptions() *sql.TxOptions {
|
||||
return &sql.TxOptions{Isolation: sql.LevelSerializable}
|
||||
}
|
||||
|
||||
// compactTxOptions escalates to SERIALIZABLE so the raw scan and the raw
|
||||
// deletion share one snapshot. A concurrent insert into the scanned range that
|
||||
// would otherwise be deleted without a rollup instead triggers a serialization
|
||||
// failure that CompactMetric retries on a fresh snapshot.
|
||||
//
|
||||
// compactTxOptions 提升到 SERIALIZABLE,使 raw 扫描和 raw 删除共享同一个快照。
|
||||
// 对扫描范围内的并发插入(否则会被删除却没有进入 rollup)会触发序列化失败,
|
||||
// 由 CompactMetric 在新快照上重试。
|
||||
func (postgresDialect) compactTxOptions() *sql.TxOptions {
|
||||
return &sql.TxOptions{Isolation: sql.LevelSerializable}
|
||||
}
|
||||
|
||||
// upsertRollupSQL builds a single-row upsert for a rollup cell, keyed by
|
||||
// (metric_name, entity_id, tags_hash, resolution_nano, bucket_nano). Compact
|
||||
// recomputes a bucket wholesale from its inputs each run, so the conflict action
|
||||
// replaces every summary column rather than trying to merge in SQL.
|
||||
//
|
||||
// upsertRollupSQL 构造单个 rollup 单元的 upsert SQL,其 key 为
|
||||
// (metric_name, entity_id, tags_hash, resolution_nano, bucket_nano)。Compact
|
||||
// 每次都会从输入整体重新计算一个桶,因此冲突动作会替换每个摘要列,
|
||||
// 而不是尝试在 SQL 中合并。
|
||||
func (d sqliteDialect) upsertRollupSQL(t tables) string {
|
||||
return buildUpsertRollupSQL(t.rollups, d, sqliteRollupConflict)
|
||||
}
|
||||
|
||||
// upsertRollupSQL builds backend-specific SQL for upserting a rollup cell.
|
||||
//
|
||||
// upsertRollupSQL 构造 MySQL rollup 单元 upsert SQL。
|
||||
func (d mysqlDialect) upsertRollupSQL(t tables) string {
|
||||
return buildUpsertRollupSQL(t.rollups, d, mysqlRollupConflict)
|
||||
}
|
||||
|
||||
// upsertRollupSQL builds backend-specific SQL for upserting a rollup cell.
|
||||
//
|
||||
// upsertRollupSQL 构造 PostgreSQL rollup 单元 upsert SQL。
|
||||
func (d postgresDialect) upsertRollupSQL(t tables) string {
|
||||
return buildUpsertRollupSQL(t.rollups, d, postgresRollupConflict)
|
||||
}
|
||||
|
||||
// sqliteRollupConflict is the SQLite conflict clause for replacing rollup cells.
|
||||
//
|
||||
// sqliteRollupConflict 是 SQLite 用于替换 rollup 单元的冲突处理子句。
|
||||
const sqliteRollupConflict = `ON CONFLICT(metric_name, entity_id, tags_hash, resolution_nano, bucket_nano) DO UPDATE SET
|
||||
tags = excluded.tags,
|
||||
count = excluded.count, sum = excluded.sum, sum_sq = excluded.sum_sq,
|
||||
min_val = excluded.min_val, max_val = excluded.max_val,
|
||||
first_val = excluded.first_val, first_ts = excluded.first_ts,
|
||||
last_val = excluded.last_val, last_ts = excluded.last_ts,
|
||||
digest = excluded.digest, created_at = excluded.created_at`
|
||||
|
||||
// postgresRollupConflict is the PostgreSQL conflict clause for replacing rollup cells.
|
||||
//
|
||||
// postgresRollupConflict 是 PostgreSQL 用于替换 rollup 单元的冲突处理子句。
|
||||
const postgresRollupConflict = `ON CONFLICT(metric_name, entity_id, tags_hash, resolution_nano, bucket_nano) DO UPDATE SET
|
||||
tags = EXCLUDED.tags,
|
||||
count = EXCLUDED.count, sum = EXCLUDED.sum, sum_sq = EXCLUDED.sum_sq,
|
||||
min_val = EXCLUDED.min_val, max_val = EXCLUDED.max_val,
|
||||
first_val = EXCLUDED.first_val, first_ts = EXCLUDED.first_ts,
|
||||
last_val = EXCLUDED.last_val, last_ts = EXCLUDED.last_ts,
|
||||
digest = EXCLUDED.digest, created_at = EXCLUDED.created_at`
|
||||
|
||||
// mysqlRollupConflict is the MySQL conflict clause for replacing rollup cells.
|
||||
//
|
||||
// mysqlRollupConflict 是 MySQL 用于替换 rollup 单元的冲突处理子句。
|
||||
const mysqlRollupConflict = `ON DUPLICATE KEY UPDATE
|
||||
tags = VALUES(tags),
|
||||
count = VALUES(count), sum = VALUES(sum), sum_sq = VALUES(sum_sq),
|
||||
min_val = VALUES(min_val), max_val = VALUES(max_val),
|
||||
first_val = VALUES(first_val), first_ts = VALUES(first_ts),
|
||||
last_val = VALUES(last_val), last_ts = VALUES(last_ts),
|
||||
digest = VALUES(digest), created_at = VALUES(created_at)`
|
||||
|
||||
// buildUpsertRollupSQL builds a single-row rollup upsert statement.
|
||||
//
|
||||
// buildUpsertRollupSQL 构造单行 rollup upsert SQL。
|
||||
func buildUpsertRollupSQL(table string, d dialect, conflict string) string {
|
||||
ph := make([]string, rollupColumnCount)
|
||||
for i := 0; i < rollupColumnCount; i++ {
|
||||
if i+1 == rollupTagsArgIndex {
|
||||
// The tags column is JSON; PostgreSQL requires an explicit ::jsonb cast.
|
||||
ph[i] = d.jsonPlaceholder(i + 1)
|
||||
} else {
|
||||
ph[i] = d.placeholder(i + 1)
|
||||
}
|
||||
}
|
||||
return fmt.Sprintf("INSERT INTO %s (%s) VALUES (%s) %s",
|
||||
table, rollupColumns, strings.Join(ph, ", "), conflict)
|
||||
}
|
||||
@@ -1,9 +1,6 @@
|
||||
package metric
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
import "testing"
|
||||
|
||||
// TestDialectsGenerateBackendSpecificSQL verifies backend-specific SQL rendering.
|
||||
//
|
||||
@@ -13,29 +10,29 @@ func TestDialectsGenerateBackendSpecificSQL(t *testing.T) {
|
||||
name string
|
||||
driver Driver
|
||||
placeholder string
|
||||
upsert string
|
||||
jsonType string
|
||||
blobType string
|
||||
}{
|
||||
{
|
||||
name: "sqlite",
|
||||
driver: DriverSQLite,
|
||||
placeholder: "?",
|
||||
upsert: "ON CONFLICT(metric_name, entity_id, tags_hash, ts_nano)",
|
||||
jsonType: "TEXT",
|
||||
blobType: "BLOB",
|
||||
},
|
||||
{
|
||||
name: "mysql",
|
||||
driver: DriverMySQL,
|
||||
placeholder: "?",
|
||||
upsert: "ON DUPLICATE KEY UPDATE",
|
||||
jsonType: "JSON",
|
||||
blobType: "LONGBLOB",
|
||||
},
|
||||
{
|
||||
name: "postgresql",
|
||||
driver: DriverPostgreSQL,
|
||||
placeholder: "$1",
|
||||
upsert: "ON CONFLICT(metric_name, entity_id, tags_hash, ts_nano)",
|
||||
jsonType: "JSONB",
|
||||
blobType: "BYTEA",
|
||||
},
|
||||
}
|
||||
|
||||
@@ -48,12 +45,8 @@ func TestDialectsGenerateBackendSpecificSQL(t *testing.T) {
|
||||
if got := d.jsonType(); got != tt.jsonType {
|
||||
t.Fatalf("json type: expected %q, got %q", tt.jsonType, got)
|
||||
}
|
||||
sql := d.upsertPointSQL(tables{points: "metric_points"}, 2)
|
||||
if !strings.Contains(sql, "metric_points") {
|
||||
t.Fatalf("upsert sql should reference metric_points: %s", sql)
|
||||
}
|
||||
if !strings.Contains(sql, tt.upsert) {
|
||||
t.Fatalf("upsert sql should contain %q: %s", tt.upsert, sql)
|
||||
if got := d.blobType(); got != tt.blobType {
|
||||
t.Fatalf("blob type: expected %q, got %q", tt.blobType, got)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
+18
-18
@@ -20,7 +20,7 @@ func Example_rollupTags() {
|
||||
store, err := metric.Open(ctx, metric.SQLite(
|
||||
"file:metric-example?mode=memory&cache=shared",
|
||||
metric.WithRollupPolicy(metric.RollupPolicy{
|
||||
RawRetention: 2 * time.Minute,
|
||||
RawRetention: 10 * time.Minute,
|
||||
Tiers: []metric.RollupTier{
|
||||
{Interval: time.Minute, Retention: 24 * time.Hour},
|
||||
},
|
||||
@@ -32,10 +32,10 @@ func Example_rollupTags() {
|
||||
defer store.Close()
|
||||
|
||||
if err := store.CreateMetric(ctx, metric.Definition{
|
||||
Name: "gpu.usage",
|
||||
Type: metric.TypeGauge,
|
||||
Unit: "%",
|
||||
}); err != nil {
|
||||
Name: "gpu.usage",
|
||||
Type: metric.TypeGauge,
|
||||
Unit: "%",
|
||||
RetentionDays: 30}); err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
|
||||
@@ -63,8 +63,8 @@ func Example_rollupTags() {
|
||||
log.Fatal(err)
|
||||
}
|
||||
|
||||
// Compact builds one rollup series per tag set. Raw points older than the
|
||||
// policy's RawRetention are deleted after their rollups are materialized.
|
||||
// Compact builds one rollup series per tag set. Exact raw points stay only
|
||||
// in the Store's fixed ten-minute memory window.
|
||||
now := base.Add(time.Hour)
|
||||
if _, err := store.Compact(ctx, now); err != nil {
|
||||
log.Fatal(err)
|
||||
@@ -104,7 +104,7 @@ func Example_agentFleetLifecycle() {
|
||||
store, err := metric.Open(ctx, metric.SQLite(
|
||||
"file:metric-agent-fleet?mode=memory&cache=shared",
|
||||
metric.WithRollupPolicy(metric.RollupPolicy{
|
||||
RawRetention: 2 * time.Minute,
|
||||
RawRetention: 10 * time.Minute,
|
||||
Tiers: []metric.RollupTier{
|
||||
{Interval: time.Minute, Retention: 24 * time.Hour},
|
||||
{Interval: 5 * time.Minute, Retention: 30 * 24 * time.Hour},
|
||||
@@ -117,15 +117,15 @@ func Example_agentFleetLifecycle() {
|
||||
defer store.Close()
|
||||
|
||||
for _, def := range []metric.Definition{
|
||||
{Name: "cpu.usage", Type: metric.TypeGauge, Unit: "%"},
|
||||
{Name: "mem.usage", Type: metric.TypeGauge, Unit: "%"},
|
||||
{Name: "disk.usage", Type: metric.TypeGauge, Unit: "%"},
|
||||
{Name: "swap.usage", Type: metric.TypeGauge, Unit: "%"},
|
||||
{Name: "net.in.bytes", Type: metric.TypeCounter, Unit: "bytes"},
|
||||
{Name: "net.out.bytes", Type: metric.TypeCounter, Unit: "bytes"},
|
||||
{Name: "process.count", Type: metric.TypeGauge, Unit: "count"},
|
||||
{Name: "gpu.usage", Type: metric.TypeGauge, Unit: "%"},
|
||||
{Name: "ping.latency_ms", Type: metric.TypeGauge, Unit: "ms"},
|
||||
{Name: "cpu.usage", Type: metric.TypeGauge, Unit: "%", RetentionDays: 30},
|
||||
{Name: "mem.usage", Type: metric.TypeGauge, Unit: "%", RetentionDays: 30},
|
||||
{Name: "disk.usage", Type: metric.TypeGauge, Unit: "%", RetentionDays: 30},
|
||||
{Name: "swap.usage", Type: metric.TypeGauge, Unit: "%", RetentionDays: 30},
|
||||
{Name: "net.in.bytes", Type: metric.TypeCounter, Unit: "bytes", RetentionDays: 30},
|
||||
{Name: "net.out.bytes", Type: metric.TypeCounter, Unit: "bytes", RetentionDays: 30},
|
||||
{Name: "process.count", Type: metric.TypeGauge, Unit: "count", RetentionDays: 30},
|
||||
{Name: "gpu.usage", Type: metric.TypeGauge, Unit: "%", RetentionDays: 30},
|
||||
{Name: "ping.latency_ms", Type: metric.TypeGauge, Unit: "ms", RetentionDays: 30},
|
||||
} {
|
||||
if err := store.UpsertMetric(ctx, def); err != nil {
|
||||
log.Fatal(err)
|
||||
@@ -158,7 +158,7 @@ func Example_agentFleetLifecycle() {
|
||||
}
|
||||
if err := store.UpsertMetric(ctx, metric.Definition{
|
||||
Name: "ping.latency_ms", Type: metric.TypeGauge, Unit: "ms",
|
||||
}); err != nil {
|
||||
RetentionDays: 30}); err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
case 85:
|
||||
|
||||
@@ -0,0 +1,290 @@
|
||||
package metric
|
||||
|
||||
import (
|
||||
"context"
|
||||
"sort"
|
||||
"time"
|
||||
)
|
||||
|
||||
// hotRollupKey identifies one active minute bucket. The dictionary dimensions
|
||||
// remain compact strings; full tag and label JSON lives only on the bucket.
|
||||
type hotRollupKey struct {
|
||||
metricName string
|
||||
entityID string
|
||||
tagsHash string
|
||||
labelsHash string
|
||||
bucket int64
|
||||
}
|
||||
|
||||
type hotRollup struct {
|
||||
key hotRollupKey
|
||||
bucket *rollupBucket
|
||||
replace bool
|
||||
}
|
||||
|
||||
func (s *Store) writePreparedHotRollups(ctx context.Context, prepared []preparedMetricPoint, now time.Time, rebuild map[hotRollupKey]struct{}) error {
|
||||
minuteMillis := time.Minute.Milliseconds()
|
||||
compression := s.cfg.RollupPolicy.compression()
|
||||
if len(rebuild) > 0 {
|
||||
s.rollupViewMu.Lock()
|
||||
defer s.rollupViewMu.Unlock()
|
||||
}
|
||||
s.hotMu.Lock()
|
||||
for _, point := range prepared {
|
||||
key := hotRollupKey{
|
||||
metricName: point.metricName,
|
||||
entityID: point.entityID,
|
||||
tagsHash: point.tagsHash,
|
||||
labelsHash: point.labelsHash,
|
||||
bucket: bucketStartMillis(point.timestamp, minuteMillis),
|
||||
}
|
||||
bucket := s.hot[key]
|
||||
if bucket == nil {
|
||||
bucket = newRollupBucket(compression)
|
||||
bucket.tagsHash, bucket.tagsJSON = point.tagsHash, point.tagsJSON
|
||||
bucket.labelsHash, bucket.labelsJSON = point.labelsHash, point.labelsJSON
|
||||
s.hot[key] = bucket
|
||||
}
|
||||
bucket.addPoint(point.value, point.timestamp)
|
||||
}
|
||||
if len(rebuild) > 0 {
|
||||
s.rebuildHotRollupsLocked(rebuild, compression)
|
||||
}
|
||||
s.hotMu.Unlock()
|
||||
var err error
|
||||
if len(rebuild) > 0 {
|
||||
_, err = s.flushClosedHotRollupsUnderView(ctx, now)
|
||||
} else {
|
||||
_, err = s.flushClosedHotRollups(ctx, now)
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
func (s *Store) rebuildHotRollupsLocked(keys map[hotRollupKey]struct{}, compression float64) {
|
||||
s.rawMu.RLock()
|
||||
defer s.rawMu.RUnlock()
|
||||
minuteMillis := time.Minute.Milliseconds()
|
||||
for key := range keys {
|
||||
series := s.raw[rawSeriesKey{metricName: key.metricName, entityID: key.entityID, tagsHash: key.tagsHash}]
|
||||
bucket := newRollupBucket(compression)
|
||||
bucket.tagsHash = key.tagsHash
|
||||
bucket.labelsHash = key.labelsHash
|
||||
if series != nil {
|
||||
bucket.tagsJSON = series.tagsJSON
|
||||
if labelID, ok := series.labelIDs[key.labelsHash]; ok {
|
||||
bucket.labelsJSON = series.labelsJSON[labelID]
|
||||
}
|
||||
for _, sample := range series.allSamples() {
|
||||
if bucketStartMillis(sample.timestamp, minuteMillis) != key.bucket || series.labelHashes[sample.labelID] != key.labelsHash {
|
||||
continue
|
||||
}
|
||||
bucket.labelsJSON = series.labelsJSON[sample.labelID]
|
||||
bucket.addPoint(sample.value, sample.timestamp)
|
||||
}
|
||||
}
|
||||
s.hot[key] = bucket
|
||||
s.hotReplace[key] = struct{}{}
|
||||
}
|
||||
}
|
||||
|
||||
func (s *Store) flushClosedHotRollups(ctx context.Context, now time.Time) (int, error) {
|
||||
s.rollupViewMu.Lock()
|
||||
defer s.rollupViewMu.Unlock()
|
||||
return s.flushClosedHotRollupsUnderView(ctx, now)
|
||||
}
|
||||
|
||||
func (s *Store) flushClosedHotRollupsUnderView(ctx context.Context, now time.Time) (int, error) {
|
||||
flushBefore := bucketStartMillis(now.UTC().UnixMilli(), time.Minute.Milliseconds())
|
||||
closed := s.takeClosedHotRollups(flushBefore)
|
||||
if len(closed) == 0 {
|
||||
return 0, nil
|
||||
}
|
||||
if err := s.persistHotRollups(ctx, closed); err != nil {
|
||||
s.restoreHotRollups(closed)
|
||||
return 0, err
|
||||
}
|
||||
return len(closed), nil
|
||||
}
|
||||
|
||||
func (s *Store) flushAllHotRollups(ctx context.Context) error {
|
||||
s.rollupViewMu.Lock()
|
||||
defer s.rollupViewMu.Unlock()
|
||||
s.hotMu.Lock()
|
||||
all := make([]hotRollup, 0, len(s.hot))
|
||||
for key, bucket := range s.hot {
|
||||
_, replace := s.hotReplace[key]
|
||||
all = append(all, hotRollup{key: key, bucket: bucket, replace: replace})
|
||||
delete(s.hot, key)
|
||||
delete(s.hotReplace, key)
|
||||
}
|
||||
s.hotMu.Unlock()
|
||||
if len(all) == 0 {
|
||||
return nil
|
||||
}
|
||||
if err := s.persistHotRollups(ctx, all); err != nil {
|
||||
s.restoreHotRollups(all)
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *Store) takeClosedHotRollups(currentBucket int64) []hotRollup {
|
||||
s.hotMu.Lock()
|
||||
defer s.hotMu.Unlock()
|
||||
closed := make([]hotRollup, 0)
|
||||
for key, bucket := range s.hot {
|
||||
if key.bucket < currentBucket {
|
||||
_, replace := s.hotReplace[key]
|
||||
closed = append(closed, hotRollup{key: key, bucket: bucket, replace: replace})
|
||||
delete(s.hot, key)
|
||||
delete(s.hotReplace, key)
|
||||
}
|
||||
}
|
||||
return closed
|
||||
}
|
||||
|
||||
func (s *Store) restoreHotRollups(closed []hotRollup) {
|
||||
s.hotMu.Lock()
|
||||
defer s.hotMu.Unlock()
|
||||
for _, item := range closed {
|
||||
current, exists := s.hot[item.key]
|
||||
_, currentReplaces := s.hotReplace[item.key]
|
||||
if !exists {
|
||||
s.hot[item.key] = item.bucket
|
||||
if item.replace {
|
||||
s.hotReplace[item.key] = struct{}{}
|
||||
}
|
||||
continue
|
||||
}
|
||||
if currentReplaces {
|
||||
continue
|
||||
}
|
||||
if item.replace {
|
||||
item.bucket.mergeStored(current)
|
||||
s.hot[item.key] = item.bucket
|
||||
s.hotReplace[item.key] = struct{}{}
|
||||
continue
|
||||
}
|
||||
current.mergeStored(item.bucket)
|
||||
}
|
||||
}
|
||||
|
||||
func (s *Store) persistHotRollups(ctx context.Context, closed []hotRollup) error {
|
||||
type metricBuckets struct {
|
||||
merge map[rollupKey]*rollupBucket
|
||||
replace map[rollupKey]*rollupBucket
|
||||
}
|
||||
byMetric := make(map[string]*metricBuckets)
|
||||
for _, item := range closed {
|
||||
buckets := byMetric[item.key.metricName]
|
||||
if buckets == nil {
|
||||
buckets = &metricBuckets{merge: make(map[rollupKey]*rollupBucket), replace: make(map[rollupKey]*rollupBucket)}
|
||||
byMetric[item.key.metricName] = buckets
|
||||
}
|
||||
key := rollupKey{entityID: item.key.entityID, tagsHash: item.key.tagsHash, labelsHash: item.key.labelsHash, bucket: item.key.bucket}
|
||||
if item.replace {
|
||||
buckets.replace[key] = item.bucket
|
||||
} else {
|
||||
buckets.merge[key] = item.bucket
|
||||
}
|
||||
}
|
||||
names := make([]string, 0, len(byMetric))
|
||||
for name := range byMetric {
|
||||
names = append(names, name)
|
||||
}
|
||||
sort.Strings(names)
|
||||
tx, err := s.db.BeginTx(ctx, nil)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer func() { _ = tx.Rollback() }()
|
||||
for _, name := range names {
|
||||
buckets := byMetric[name]
|
||||
if _, err := s.writeTierCascadeTx(ctx, name, buckets.merge, tx); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := s.replaceMinuteRollupsTx(ctx, name, buckets.replace, tx); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return tx.Commit()
|
||||
}
|
||||
|
||||
func (s *Store) hotRollupRows(metricName, entityID string, tags map[string]string, start, end time.Time, needDigest bool) ([]storedRollup, error) {
|
||||
startMilli, endMilli := start.UTC().UnixMilli(), end.UTC().UnixMilli()
|
||||
s.hotMu.RLock()
|
||||
defer s.hotMu.RUnlock()
|
||||
out := make([]storedRollup, 0)
|
||||
for key, bucket := range s.hot {
|
||||
if key.metricName != metricName || key.bucket+time.Minute.Milliseconds() <= startMilli || key.bucket > endMilli || (entityID != "" && key.entityID != entityID) {
|
||||
continue
|
||||
}
|
||||
if bucket.count == 0 {
|
||||
continue
|
||||
}
|
||||
bucketTags, err := rollupTagsFromJSON(bucket.tagsJSON)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
matched := true
|
||||
for name, value := range tags {
|
||||
if bucketTags[name] != value {
|
||||
matched = false
|
||||
break
|
||||
}
|
||||
}
|
||||
if !matched {
|
||||
continue
|
||||
}
|
||||
out = append(out, storedRollup{entityID: key.entityID, bucket: key.bucket, bucketData: bucket.clone(needDigest)})
|
||||
}
|
||||
sort.Slice(out, func(i, j int) bool { return out[i].bucket < out[j].bucket })
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// deleteHotRollups removes active minute buckets matched by the same series
|
||||
// dimensions used by persisted deletes. A non-nil cutoff removes only buckets
|
||||
// whose minute starts before it.
|
||||
func (s *Store) deleteHotRollups(metricName, entityID string, tags map[string]string, cutoff *time.Time) (int64, error) {
|
||||
s.hotMu.Lock()
|
||||
defer s.hotMu.Unlock()
|
||||
|
||||
var cutoffMilli int64
|
||||
if cutoff != nil {
|
||||
cutoffMilli = cutoff.UTC().UnixMilli()
|
||||
}
|
||||
matched := make([]hotRollupKey, 0)
|
||||
for key, bucket := range s.hot {
|
||||
if metricName != "" && key.metricName != metricName {
|
||||
continue
|
||||
}
|
||||
if entityID != "" && key.entityID != entityID {
|
||||
continue
|
||||
}
|
||||
if cutoff != nil && key.bucket >= cutoffMilli {
|
||||
continue
|
||||
}
|
||||
if len(tags) > 0 {
|
||||
bucketTags, err := rollupTagsFromJSON(bucket.tagsJSON)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
matches := true
|
||||
for name, value := range tags {
|
||||
if bucketTags[name] != value {
|
||||
matches = false
|
||||
break
|
||||
}
|
||||
}
|
||||
if !matches {
|
||||
continue
|
||||
}
|
||||
}
|
||||
matched = append(matched, key)
|
||||
}
|
||||
for _, key := range matched {
|
||||
delete(s.hot, key)
|
||||
delete(s.hotReplace, key)
|
||||
}
|
||||
return int64(len(matched)), nil
|
||||
}
|
||||
@@ -0,0 +1,771 @@
|
||||
package metric
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestWriteBatchKeepsRawInMemoryAndBuildsMinuteRollup(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
s, err := Open(ctx, SQLite(":memory:", WithRollupPolicy(RollupPolicy{
|
||||
RawRetention: time.Minute,
|
||||
Tiers: []RollupTier{{Interval: time.Minute, Retention: time.Hour}},
|
||||
Compression: 30,
|
||||
})))
|
||||
if err != nil {
|
||||
t.Fatalf("open store: %v", err)
|
||||
}
|
||||
defer s.Close()
|
||||
if err := s.CreateMetric(ctx, Definition{Name: "hot", RetentionDays: 1}); err != nil {
|
||||
t.Fatalf("create metric: %v", err)
|
||||
}
|
||||
base := time.Now().UTC().Truncate(time.Minute)
|
||||
if err := s.WriteBatch(ctx, []Point{
|
||||
{MetricName: "hot", EntityID: "n1", Timestamp: base.Add(10 * time.Second), Value: 10},
|
||||
{MetricName: "hot", EntityID: "n1", Timestamp: base.Add(20 * time.Second), Value: 20},
|
||||
}); err != nil {
|
||||
t.Fatalf("write samples: %v", err)
|
||||
}
|
||||
var rawTableCount int
|
||||
if err := s.db.QueryRowContext(ctx, `SELECT COUNT(*) FROM sqlite_master WHERE type = 'table' AND name = ?`, s.tables.points).Scan(&rawTableCount); err != nil {
|
||||
t.Fatalf("inspect points table: %v", err)
|
||||
}
|
||||
if rawTableCount != 0 {
|
||||
t.Fatalf("metric_points table count = %d, want 0", rawTableCount)
|
||||
}
|
||||
points, err := s.Query(ctx, Query{MetricName: "hot", EntityID: "n1", Start: base, End: base.Add(time.Minute)})
|
||||
if err != nil {
|
||||
t.Fatalf("query minute rollup: %v", err)
|
||||
}
|
||||
if len(points) != 2 || points[0].Value != 10 || points[1].Value != 20 {
|
||||
t.Fatalf("points = %#v, want both raw samples", points)
|
||||
}
|
||||
series, err := s.Series(ctx, AggregateQuery{
|
||||
Query: Query{MetricName: "hot", EntityID: "n1", Start: base, End: base.Add(time.Minute)},
|
||||
Aggregation: AggAvg, Interval: time.Minute, PreserveSeries: true,
|
||||
}, base.Add(2*time.Minute))
|
||||
if err != nil {
|
||||
t.Fatalf("query minute rollup: %v", err)
|
||||
}
|
||||
if len(series) != 1 || series[0].Count != 2 || series[0].Value != 15 {
|
||||
t.Fatalf("minute rollup = %#v, want avg=15 count=2", series)
|
||||
}
|
||||
}
|
||||
|
||||
func TestClosedMinutePersistsAndAcceptsExactUpsert(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
s, err := Open(ctx, SQLite(":memory:", WithRollupPolicy(RollupPolicy{
|
||||
RawRetention: time.Minute,
|
||||
Tiers: []RollupTier{{Interval: time.Minute, Retention: time.Hour}},
|
||||
Compression: 30,
|
||||
})))
|
||||
if err != nil {
|
||||
t.Fatalf("open store: %v", err)
|
||||
}
|
||||
defer s.Close()
|
||||
if err := s.CreateMetric(ctx, Definition{Name: "late-upsert", RetentionDays: 1}); err != nil {
|
||||
t.Fatalf("create metric: %v", err)
|
||||
}
|
||||
|
||||
now := time.Date(2026, 7, 27, 12, 0, 10, 0, time.UTC)
|
||||
at := now.Add(-20 * time.Second)
|
||||
for _, value := range []float64{1, 2} {
|
||||
prepared, err := prepareMetricPoints([]Point{{MetricName: "late-upsert", EntityID: "n1", Timestamp: at, Value: value}})
|
||||
if err != nil {
|
||||
t.Fatalf("prepare point: %v", err)
|
||||
}
|
||||
rebuild := s.writeRawPointsAt(prepared, now)
|
||||
if err := s.writePreparedHotRollups(ctx, prepared, now, rebuild); err != nil {
|
||||
t.Fatalf("write hot rollup: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
var persisted int
|
||||
if err := s.db.QueryRowContext(ctx, "SELECT COUNT(*) FROM "+s.tables.rollups).Scan(&persisted); err != nil {
|
||||
t.Fatalf("count early persisted rollups: %v", err)
|
||||
}
|
||||
if persisted != 1 {
|
||||
t.Fatalf("persisted closed-minute rollups = %d, want 1", persisted)
|
||||
}
|
||||
series, err := s.Series(ctx, AggregateQuery{
|
||||
Query: Query{MetricName: "late-upsert", EntityID: "n1", Start: at.Add(-time.Minute), End: now},
|
||||
Aggregation: AggCount, Interval: time.Minute, PreserveSeries: true,
|
||||
}, now)
|
||||
if err != nil {
|
||||
t.Fatalf("query hot rollup: %v", err)
|
||||
}
|
||||
if len(series) != 1 || series[0].Count != 1 || series[0].Value != 1 {
|
||||
t.Fatalf("late upsert was counted twice: %#v", series)
|
||||
}
|
||||
|
||||
var count int
|
||||
if err := s.db.QueryRowContext(ctx, "SELECT count FROM "+s.tables.rollups).Scan(&count); err != nil {
|
||||
t.Fatalf("read persisted rollup: %v", err)
|
||||
}
|
||||
if count != 1 {
|
||||
t.Fatalf("persisted rollup count = %d, want 1", count)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRawWindowKeepsCompressedAndDirectSamples(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
now := time.Now().UTC()
|
||||
s, err := Open(ctx, SQLite(":memory:", WithRollupPolicy(RollupPolicy{
|
||||
RawRetention: time.Minute,
|
||||
Tiers: []RollupTier{{Interval: time.Minute, Retention: time.Hour}},
|
||||
Compression: 30,
|
||||
})))
|
||||
if err != nil {
|
||||
t.Fatalf("open store: %v", err)
|
||||
}
|
||||
defer s.Close()
|
||||
if err := s.CreateMetric(ctx, Definition{Name: "raw-window", RetentionDays: 1}); err != nil {
|
||||
t.Fatalf("create metric: %v", err)
|
||||
}
|
||||
if err := s.WriteBatch(ctx, []Point{
|
||||
{MetricName: "raw-window", EntityID: "n1", Timestamp: now.Add(-70 * time.Second), Value: 1},
|
||||
{MetricName: "raw-window", EntityID: "n1", Timestamp: now.Add(-50 * time.Second), Value: 2},
|
||||
{MetricName: "raw-window", EntityID: "n1", Timestamp: now.Add(-40 * time.Second), Value: 3},
|
||||
}); err != nil {
|
||||
t.Fatalf("write samples: %v", err)
|
||||
}
|
||||
if _, err := s.Compact(ctx, now); err != nil {
|
||||
t.Fatalf("compact: %v", err)
|
||||
}
|
||||
raw, err := s.Query(ctx, Query{MetricName: "raw-window", EntityID: "n1", Start: now.Add(-time.Hour), End: now})
|
||||
if err != nil {
|
||||
t.Fatalf("query raw: %v", err)
|
||||
}
|
||||
if len(raw) != 3 || raw[0].Value != 1 || raw[1].Value != 2 || raw[2].Value != 3 {
|
||||
t.Fatalf("retained raw = %#v, want all ten-minute samples", raw)
|
||||
}
|
||||
series, err := s.Series(ctx, AggregateQuery{
|
||||
Query: Query{MetricName: "raw-window", EntityID: "n1", Start: now.Add(-time.Hour), End: now},
|
||||
Aggregation: AggCount, Interval: time.Minute, PreserveSeries: true,
|
||||
}, now)
|
||||
if err != nil {
|
||||
t.Fatalf("query rollup: %v", err)
|
||||
}
|
||||
var count int
|
||||
for _, point := range series {
|
||||
count += point.Count
|
||||
}
|
||||
if count != 3 {
|
||||
t.Fatalf("rollups lost samples after raw cleanup: %#v", series)
|
||||
}
|
||||
}
|
||||
|
||||
func TestOldestRawMinuteRebuildKeepsHiddenPrefix(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
policy := RollupPolicy{
|
||||
RawRetention: rawMemoryRetention,
|
||||
Tiers: []RollupTier{{Interval: time.Minute, Retention: time.Hour}},
|
||||
Compression: 30,
|
||||
}
|
||||
s, err := Open(ctx, SQLite(":memory:", WithRollupPolicy(policy)))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer s.Close()
|
||||
if err := s.CreateMetric(ctx, Definition{Name: "oldest-minute", RetentionDays: 1}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
now := time.Date(2026, 7, 27, 12, 10, 30, 0, time.UTC)
|
||||
oldestMinute := now.Add(-rawMemoryRetention).Truncate(time.Minute)
|
||||
write := func(points ...Point) {
|
||||
t.Helper()
|
||||
prepared, err := prepareMetricPoints(points)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
rebuild := s.writeRawPointsAt(prepared, now)
|
||||
if err := s.writePreparedHotRollups(ctx, prepared, now, rebuild); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
write(
|
||||
Point{MetricName: "oldest-minute", EntityID: "n1", Timestamp: oldestMinute.Add(5 * time.Second), Value: 1},
|
||||
Point{MetricName: "oldest-minute", EntityID: "n1", Timestamp: oldestMinute.Add(40 * time.Second), Value: 2},
|
||||
)
|
||||
write(Point{MetricName: "oldest-minute", EntityID: "n1", Timestamp: oldestMinute.Add(40 * time.Second), Value: 4})
|
||||
|
||||
rows, err := s.scanRollupRowsBetween(ctx, "oldest-minute", "n1", nil, time.Minute.Milliseconds(), oldestMinute.UnixMilli(), oldestMinute.UnixMilli(), false)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(rows) != 1 || rows[0].bucketData.count != 2 || rows[0].bucketData.sum != 5 {
|
||||
t.Fatalf("rebuilt oldest minute = %#v, want count=2 sum=5", rows)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLateLabelReplacementRebuildsPersistedCascade(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
policy := RollupPolicy{
|
||||
RawRetention: rawMemoryRetention,
|
||||
Tiers: []RollupTier{
|
||||
{Interval: time.Minute, Retention: 24 * time.Hour},
|
||||
{Interval: 5 * time.Minute, Retention: 7 * 24 * time.Hour},
|
||||
{Interval: time.Hour, Retention: 30 * 24 * time.Hour},
|
||||
{Interval: 24 * time.Hour, Retention: 365 * 24 * time.Hour},
|
||||
},
|
||||
Compression: 30,
|
||||
}
|
||||
s, err := Open(ctx, SQLite(":memory:", WithRollupPolicy(policy)))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer s.Close()
|
||||
if err := s.CreateMetric(ctx, Definition{Name: "cascade-replace", RetentionDays: 30}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
now := time.Date(2026, 7, 27, 12, 2, 10, 0, time.UTC)
|
||||
at := now.Add(-90 * time.Second)
|
||||
write := func(point Point) {
|
||||
t.Helper()
|
||||
prepared, err := prepareMetricPoints([]Point{point})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
rebuild := s.writeRawPointsAt(prepared, now)
|
||||
if err := s.writePreparedHotRollups(ctx, prepared, now, rebuild); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
write(Point{MetricName: "cascade-replace", EntityID: "n1", Timestamp: at, Value: 1, Labels: map[string]string{"source": "old"}})
|
||||
write(Point{MetricName: "cascade-replace", EntityID: "n1", Timestamp: at, Value: 7, Labels: map[string]string{"source": "new"}})
|
||||
|
||||
for _, tier := range policy.Tiers {
|
||||
start := bucketStartMillis(at.UnixMilli(), tier.Interval.Milliseconds())
|
||||
rows, err := s.scanRollupRowsBetween(ctx, "cascade-replace", "n1", nil, tier.Interval.Milliseconds(), start, start, true)
|
||||
if err != nil {
|
||||
t.Fatalf("scan %s tier: %v", tier.Interval, err)
|
||||
}
|
||||
if len(rows) != 1 || rows[0].bucketData.count != 1 || rows[0].bucketData.sum != 7 {
|
||||
t.Fatalf("%s replacement rows = %#v, want one count=1 sum=7 row", tier.Interval, rows)
|
||||
}
|
||||
labels, err := decodeMapString(rows[0].bucketData.labelsJSON)
|
||||
if err != nil || labels["source"] != "new" {
|
||||
t.Fatalf("%s replacement labels = %#v, %v", tier.Interval, labels, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestFlushTrimsInactiveRawWindow(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
s := newMemStore(t)
|
||||
if err := s.CreateMetric(ctx, Definition{Name: "idle-raw", RetentionDays: 1}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
now := time.Now().UTC()
|
||||
prepared, err := prepareMetricPoints([]Point{{MetricName: "idle-raw", EntityID: "n1", Timestamp: now.Add(-time.Minute), Value: 1}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
rebuild := s.writeRawPointsAt(prepared, now)
|
||||
if err := s.writePreparedHotRollups(ctx, prepared, now, rebuild); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := s.Flush(ctx, now.Add(12*time.Minute)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
s.rawMu.RLock()
|
||||
defer s.rawMu.RUnlock()
|
||||
if len(s.raw) != 0 {
|
||||
t.Fatalf("inactive raw series retained after scheduled flush: %#v", s.raw)
|
||||
}
|
||||
}
|
||||
|
||||
func TestConcurrentHotPercentileQueriesUseDigestSnapshots(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
s := newMemStore(t)
|
||||
if err := s.CreateMetric(ctx, Definition{Name: "hot-percentile", RetentionDays: 1}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
base := time.Now().UTC()
|
||||
if err := s.Write(ctx, Point{MetricName: "hot-percentile", EntityID: "n1", Timestamp: base, Value: 1}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
errCh := make(chan error, 2)
|
||||
var workers sync.WaitGroup
|
||||
workers.Add(2)
|
||||
go func() {
|
||||
defer workers.Done()
|
||||
for i := 1; i <= 200; i++ {
|
||||
if err := s.Write(ctx, Point{MetricName: "hot-percentile", EntityID: "n1", Timestamp: base.Add(time.Duration(i) * time.Millisecond), Value: float64(i)}); err != nil {
|
||||
errCh <- err
|
||||
return
|
||||
}
|
||||
}
|
||||
}()
|
||||
go func() {
|
||||
defer workers.Done()
|
||||
for i := 0; i < 200; i++ {
|
||||
points, err := s.Series(ctx, AggregateQuery{
|
||||
Query: Query{MetricName: "hot-percentile", EntityID: "n1", Start: base.Add(-time.Minute), End: base.Add(time.Minute)},
|
||||
Aggregation: AggP95, Interval: time.Minute, PreserveSeries: true,
|
||||
}, base)
|
||||
if err != nil {
|
||||
errCh <- err
|
||||
return
|
||||
}
|
||||
if len(points) == 0 {
|
||||
errCh <- fmt.Errorf("percentile query returned no points")
|
||||
return
|
||||
}
|
||||
}
|
||||
}()
|
||||
workers.Wait()
|
||||
close(errCh)
|
||||
for err := range errCh {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestConcurrentPersistedReplacementQueriesStayConsistent(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
policy := RollupPolicy{
|
||||
RawRetention: rawMemoryRetention,
|
||||
Tiers: []RollupTier{{Interval: time.Minute, Retention: time.Hour}},
|
||||
Compression: 30,
|
||||
}
|
||||
s, err := Open(ctx, SQLite(":memory:", WithRollupPolicy(policy)))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer s.Close()
|
||||
if err := s.CreateMetric(ctx, Definition{Name: "replacement-view", RetentionDays: 1}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
now := time.Now().UTC()
|
||||
at := now.Add(-90 * time.Second)
|
||||
if err := s.Write(ctx, Point{MetricName: "replacement-view", EntityID: "n1", Timestamp: at, Value: 1}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
errCh := make(chan error, 2)
|
||||
var workers sync.WaitGroup
|
||||
workers.Add(2)
|
||||
go func() {
|
||||
defer workers.Done()
|
||||
for i := 2; i <= 100; i++ {
|
||||
if err := s.Write(ctx, Point{MetricName: "replacement-view", EntityID: "n1", Timestamp: at, Value: float64(i)}); err != nil {
|
||||
errCh <- err
|
||||
return
|
||||
}
|
||||
}
|
||||
}()
|
||||
go func() {
|
||||
defer workers.Done()
|
||||
for i := 0; i < 300; i++ {
|
||||
points, err := s.Series(ctx, AggregateQuery{
|
||||
Query: Query{MetricName: "replacement-view", EntityID: "n1", Start: at.Add(-time.Minute), End: now},
|
||||
Aggregation: AggCount, Interval: time.Minute, PreserveSeries: true,
|
||||
}, now)
|
||||
if err != nil {
|
||||
errCh <- err
|
||||
return
|
||||
}
|
||||
if len(points) != 1 || points[0].Count != 1 || points[0].Value != 1 {
|
||||
errCh <- fmt.Errorf("replacement query observed a split view: %#v", points)
|
||||
return
|
||||
}
|
||||
}
|
||||
}()
|
||||
workers.Wait()
|
||||
close(errCh)
|
||||
for err := range errCh {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRawQueryOffsetWithoutLimit(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
s := newMemStore(t)
|
||||
if err := s.CreateMetric(ctx, Definition{Name: "raw-page", RetentionDays: 1}); err != nil {
|
||||
t.Fatalf("create metric: %v", err)
|
||||
}
|
||||
base := time.Now().UTC().Truncate(time.Second)
|
||||
if err := s.WriteBatch(ctx, []Point{
|
||||
{MetricName: "raw-page", EntityID: "n1", Timestamp: base, Value: 1},
|
||||
{MetricName: "raw-page", EntityID: "n1", Timestamp: base.Add(time.Second), Value: 2},
|
||||
{MetricName: "raw-page", EntityID: "n1", Timestamp: base.Add(2 * time.Second), Value: 3},
|
||||
}); err != nil {
|
||||
t.Fatalf("write samples: %v", err)
|
||||
}
|
||||
points, err := s.Query(ctx, Query{MetricName: "raw-page", Start: base, End: base.Add(2 * time.Second), Offset: 1})
|
||||
if err != nil {
|
||||
t.Fatalf("query offset: %v", err)
|
||||
}
|
||||
if len(points) != 2 || points[0].Value != 2 || points[1].Value != 3 {
|
||||
t.Fatalf("offset raw points = %#v", points)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSeriesMergesLabelsWithoutCollapsingRawShape(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
s, err := Open(ctx, SQLite(":memory:", WithRollupPolicy(RollupPolicy{
|
||||
Tiers: []RollupTier{{Interval: time.Minute, Retention: time.Hour}},
|
||||
Compression: 30,
|
||||
})))
|
||||
if err != nil {
|
||||
t.Fatalf("open store: %v", err)
|
||||
}
|
||||
defer s.Close()
|
||||
if err := s.CreateMetric(ctx, Definition{Name: "labels", RetentionDays: 1}); err != nil {
|
||||
t.Fatalf("create metric: %v", err)
|
||||
}
|
||||
base := time.Now().UTC().Add(-30 * time.Second)
|
||||
if err := s.WriteBatch(ctx, []Point{
|
||||
{MetricName: "labels", EntityID: "n1", Timestamp: base.Add(10 * time.Second), Value: 10, Tags: map[string]string{"disk": "sda"}, Labels: map[string]string{"source": "a"}},
|
||||
{MetricName: "labels", EntityID: "n1", Timestamp: base.Add(20 * time.Second), Value: 20, Tags: map[string]string{"disk": "sda"}, Labels: map[string]string{"source": "b"}},
|
||||
}); err != nil {
|
||||
t.Fatalf("write labeled samples: %v", err)
|
||||
}
|
||||
|
||||
series, err := s.Series(ctx, AggregateQuery{
|
||||
Query: Query{MetricName: "labels", EntityID: "n1", Start: base, End: base.Add(time.Minute)},
|
||||
Aggregation: AggAvg,
|
||||
Interval: time.Minute,
|
||||
PreserveSeries: true,
|
||||
}, base.Add(2*time.Minute))
|
||||
if err != nil {
|
||||
t.Fatalf("aggregate labels: %v", err)
|
||||
}
|
||||
var count int
|
||||
var sum float64
|
||||
for _, point := range series {
|
||||
count += point.Count
|
||||
sum += point.Value * float64(point.Count)
|
||||
}
|
||||
if count != 2 || sum/float64(count) != 15 {
|
||||
t.Fatalf("label sets split the public series: %#v", series)
|
||||
}
|
||||
|
||||
points, err := s.Query(ctx, Query{MetricName: "labels", EntityID: "n1", Start: base, End: base.Add(time.Minute)})
|
||||
if err != nil {
|
||||
t.Fatalf("query labeled raw samples: %v", err)
|
||||
}
|
||||
if len(points) != 2 {
|
||||
t.Fatalf("query lost independently indexed label sets: %#v", points)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCoveringSeriesResolutionUsesFinestTierCoveringStart(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
policy := RollupPolicy{
|
||||
Tiers: []RollupTier{
|
||||
{Interval: time.Minute, Retention: 10 * time.Hour},
|
||||
{Interval: 5 * time.Minute, Retention: 50 * time.Hour},
|
||||
{Interval: time.Hour, Retention: 25 * 24 * time.Hour},
|
||||
{Interval: 24 * time.Hour, Retention: 100 * 365 * 24 * time.Hour},
|
||||
},
|
||||
Compression: 30,
|
||||
}
|
||||
s, err := Open(ctx, SQLite(":memory:", WithRollupPolicy(policy)))
|
||||
if err != nil {
|
||||
t.Fatalf("open store: %v", err)
|
||||
}
|
||||
defer s.Close()
|
||||
if err := s.CreateMetric(ctx, Definition{Name: "coverage-tier", RetentionDays: 60}); err != nil {
|
||||
t.Fatalf("create metric: %v", err)
|
||||
}
|
||||
|
||||
tagsHash, tagsJSON, err := tagsFingerprint(nil)
|
||||
if err != nil {
|
||||
t.Fatalf("fingerprint tags: %v", err)
|
||||
}
|
||||
now := time.Date(2026, 7, 24, 12, 0, 0, 0, time.UTC)
|
||||
tx, err := s.db.BeginTx(ctx, nil)
|
||||
if err != nil {
|
||||
t.Fatalf("begin seed transaction: %v", err)
|
||||
}
|
||||
for _, seed := range []struct {
|
||||
interval time.Duration
|
||||
at time.Time
|
||||
}{
|
||||
{interval: time.Minute, at: now.Add(-30 * time.Minute)},
|
||||
{interval: 5 * time.Minute, at: now.Add(-7 * time.Hour)},
|
||||
{interval: 5 * time.Minute, at: now.Add(-5 * time.Hour)},
|
||||
{interval: time.Hour, at: now.Add(-30 * 24 * time.Hour)},
|
||||
} {
|
||||
bucket := newRollupBucket(policy.compression())
|
||||
bucket.tagsHash, bucket.tagsJSON = tagsHash, tagsJSON
|
||||
bucket.labelsHash, bucket.labelsJSON = emptyLabelsHash, "{}"
|
||||
bucket.addPoint(1, seed.at.UnixMilli())
|
||||
key := rollupKey{entityID: "n1", tagsHash: tagsHash, labelsHash: emptyLabelsHash, bucket: bucketStartMillis(seed.at.UnixMilli(), seed.interval.Milliseconds())}
|
||||
if _, err := s.writeRollupBucketsTx(ctx, "coverage-tier", seed.interval, map[rollupKey]*rollupBucket{key: bucket}, tx); err != nil {
|
||||
_ = tx.Rollback()
|
||||
t.Fatalf("seed %s tier: %v", seed.interval, err)
|
||||
}
|
||||
}
|
||||
if err := tx.Commit(); err != nil {
|
||||
t.Fatalf("commit seed tiers: %v", err)
|
||||
}
|
||||
|
||||
resolution, err := s.coveringSeriesResolution(ctx, Query{MetricName: "coverage-tier", EntityID: "n1", Start: now.Add(-6 * time.Hour), End: now}, time.Minute)
|
||||
if err != nil {
|
||||
t.Fatalf("select covering tier: %v", err)
|
||||
}
|
||||
if resolution != 5*time.Minute {
|
||||
t.Fatalf("covering tier = %s, want 5m", resolution)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSeriesFallsBackWhenPreferredPolicyTierHasNoRows(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
policy := RollupPolicy{
|
||||
Tiers: []RollupTier{
|
||||
{Interval: time.Minute, Retention: 10 * time.Hour},
|
||||
{Interval: time.Hour, Retention: 25 * 24 * time.Hour},
|
||||
},
|
||||
Compression: 30,
|
||||
}
|
||||
s, err := Open(ctx, SQLite(":memory:", WithRollupPolicy(policy)))
|
||||
if err != nil {
|
||||
t.Fatalf("open store: %v", err)
|
||||
}
|
||||
defer s.Close()
|
||||
if err := s.CreateMetric(ctx, Definition{Name: "missing-fine-tier", RetentionDays: 60}); err != nil {
|
||||
t.Fatalf("create metric: %v", err)
|
||||
}
|
||||
now := time.Date(2026, 7, 24, 12, 30, 0, 0, time.UTC)
|
||||
at := now.Add(-30 * time.Minute)
|
||||
tagsHash, tagsJSON, err := tagsFingerprint(nil)
|
||||
if err != nil {
|
||||
t.Fatalf("fingerprint tags: %v", err)
|
||||
}
|
||||
bucket := newRollupBucket(policy.compression())
|
||||
bucket.tagsHash, bucket.tagsJSON = tagsHash, tagsJSON
|
||||
bucket.labelsHash, bucket.labelsJSON = emptyLabelsHash, "{}"
|
||||
bucket.addPoint(42, at.UnixMilli())
|
||||
key := rollupKey{entityID: "n1", tagsHash: tagsHash, labelsHash: emptyLabelsHash, bucket: bucketStartMillis(at.UnixMilli(), time.Hour.Milliseconds())}
|
||||
tx, err := s.db.BeginTx(ctx, nil)
|
||||
if err != nil {
|
||||
t.Fatalf("begin seed transaction: %v", err)
|
||||
}
|
||||
if _, err := s.writeRollupBucketsTx(ctx, "missing-fine-tier", time.Hour, map[rollupKey]*rollupBucket{key: bucket}, tx); err != nil {
|
||||
_ = tx.Rollback()
|
||||
t.Fatalf("seed hourly tier: %v", err)
|
||||
}
|
||||
if err := tx.Commit(); err != nil {
|
||||
t.Fatalf("commit hourly tier: %v", err)
|
||||
}
|
||||
|
||||
got, err := s.Series(ctx, AggregateQuery{
|
||||
Query: Query{MetricName: "missing-fine-tier", EntityID: "n1", Start: now.Add(-time.Hour), End: now},
|
||||
Aggregation: AggAvg,
|
||||
Interval: time.Minute,
|
||||
}, now)
|
||||
if err != nil {
|
||||
t.Fatalf("query series: %v", err)
|
||||
}
|
||||
if len(got) != 1 || got[0].Value != 42 || got[0].Count != 1 {
|
||||
t.Fatalf("series did not fall back to existing hourly tier: %#v", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSeriesDoesNotFallBackWhenPreferredTierCoversWindow(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
policy := RollupPolicy{
|
||||
Tiers: []RollupTier{
|
||||
{Interval: time.Minute, Retention: 10 * time.Hour},
|
||||
{Interval: 24 * time.Hour, Retention: 100 * 365 * 24 * time.Hour},
|
||||
},
|
||||
Compression: 30,
|
||||
}
|
||||
s, err := Open(ctx, SQLite(":memory:", WithRollupPolicy(policy)))
|
||||
if err != nil {
|
||||
t.Fatalf("open store: %v", err)
|
||||
}
|
||||
defer s.Close()
|
||||
if err := s.CreateMetric(ctx, Definition{Name: "new-series", RetentionDays: 60}); err != nil {
|
||||
t.Fatalf("create metric: %v", err)
|
||||
}
|
||||
|
||||
now := time.Date(2026, 7, 27, 3, 0, 0, 0, time.UTC)
|
||||
tagsHash, tagsJSON, err := tagsFingerprint(nil)
|
||||
if err != nil {
|
||||
t.Fatalf("fingerprint tags: %v", err)
|
||||
}
|
||||
tx, err := s.db.BeginTx(ctx, nil)
|
||||
if err != nil {
|
||||
t.Fatalf("begin seed transaction: %v", err)
|
||||
}
|
||||
for _, seed := range []struct {
|
||||
interval time.Duration
|
||||
at time.Time
|
||||
value float64
|
||||
}{
|
||||
{interval: 24 * time.Hour, at: now.Add(-30 * 24 * time.Hour), value: 10},
|
||||
{interval: time.Minute, at: now.Add(-5 * time.Minute), value: 20},
|
||||
} {
|
||||
bucket := newRollupBucket(policy.compression())
|
||||
bucket.tagsHash, bucket.tagsJSON = tagsHash, tagsJSON
|
||||
bucket.labelsHash, bucket.labelsJSON = emptyLabelsHash, "{}"
|
||||
bucket.addPoint(seed.value, seed.at.UnixMilli())
|
||||
key := rollupKey{entityID: "n1", tagsHash: tagsHash, labelsHash: emptyLabelsHash, bucket: bucketStartMillis(seed.at.UnixMilli(), seed.interval.Milliseconds())}
|
||||
if _, err := s.writeRollupBucketsTx(ctx, "new-series", seed.interval, map[rollupKey]*rollupBucket{key: bucket}, tx); err != nil {
|
||||
_ = tx.Rollback()
|
||||
t.Fatalf("seed %s tier: %v", seed.interval, err)
|
||||
}
|
||||
}
|
||||
if err := tx.Commit(); err != nil {
|
||||
t.Fatalf("commit seed tiers: %v", err)
|
||||
}
|
||||
|
||||
got, err := s.Series(ctx, AggregateQuery{
|
||||
Query: Query{MetricName: "new-series", EntityID: "n1", Start: now.Add(-time.Hour), End: now},
|
||||
Aggregation: AggAvg,
|
||||
Interval: time.Minute,
|
||||
}, now)
|
||||
if err != nil {
|
||||
t.Fatalf("query recent series: %v", err)
|
||||
}
|
||||
if len(got) != 1 || got[0].Bucket.Before(now.Add(-time.Hour)) || got[0].Value != 20 {
|
||||
t.Fatalf("recent query fell back to an older coarse tier: %#v", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDeleteBeforeKeepsBucketsThatStraddleCutoff(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
policy := RollupPolicy{
|
||||
Tiers: []RollupTier{
|
||||
{Interval: time.Minute, Retention: 10 * time.Hour},
|
||||
{Interval: 5 * time.Minute, Retention: 50 * time.Hour},
|
||||
{Interval: time.Hour, Retention: 25 * 24 * time.Hour},
|
||||
{Interval: 24 * time.Hour, Retention: 100 * 365 * 24 * time.Hour},
|
||||
},
|
||||
Compression: 30,
|
||||
}
|
||||
s, err := Open(ctx, SQLite(":memory:", WithRollupPolicy(policy)))
|
||||
if err != nil {
|
||||
t.Fatalf("open store: %v", err)
|
||||
}
|
||||
defer s.Close()
|
||||
if err := s.CreateMetric(ctx, Definition{Name: "retention-boundary", RetentionDays: 60}); err != nil {
|
||||
t.Fatalf("create metric: %v", err)
|
||||
}
|
||||
day := time.Now().UTC().Truncate(24 * time.Hour).Add(-48 * time.Hour)
|
||||
if err := s.WriteBatch(ctx, []Point{
|
||||
{MetricName: "retention-boundary", EntityID: "n1", Timestamp: day.Add(12 * time.Hour), Value: 10},
|
||||
{MetricName: "retention-boundary", EntityID: "n1", Timestamp: day.Add(20 * time.Hour), Value: 20},
|
||||
}); err != nil {
|
||||
t.Fatalf("write daily boundary points: %v", err)
|
||||
}
|
||||
if _, err := s.DeleteBefore(ctx, "retention-boundary", day.Add(18*time.Hour)); err != nil {
|
||||
t.Fatalf("delete before cutoff: %v", err)
|
||||
}
|
||||
var dailyCount int64
|
||||
if err := s.db.QueryRowContext(ctx, fmt.Sprintf(`SELECT r.count FROM %s r
|
||||
JOIN %s series ON series.id = r.series_id
|
||||
JOIN %s resolutions ON resolutions.id = r.resolution_id
|
||||
WHERE series.metric_name = ? AND resolutions.resolution_milli = ?`,
|
||||
s.tables.rollups, s.tables.series, s.tables.resolutions), "retention-boundary", (24 * time.Hour).Milliseconds()).Scan(&dailyCount); err != nil {
|
||||
t.Fatalf("read straddling daily bucket: %v", err)
|
||||
}
|
||||
if dailyCount != 2 {
|
||||
t.Fatalf("straddling daily bucket count = %d, want 2", dailyCount)
|
||||
}
|
||||
|
||||
futureMinute := time.Now().UTC().Truncate(time.Minute).Add(2 * time.Minute)
|
||||
if err := s.Write(ctx, Point{MetricName: "retention-boundary", EntityID: "n1", Timestamp: futureMinute.Add(40 * time.Second), Value: 30}); err != nil {
|
||||
t.Fatalf("write active boundary point: %v", err)
|
||||
}
|
||||
if _, err := s.DeleteBefore(ctx, "retention-boundary", futureMinute.Add(30*time.Second)); err != nil {
|
||||
t.Fatalf("delete before active-minute cutoff: %v", err)
|
||||
}
|
||||
if len(s.hot) != 1 {
|
||||
t.Fatalf("active minute was deleted across a partial cutoff: %#v", s.hot)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCompatibleIntervalsCoverDashboardRanges(t *testing.T) {
|
||||
now := time.Date(2026, 7, 24, 12, 0, 0, 0, time.UTC)
|
||||
policy := RollupPolicy{
|
||||
RawRetention: time.Minute,
|
||||
Tiers: []RollupTier{
|
||||
{Interval: time.Minute, Retention: 10 * time.Hour},
|
||||
{Interval: 5 * time.Minute, Retention: 50 * time.Hour},
|
||||
{Interval: time.Hour, Retention: 25 * 24 * time.Hour},
|
||||
{Interval: 24 * time.Hour, Retention: 100 * 365 * 24 * time.Hour},
|
||||
},
|
||||
}
|
||||
s := &Store{cfg: Config{RollupPolicy: policy}}
|
||||
for _, test := range []struct {
|
||||
rangeDuration time.Duration
|
||||
interval time.Duration
|
||||
}{
|
||||
{time.Hour, time.Minute},
|
||||
{6 * time.Hour, time.Minute},
|
||||
{7 * 24 * time.Hour, 15 * time.Minute},
|
||||
{30 * 24 * time.Hour, time.Hour},
|
||||
{60 * 24 * time.Hour, time.Hour},
|
||||
} {
|
||||
got := s.CompatibleSeriesInterval(now.Add(-test.rangeDuration), now, test.interval)
|
||||
if got <= 0 {
|
||||
t.Fatalf("range %s returned invalid interval %s", test.rangeDuration, got)
|
||||
}
|
||||
if got < time.Minute {
|
||||
t.Fatalf("range %s returned %s below persisted minute precision", test.rangeDuration, got)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestSeriesHasNoTrailingEmptyBucketAcrossDashboardRanges(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
policy := RollupPolicy{
|
||||
RawRetention: time.Minute,
|
||||
Tiers: []RollupTier{
|
||||
{Interval: time.Minute, Retention: 10 * time.Hour},
|
||||
{Interval: 5 * time.Minute, Retention: 50 * time.Hour},
|
||||
{Interval: time.Hour, Retention: 25 * 24 * time.Hour},
|
||||
{Interval: 24 * time.Hour, Retention: 100 * 365 * 24 * time.Hour},
|
||||
},
|
||||
Compression: 30,
|
||||
}
|
||||
s, err := Open(ctx, SQLite(":memory:", WithRollupPolicy(policy)))
|
||||
if err != nil {
|
||||
t.Fatalf("open store: %v", err)
|
||||
}
|
||||
defer s.Close()
|
||||
if err := s.CreateMetric(ctx, Definition{Name: "coverage", RetentionDays: 90}); err != nil {
|
||||
t.Fatalf("create metric: %v", err)
|
||||
}
|
||||
now := time.Date(2026, 7, 24, 12, 0, 0, 0, time.UTC)
|
||||
points := make([]Point, 0, 60*24*60)
|
||||
for at := now.Add(-60 * 24 * time.Hour); at.Before(now); at = at.Add(time.Minute) {
|
||||
points = append(points, Point{MetricName: "coverage", EntityID: "n1", Timestamp: at, Value: 1})
|
||||
}
|
||||
if err := s.WriteBatch(ctx, points); err != nil {
|
||||
t.Fatalf("write history: %v", err)
|
||||
}
|
||||
if _, err := s.Compact(ctx, now); err != nil {
|
||||
t.Fatalf("compact history: %v", err)
|
||||
}
|
||||
for _, duration := range []time.Duration{time.Hour, 6 * time.Hour, 7 * 24 * time.Hour, 30 * 24 * time.Hour, 60 * 24 * time.Hour} {
|
||||
interval := s.CompatibleSeriesInterval(now.Add(-duration), now, time.Minute)
|
||||
series, err := s.Series(ctx, AggregateQuery{
|
||||
Query: Query{MetricName: "coverage", EntityID: "n1", Start: now.Add(-duration), End: now},
|
||||
Aggregation: AggAvg,
|
||||
Interval: interval,
|
||||
}, now)
|
||||
if err != nil {
|
||||
t.Fatalf("query %s: %v", duration, err)
|
||||
}
|
||||
if len(series) == 0 {
|
||||
t.Fatalf("query %s returned no retained buckets", duration)
|
||||
}
|
||||
if series[len(series)-1].Count == 0 {
|
||||
t.Fatalf("query %s ended with an empty bucket: %#v", duration, series[len(series)-1])
|
||||
}
|
||||
for i := 1; i < len(series); i++ {
|
||||
if delta := series[i].Bucket.Sub(series[i-1].Bucket); delta != interval {
|
||||
t.Fatalf("query %s has a break between %s and %s: got %s, want %s", duration, series[i-1].Bucket, series[i].Bucket, delta, interval)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -28,7 +28,7 @@ func newMemStore(t *testing.T) *Store {
|
||||
func TestCreateMetricRejectsDuplicate(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
s := newMemStore(t)
|
||||
def := Definition{Name: "dup.metric", Type: TypeGauge}
|
||||
def := Definition{Name: "dup.metric", Type: TypeGauge, RetentionDays: 30}
|
||||
if err := s.CreateMetric(ctx, def); err != nil {
|
||||
t.Fatalf("first create: %v", err)
|
||||
}
|
||||
@@ -44,10 +44,10 @@ func TestCreateMetricRejectsDuplicate(t *testing.T) {
|
||||
func TestUpsertMetricOverwrites(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
s := newMemStore(t)
|
||||
if err := s.CreateMetric(ctx, Definition{Name: "m", Type: TypeGauge, Unit: "ms"}); err != nil {
|
||||
if err := s.CreateMetric(ctx, Definition{Name: "m", Type: TypeGauge, Unit: "ms", RetentionDays: 30}); err != nil {
|
||||
t.Fatalf("create: %v", err)
|
||||
}
|
||||
if err := s.UpsertMetric(ctx, Definition{Name: "m", Type: TypeCounter, Unit: "count"}); err != nil {
|
||||
if err := s.UpsertMetric(ctx, Definition{Name: "m", Type: TypeCounter, Unit: "count", RetentionDays: 30}); err != nil {
|
||||
t.Fatalf("upsert: %v", err)
|
||||
}
|
||||
got, err := s.GetMetric(ctx, "m")
|
||||
@@ -65,10 +65,10 @@ func TestUpsertMetricOverwrites(t *testing.T) {
|
||||
func TestTagFilterPushdownWithPaging(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
s := newMemStore(t)
|
||||
if err := s.CreateMetric(ctx, Definition{Name: "req", Type: TypeCounter}); err != nil {
|
||||
if err := s.CreateMetric(ctx, Definition{Name: "req", Type: TypeCounter, RetentionDays: 30}); err != nil {
|
||||
t.Fatalf("create: %v", err)
|
||||
}
|
||||
base := time.Date(2026, 6, 18, 0, 0, 0, 0, time.UTC)
|
||||
base := time.Now().UTC().Truncate(time.Minute)
|
||||
var batch []Point
|
||||
for i := 0; i < 6; i++ {
|
||||
env := "prod"
|
||||
@@ -77,7 +77,7 @@ func TestTagFilterPushdownWithPaging(t *testing.T) {
|
||||
}
|
||||
batch = append(batch, Point{
|
||||
MetricName: "req", EntityID: "n1",
|
||||
Timestamp: base.Add(time.Duration(i) * time.Minute),
|
||||
Timestamp: base.Add(time.Duration(i) * 5 * time.Second),
|
||||
Value: float64(i),
|
||||
Tags: map[string]string{"env": env},
|
||||
})
|
||||
@@ -89,7 +89,7 @@ func TestTagFilterPushdownWithPaging(t *testing.T) {
|
||||
// tag-filtered set, returning the first two prod points in time order.
|
||||
got, err := s.Query(ctx, Query{
|
||||
MetricName: "req", EntityID: "n1",
|
||||
Start: base, End: base.Add(time.Hour),
|
||||
Start: base, End: base.Add(time.Minute),
|
||||
Tags: map[string]string{"env": "prod"},
|
||||
Limit: 2,
|
||||
})
|
||||
@@ -110,16 +110,16 @@ func TestTagFilterPushdownWithPaging(t *testing.T) {
|
||||
func TestSQLAggregateMatchesInMemory(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
s := newMemStore(t)
|
||||
if err := s.CreateMetric(ctx, Definition{Name: "g", Type: TypeGauge}); err != nil {
|
||||
if err := s.CreateMetric(ctx, Definition{Name: "g", Type: TypeGauge, RetentionDays: 30}); err != nil {
|
||||
t.Fatalf("create: %v", err)
|
||||
}
|
||||
base := time.Date(2026, 6, 18, 0, 0, 0, 0, time.UTC)
|
||||
base := time.Now().UTC().Truncate(time.Minute)
|
||||
var batch []Point
|
||||
vals := []float64{5, 15, 25, 35, 100, 200}
|
||||
for i, v := range vals {
|
||||
batch = append(batch, Point{
|
||||
MetricName: "g", EntityID: "n1",
|
||||
Timestamp: base.Add(time.Duration(i) * time.Minute),
|
||||
Timestamp: base.Add(time.Duration(i) * 5 * time.Second),
|
||||
Value: v,
|
||||
})
|
||||
}
|
||||
@@ -127,9 +127,9 @@ func TestSQLAggregateMatchesInMemory(t *testing.T) {
|
||||
t.Fatalf("write: %v", err)
|
||||
}
|
||||
q := AggregateQuery{
|
||||
Query: Query{MetricName: "g", EntityID: "n1", Start: base, End: base.Add(time.Hour)},
|
||||
Query: Query{MetricName: "g", EntityID: "n1", Start: base, End: base.Add(time.Minute)},
|
||||
Aggregation: AggAvg,
|
||||
Interval: 3 * time.Minute,
|
||||
Interval: 15 * time.Second,
|
||||
}
|
||||
// SQL pushdown path.
|
||||
sqlRes, err := s.Aggregate(ctx, q)
|
||||
@@ -159,7 +159,7 @@ func TestSQLAggregateMatchesInMemory(t *testing.T) {
|
||||
//
|
||||
// TestCounterRateHandlesReset 验证计数器重置时速率计算仍然稳定。
|
||||
func TestCounterRateHandlesReset(t *testing.T) {
|
||||
base := time.Date(2026, 6, 18, 0, 0, 0, 0, time.UTC)
|
||||
base := time.Now().UTC().Truncate(time.Minute)
|
||||
// Counter goes 0 -> 10 -> reset -> 5; naive (last-first)/sec would give a
|
||||
// positive-but-wrong 5/30s. Correct counter rate sums positive deltas:
|
||||
// (10-0) + (5-0 after reset) = 15 over 30s = 0.5/s.
|
||||
@@ -195,12 +195,12 @@ func TestAlignTimeNegativeTimestamp(t *testing.T) {
|
||||
func TestLatestReturnsMostRecent(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
s := newMemStore(t)
|
||||
if err := s.CreateMetric(ctx, Definition{Name: "l", Type: TypeGauge}); err != nil {
|
||||
if err := s.CreateMetric(ctx, Definition{Name: "l", Type: TypeGauge, RetentionDays: 30}); err != nil {
|
||||
t.Fatalf("create: %v", err)
|
||||
}
|
||||
base := time.Date(2026, 6, 18, 0, 0, 0, 0, time.UTC)
|
||||
base := time.Now().UTC().Add(-30 * time.Second)
|
||||
for i := 0; i < 5; i++ {
|
||||
if err := s.Write(ctx, Point{MetricName: "l", EntityID: "n1", Timestamp: base.Add(time.Duration(i) * time.Minute), Value: float64(i)}); err != nil {
|
||||
if err := s.Write(ctx, Point{MetricName: "l", EntityID: "n1", Timestamp: base.Add(time.Duration(i) * 5 * time.Second), Value: float64(i)}); err != nil {
|
||||
t.Fatalf("write: %v", err)
|
||||
}
|
||||
}
|
||||
@@ -219,7 +219,7 @@ func TestLatestReturnsMostRecent(t *testing.T) {
|
||||
func TestStatsDistinguishesNoDataFromUnknownMetric(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
s := newMemStore(t)
|
||||
base := time.Date(2026, 6, 18, 0, 0, 0, 0, time.UTC)
|
||||
base := time.Now().UTC().Truncate(time.Minute)
|
||||
|
||||
// Unknown metric -> ErrNotFound.
|
||||
_, err := s.Stats(ctx, Query{MetricName: "nope", Start: base, End: base.Add(time.Hour)})
|
||||
@@ -228,7 +228,7 @@ func TestStatsDistinguishesNoDataFromUnknownMetric(t *testing.T) {
|
||||
}
|
||||
|
||||
// Known metric but empty window -> ErrNoData.
|
||||
if err := s.CreateMetric(ctx, Definition{Name: "known", Type: TypeGauge}); err != nil {
|
||||
if err := s.CreateMetric(ctx, Definition{Name: "known", Type: TypeGauge, RetentionDays: 30}); err != nil {
|
||||
t.Fatalf("create: %v", err)
|
||||
}
|
||||
_, err = s.Stats(ctx, Query{MetricName: "known", Start: base, End: base.Add(time.Hour)})
|
||||
@@ -266,19 +266,19 @@ func TestStdDevPopMatchesCalculateStats(t *testing.T) {
|
||||
func TestAggregateStdDevSQLiteUsesMemoryPath(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
s := newMemStore(t)
|
||||
if err := s.CreateMetric(ctx, Definition{Name: "sd", Type: TypeGauge}); err != nil {
|
||||
if err := s.CreateMetric(ctx, Definition{Name: "sd", Type: TypeGauge, RetentionDays: 30}); err != nil {
|
||||
t.Fatalf("create: %v", err)
|
||||
}
|
||||
base := time.Date(2026, 6, 18, 0, 0, 0, 0, time.UTC)
|
||||
base := time.Now().UTC().Truncate(time.Minute)
|
||||
var batch []Point
|
||||
for i, v := range []float64{10, 20, 30} {
|
||||
batch = append(batch, Point{MetricName: "sd", EntityID: "n1", Timestamp: base.Add(time.Duration(i) * time.Minute), Value: v})
|
||||
batch = append(batch, Point{MetricName: "sd", EntityID: "n1", Timestamp: base.Add(time.Duration(i) * 5 * time.Second), Value: v})
|
||||
}
|
||||
if err := s.WriteBatch(ctx, batch); err != nil {
|
||||
t.Fatalf("write: %v", err)
|
||||
}
|
||||
res, err := s.Aggregate(ctx, AggregateQuery{
|
||||
Query: Query{MetricName: "sd", EntityID: "n1", Start: base, End: base.Add(time.Hour)},
|
||||
Query: Query{MetricName: "sd", EntityID: "n1", Start: base, End: base.Add(time.Minute)},
|
||||
Aggregation: AggStdDev,
|
||||
Interval: time.Hour,
|
||||
})
|
||||
@@ -342,10 +342,10 @@ func TestSQLiteReadPoolOpens(t *testing.T) {
|
||||
t.Fatalf("reader() should return the dedicated read pool")
|
||||
}
|
||||
// Round-trip a write (primary) and a read (read pool).
|
||||
if err := store.CreateMetric(ctx, Definition{Name: "rp", Type: TypeGauge}); err != nil {
|
||||
if err := store.CreateMetric(ctx, Definition{Name: "rp", Type: TypeGauge, RetentionDays: 30}); err != nil {
|
||||
t.Fatalf("create: %v", err)
|
||||
}
|
||||
base := time.Date(2026, 6, 18, 0, 0, 0, 0, time.UTC)
|
||||
base := time.Now().UTC()
|
||||
if err := store.Write(ctx, Point{MetricName: "rp", EntityID: "n1", Timestamp: base, Value: 7}); err != nil {
|
||||
t.Fatalf("write: %v", err)
|
||||
}
|
||||
@@ -379,7 +379,7 @@ func TestMemoryDSNSkipsReadPool(t *testing.T) {
|
||||
func TestWriteBatchAtomicAcrossChunks(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
s := newMemStore(t)
|
||||
if err := s.CreateMetric(ctx, Definition{Name: "atom", Type: TypeGauge}); err != nil {
|
||||
if err := s.CreateMetric(ctx, Definition{Name: "atom", Type: TypeGauge, RetentionDays: 30}); err != nil {
|
||||
t.Fatalf("create: %v", err)
|
||||
}
|
||||
base := time.Date(2026, 6, 18, 0, 0, 0, 0, time.UTC)
|
||||
@@ -409,23 +409,23 @@ func TestWriteBatchAtomicAcrossChunks(t *testing.T) {
|
||||
func TestAggregateBucketPagingSQLPath(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
s := newMemStore(t)
|
||||
if err := s.CreateMetric(ctx, Definition{Name: "bp", Type: TypeGauge}); err != nil {
|
||||
if err := s.CreateMetric(ctx, Definition{Name: "bp", Type: TypeGauge, RetentionDays: 30}); err != nil {
|
||||
t.Fatalf("create: %v", err)
|
||||
}
|
||||
base := time.Date(2026, 6, 18, 0, 0, 0, 0, time.UTC)
|
||||
// 6 points across 6 one-minute buckets, values 0..5.
|
||||
base := time.Now().UTC().Truncate(time.Minute)
|
||||
// 6 points across 6 ten-second buckets, values 0..5.
|
||||
var batch []Point
|
||||
for i := 0; i < 6; i++ {
|
||||
batch = append(batch, Point{MetricName: "bp", EntityID: "n1", Timestamp: base.Add(time.Duration(i) * time.Minute), Value: float64(i)})
|
||||
batch = append(batch, Point{MetricName: "bp", EntityID: "n1", Timestamp: base.Add(time.Duration(i) * 10 * time.Second), Value: float64(i)})
|
||||
}
|
||||
if err := s.WriteBatch(ctx, batch); err != nil {
|
||||
t.Fatalf("write: %v", err)
|
||||
}
|
||||
// AggAvg -> SQL pushdown path. One bucket per minute. Page: offset 1, limit 2.
|
||||
// One bucket per ten seconds. Page: offset 1, limit 2.
|
||||
res, err := s.Aggregate(ctx, AggregateQuery{
|
||||
Query: Query{MetricName: "bp", EntityID: "n1", Start: base, End: base.Add(time.Hour)},
|
||||
Query: Query{MetricName: "bp", EntityID: "n1", Start: base, End: base.Add(time.Minute)},
|
||||
Aggregation: AggAvg,
|
||||
Interval: time.Minute,
|
||||
Interval: 10 * time.Second,
|
||||
BucketLimit: 2,
|
||||
BucketOffset: 1,
|
||||
})
|
||||
@@ -447,25 +447,25 @@ func TestAggregateBucketPagingSQLPath(t *testing.T) {
|
||||
func TestAggregateBucketPagingMemoryPathMatchesSQL(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
s := newMemStore(t)
|
||||
if err := s.CreateMetric(ctx, Definition{Name: "bp2", Type: TypeGauge}); err != nil {
|
||||
if err := s.CreateMetric(ctx, Definition{Name: "bp2", Type: TypeGauge, RetentionDays: 30}); err != nil {
|
||||
t.Fatalf("create: %v", err)
|
||||
}
|
||||
base := time.Date(2026, 6, 18, 0, 0, 0, 0, time.UTC)
|
||||
base := time.Now().UTC().Truncate(time.Minute)
|
||||
var batch []Point
|
||||
for i := 0; i < 6; i++ {
|
||||
batch = append(batch, Point{MetricName: "bp2", EntityID: "n1", Timestamp: base.Add(time.Duration(i) * time.Minute), Value: float64(i * 10)})
|
||||
batch = append(batch, Point{MetricName: "bp2", EntityID: "n1", Timestamp: base.Add(time.Duration(i) * 10 * time.Second), Value: float64(i * 10)})
|
||||
}
|
||||
if err := s.WriteBatch(ctx, batch); err != nil {
|
||||
t.Fatalf("write: %v", err)
|
||||
}
|
||||
q := Query{MetricName: "bp2", EntityID: "n1", Start: base, End: base.Add(time.Hour)}
|
||||
q := Query{MetricName: "bp2", EntityID: "n1", Start: base, End: base.Add(time.Minute)}
|
||||
// AggMin (SQL pushdown) and AggP50 (memory fallback on sqlite) with identical
|
||||
// bucket paging must select the same bucket window (the same timestamps).
|
||||
sqlPaged, err := s.Aggregate(ctx, AggregateQuery{Query: q, Aggregation: AggMin, Interval: time.Minute, BucketLimit: 3, BucketOffset: 2})
|
||||
sqlPaged, err := s.Aggregate(ctx, AggregateQuery{Query: q, Aggregation: AggMin, Interval: 10 * time.Second, BucketLimit: 3, BucketOffset: 2})
|
||||
if err != nil {
|
||||
t.Fatalf("sql aggregate: %v", err)
|
||||
}
|
||||
memPaged, err := s.Aggregate(ctx, AggregateQuery{Query: q, Aggregation: AggP50, Interval: time.Minute, BucketLimit: 3, BucketOffset: 2})
|
||||
memPaged, err := s.Aggregate(ctx, AggregateQuery{Query: q, Aggregation: AggP50, Interval: 10 * time.Second, BucketLimit: 3, BucketOffset: 2})
|
||||
if err != nil {
|
||||
t.Fatalf("mem aggregate: %v", err)
|
||||
}
|
||||
@@ -485,19 +485,74 @@ func TestAggregateBucketPagingMemoryPathMatchesSQL(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestAggregateSeparatesTagSeries(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
s := newMemStore(t)
|
||||
if err := s.CreateMetric(ctx, Definition{Name: "tagged.util", Type: TypeGauge, RetentionDays: 30}); err != nil {
|
||||
t.Fatalf("create: %v", err)
|
||||
}
|
||||
base := time.Now().UTC().Truncate(time.Minute)
|
||||
points := []Point{
|
||||
{MetricName: "tagged.util", EntityID: "n1", Timestamp: base.Add(10 * time.Second), Value: 10, Tags: map[string]string{"device": "0"}},
|
||||
{MetricName: "tagged.util", EntityID: "n1", Timestamp: base.Add(20 * time.Second), Value: 30, Tags: map[string]string{"device": "0"}},
|
||||
{MetricName: "tagged.util", EntityID: "n1", Timestamp: base.Add(10 * time.Second), Value: 100, Tags: map[string]string{"device": "1"}},
|
||||
{MetricName: "tagged.util", EntityID: "n1", Timestamp: base.Add(20 * time.Second), Value: 200, Tags: map[string]string{"device": "1"}},
|
||||
}
|
||||
if err := s.WriteBatch(ctx, points); err != nil {
|
||||
t.Fatalf("write: %v", err)
|
||||
}
|
||||
query := Query{MetricName: "tagged.util", EntityID: "n1", Start: base, End: base.Add(time.Minute)}
|
||||
|
||||
sqlRes, err := s.Aggregate(ctx, AggregateQuery{
|
||||
Query: query,
|
||||
Aggregation: AggAvg,
|
||||
Interval: time.Minute,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("sql aggregate: %v", err)
|
||||
}
|
||||
assertTaggedAggregate(t, sqlRes, "0", 20, 2)
|
||||
assertTaggedAggregate(t, sqlRes, "1", 150, 2)
|
||||
|
||||
memRes, err := s.Aggregate(ctx, AggregateQuery{
|
||||
Query: query,
|
||||
Aggregation: AggP50,
|
||||
Interval: time.Minute,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("memory aggregate: %v", err)
|
||||
}
|
||||
assertTaggedAggregate(t, memRes, "0", 20, 2)
|
||||
assertTaggedAggregate(t, memRes, "1", 150, 2)
|
||||
}
|
||||
|
||||
func assertTaggedAggregate(t *testing.T, points []AggregatePoint, device string, wantValue float64, wantCount int) {
|
||||
t.Helper()
|
||||
for _, point := range points {
|
||||
if point.Tags["device"] != device {
|
||||
continue
|
||||
}
|
||||
if point.Value != wantValue || point.Count != wantCount {
|
||||
t.Fatalf("device %s aggregate = value %v count %d, want value %v count %d; all=%#v", device, point.Value, point.Count, wantValue, wantCount, points)
|
||||
}
|
||||
return
|
||||
}
|
||||
t.Fatalf("missing aggregate for device %s; all=%#v", device, points)
|
||||
}
|
||||
|
||||
// TestAggregateIgnoresRawPointLimit verifies raw limits do not affect aggregation.
|
||||
//
|
||||
// TestAggregateIgnoresRawPointLimit 验证原始点分页参数不会影响聚合输入。
|
||||
func TestAggregateIgnoresRawPointLimit(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
s := newMemStore(t)
|
||||
if err := s.CreateMetric(ctx, Definition{Name: "ig", Type: TypeGauge}); err != nil {
|
||||
if err := s.CreateMetric(ctx, Definition{Name: "ig", Type: TypeGauge, RetentionDays: 30}); err != nil {
|
||||
t.Fatalf("create: %v", err)
|
||||
}
|
||||
base := time.Date(2026, 6, 18, 0, 0, 0, 0, time.UTC)
|
||||
base := time.Now().UTC().Truncate(time.Minute)
|
||||
var batch []Point
|
||||
for i := 0; i < 6; i++ {
|
||||
batch = append(batch, Point{MetricName: "ig", EntityID: "n1", Timestamp: base.Add(time.Duration(i) * time.Minute), Value: float64(i)})
|
||||
batch = append(batch, Point{MetricName: "ig", EntityID: "n1", Timestamp: base.Add(time.Duration(i) * 5 * time.Second), Value: float64(i)})
|
||||
}
|
||||
if err := s.WriteBatch(ctx, batch); err != nil {
|
||||
t.Fatalf("write: %v", err)
|
||||
@@ -523,10 +578,10 @@ func TestAggregateIgnoresRawPointLimit(t *testing.T) {
|
||||
func TestJSONTagKeyWithSpecialChars(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
s := newMemStore(t)
|
||||
if err := s.CreateMetric(ctx, Definition{Name: "tk", Type: TypeGauge}); err != nil {
|
||||
if err := s.CreateMetric(ctx, Definition{Name: "tk", Type: TypeGauge, RetentionDays: 30}); err != nil {
|
||||
t.Fatalf("create: %v", err)
|
||||
}
|
||||
base := time.Date(2026, 6, 18, 0, 0, 0, 0, time.UTC)
|
||||
base := time.Now().UTC().Truncate(time.Minute)
|
||||
// Keys containing a dot, a hyphen, a space, and a quote.
|
||||
tags := map[string]string{
|
||||
"region.zone": "ap-1",
|
||||
@@ -537,7 +592,7 @@ func TestJSONTagKeyWithSpecialChars(t *testing.T) {
|
||||
t.Fatalf("write: %v", err)
|
||||
}
|
||||
// A second point that should NOT match the filter below.
|
||||
if err := s.Write(ctx, Point{MetricName: "tk", EntityID: "n1", Timestamp: base.Add(time.Minute), Value: 2, Tags: map[string]string{"region.zone": "eu-1"}}); err != nil {
|
||||
if err := s.Write(ctx, Point{MetricName: "tk", EntityID: "n1", Timestamp: base.Add(10 * time.Second), Value: 2, Tags: map[string]string{"region.zone": "eu-1"}}); err != nil {
|
||||
t.Fatalf("write 2: %v", err)
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,48 @@
|
||||
package metric
|
||||
|
||||
import "time"
|
||||
|
||||
var standardQueryIntervals = [...]time.Duration{
|
||||
time.Second,
|
||||
5 * time.Second,
|
||||
10 * time.Second,
|
||||
15 * time.Second,
|
||||
30 * time.Second,
|
||||
time.Minute,
|
||||
2 * time.Minute,
|
||||
5 * time.Minute,
|
||||
10 * time.Minute,
|
||||
15 * time.Minute,
|
||||
30 * time.Minute,
|
||||
time.Hour,
|
||||
2 * time.Hour,
|
||||
3 * time.Hour,
|
||||
6 * time.Hour,
|
||||
12 * time.Hour,
|
||||
24 * time.Hour,
|
||||
}
|
||||
|
||||
// FloorStandardInterval returns the largest standard query interval that does
|
||||
// not exceed interval. Values below one second use the minimum interval.
|
||||
func FloorStandardInterval(interval time.Duration) time.Duration {
|
||||
selected := standardQueryIntervals[0]
|
||||
for _, candidate := range standardQueryIntervals {
|
||||
if candidate > interval {
|
||||
break
|
||||
}
|
||||
selected = candidate
|
||||
}
|
||||
return selected
|
||||
}
|
||||
|
||||
// CeilStandardInterval returns the smallest standard query interval that is
|
||||
// at least interval. Values above one day are rounded up to whole days.
|
||||
func CeilStandardInterval(interval time.Duration) time.Duration {
|
||||
for _, candidate := range standardQueryIntervals {
|
||||
if candidate >= interval {
|
||||
return candidate
|
||||
}
|
||||
}
|
||||
day := standardQueryIntervals[len(standardQueryIntervals)-1]
|
||||
return ((interval-1)/day + 1) * day
|
||||
}
|
||||
@@ -0,0 +1,31 @@
|
||||
package metric
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestStandardIntervals(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
interval time.Duration
|
||||
floor time.Duration
|
||||
ceil time.Duration
|
||||
}{
|
||||
{name: "below minimum", interval: 500 * time.Millisecond, floor: time.Second, ceil: time.Second},
|
||||
{name: "exact", interval: 5 * time.Minute, floor: 5 * time.Minute, ceil: 5 * time.Minute},
|
||||
{name: "between standards", interval: 7 * time.Minute, floor: 5 * time.Minute, ceil: 10 * time.Minute},
|
||||
{name: "above one day", interval: 25 * time.Hour, floor: 24 * time.Hour, ceil: 48 * time.Hour},
|
||||
}
|
||||
|
||||
for _, test := range tests {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
if got := FloorStandardInterval(test.interval); got != test.floor {
|
||||
t.Fatalf("FloorStandardInterval(%s) = %s, want %s", test.interval, got, test.floor)
|
||||
}
|
||||
if got := CeilStandardInterval(test.interval); got != test.ceil {
|
||||
t.Fatalf("CeilStandardInterval(%s) = %s, want %s", test.interval, got, test.ceil)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
Loaded 100 of 239 files, more files were not shown because too many files have changed in this diff.
Show more
Reference in new issue
Block a user