mirror of
https://github.com/komari-monitor/komari.git
synced 2026-11-11 08:42:39 +00:00
Compare commits
137
Commits
1.0.8-fix2
..
1.2.5
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
e446bee900 | ||
|
|
a980befd60 | ||
|
|
2e0c598dae | ||
|
|
771a192a3d | ||
|
|
95cc65e28d | ||
|
|
b6e36ee0c4 | ||
|
|
25f3e1d2bd | ||
|
|
091eafa743 | ||
|
|
cbe665591f | ||
|
|
0abd9ff9bc | ||
|
|
618ced3b8f | ||
|
|
f7fd459723 | ||
|
|
bb22a818bf | ||
|
|
b66fe47487 | ||
|
|
98122fa4d1 | ||
|
|
c3f65b1a32 | ||
|
|
7da616ce4c | ||
|
|
9d914e70b9 | ||
|
|
6f0624e9b4 | ||
|
|
3cc67e11cd | ||
|
|
742a5e953e | ||
|
|
3b82cee4b1 | ||
|
|
2b010954a3 | ||
|
|
6f23565517 | ||
|
|
7c2896c3fe | ||
|
|
721d61bc4e | ||
|
|
014b236745 | ||
|
|
80c8d7e5ee | ||
|
|
8423881cc4 | ||
|
|
a1d644977c | ||
|
|
301fb536f2 | ||
|
|
49a8d772c8 | ||
|
|
a9adae29d2 | ||
|
|
d6e23baa19 | ||
|
|
658e7603f8 | ||
|
|
fb0b9b9910 | ||
|
|
e149e8b8b7 | ||
|
|
d2cc1503de | ||
|
|
f86803feff | ||
|
|
a2e3c7218d | ||
|
|
567d2b66fb | ||
|
|
473dbb9057 | ||
|
|
89bd47d5df | ||
|
|
10c4a9dafc | ||
|
|
f4d73bf8c9 | ||
|
|
441e2e9194 | ||
|
|
c15881c463 | ||
|
|
f5acbd2efe | ||
|
|
e07378c92b | ||
|
|
cd38d32a7d | ||
|
|
53f326390e | ||
|
|
651d24128c | ||
|
|
2660026917 | ||
|
|
ac7ace27cb | ||
|
|
1de008269b | ||
|
|
4bdbff0377 | ||
|
|
3415c06256 | ||
|
|
9f3da4c17a | ||
|
|
3d0c83557e | ||
|
|
e319eb7f20 | ||
|
|
7f007b7eb9 | ||
|
|
9e665005e6 | ||
|
|
b5b32b5f64 | ||
|
|
89a699b6cb | ||
|
|
79cbf2d2d1 | ||
|
|
ab00c9529e | ||
|
|
21500a6727 | ||
|
|
066b944789 | ||
|
|
d7dd816a26 | ||
|
|
c5b464c000 | ||
|
|
645c733393 | ||
|
|
85088e64ca | ||
|
|
896b3e45fc | ||
|
|
490f951b59 | ||
|
|
69c96255d7 | ||
|
|
487542e55d | ||
|
|
ca8fe76c06 | ||
|
|
e6e3619cd2 | ||
|
|
92ab67d5f3 | ||
|
|
77f5bef1a5 | ||
|
|
58e43a0862 | ||
|
|
80c1f2747a | ||
|
|
b7b2d8b050 | ||
|
|
b9eabd7cdc | ||
|
|
0a19465140 | ||
|
|
e874b5db18 | ||
|
|
23e7d97e3c | ||
|
|
a222de2c55 | ||
|
|
bb1eed1ccb | ||
|
|
c34bc8da69 | ||
|
|
249c4376f1 | ||
|
|
d36bde7d94 | ||
|
|
493d2a2707 | ||
|
|
93a9fbe5cc | ||
|
|
b070f33bf9 | ||
|
|
ee5480dba7 | ||
|
|
2e9ef3d5ba | ||
|
|
1061b84087 | ||
|
|
0dc80f7696 | ||
|
|
212e3c689b | ||
|
|
488ca451ae | ||
|
|
f50c167b37 | ||
|
|
4e91d62d48 | ||
|
|
a1b0b1e177 | ||
|
|
147dd2011a | ||
|
|
d24b71d4af | ||
|
|
483e3a94e9 | ||
|
|
e612f5ed8f | ||
|
|
78f3c8fd4b | ||
|
|
5a4e073d63 | ||
|
|
22cbfecfa9 | ||
|
|
d1b059fc2f | ||
|
|
cf5c410aa6 | ||
|
|
9c46673a1a | ||
|
|
45c27ae981 | ||
|
|
b62fb1f708 | ||
|
|
579c196643 | ||
|
|
e8da6e213a | ||
|
|
d9426e43a9 | ||
|
|
aad0274d56 | ||
|
|
d65f7ca924 | ||
|
|
a4d95640a5 | ||
|
|
adda3a2488 | ||
|
|
34654c0969 | ||
|
|
b708f9817d | ||
|
|
1210a1cbd9 | ||
|
|
24f1a673b9 | ||
|
|
a3162e1992 | ||
|
|
17f904e4ab | ||
|
|
1324f4fa21 | ||
|
|
cf15144bb9 | ||
|
|
946b27b3b8 | ||
|
|
aed5254cf0 | ||
|
|
7b8922b7ac | ||
|
|
733ad4c6c0 | ||
|
|
ef98ddd1d3 | ||
|
|
ca28c40aca |
No files matched your search
@@ -65,13 +65,20 @@ jobs:
|
||||
|
||||
- name: Clone and build frontend
|
||||
run: |
|
||||
git clone https://github.com/komari-monitor/komari-web web
|
||||
cd web
|
||||
git clone https://github.com/komari-monitor/komari-web komari-web
|
||||
cd komari-web
|
||||
npm install
|
||||
npm run build
|
||||
cd ..
|
||||
mkdir -p public/dist
|
||||
cp -r web/dist/* public/dist/
|
||||
mkdir -p web/public/defaultTheme/dist
|
||||
rm -rf web/public/defaultTheme/dist/*
|
||||
cp -r komari-web/dist/* web/public/defaultTheme/dist/
|
||||
cp -f komari-web/komari-theme.json web/public/defaultTheme/
|
||||
if [ -f komari-web/preview.png ]; then cp -f komari-web/preview.png web/public/defaultTheme/; fi
|
||||
if [ -f komari-web/perview.png ]; then cp -f komari-web/perview.png web/public/defaultTheme/; fi
|
||||
# Compatibility: ensure both filenames exist if only one is present
|
||||
if [ -f web/public/defaultTheme/preview.png ] && [ ! -f web/public/defaultTheme/perview.png ]; then cp -f web/public/defaultTheme/preview.png web/public/defaultTheme/perview.png; fi
|
||||
if [ -f web/public/defaultTheme/perview.png ] && [ ! -f web/public/defaultTheme/preview.png ]; then cp -f web/public/defaultTheme/perview.png web/public/defaultTheme/preview.png; fi
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v5
|
||||
@@ -107,4 +114,4 @@ jobs:
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: komari-${{ matrix.goos }}-${{ matrix.goarch }}
|
||||
path: komari-${{ matrix.goos }}-${{ matrix.goarch }}*
|
||||
path: komari-${{ matrix.goos }}-${{ matrix.goarch }}*
|
||||
@@ -12,22 +12,29 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v4
|
||||
|
||||
uses: actions/checkout@v4
|
||||
|
||||
#- name: Create dummy index.html
|
||||
# run: |
|
||||
# mkdir -p public/dist
|
||||
# echo "<html><body><h1>This is komari development server.</h1></body></html>" > public/dist/index.html
|
||||
|
||||
# mkdir -p web/public/defaultTheme/dist
|
||||
# echo "<html><body><h1>This is komari development server.</h1></body></html>" > web/public/defaultTheme/dist/index.html
|
||||
|
||||
- name: Clone and build frontend
|
||||
run: |
|
||||
git clone https://github.com/komari-monitor/komari-web web
|
||||
cd web
|
||||
git clone https://github.com/komari-monitor/komari-web komari-web
|
||||
cd komari-web
|
||||
npm install
|
||||
npm run build
|
||||
cd ..
|
||||
mkdir -p public/dist
|
||||
cp -r web/dist/* public/dist/
|
||||
mkdir -p web/public/defaultTheme/dist
|
||||
rm -rf web/public/defaultTheme/dist/*
|
||||
cp -r komari-web/dist/* web/public/defaultTheme/dist/
|
||||
cp -f komari-web/komari-theme.json web/public/defaultTheme/
|
||||
if [ -f komari-web/preview.png ]; then cp -f komari-web/preview.png web/public/defaultTheme/; fi
|
||||
if [ -f komari-web/perview.png ]; then cp -f komari-web/perview.png web/public/defaultTheme/; fi
|
||||
# Compatibility: ensure both filenames exist if only one is present
|
||||
if [ -f web/public/defaultTheme/preview.png ] && [ ! -f web/public/defaultTheme/perview.png ]; then cp -f web/public/defaultTheme/preview.png web/public/defaultTheme/perview.png; fi
|
||||
if [ -f web/public/defaultTheme/perview.png ] && [ ! -f web/public/defaultTheme/preview.png ]; then cp -f web/public/defaultTheme/perview.png web/public/defaultTheme/preview.png; fi
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v5
|
||||
|
||||
@@ -28,13 +28,20 @@ jobs:
|
||||
|
||||
- name: Clone and build frontend
|
||||
run: |
|
||||
git clone https://github.com/komari-monitor/komari-web web
|
||||
cd web
|
||||
git clone https://github.com/komari-monitor/komari-web komari-web
|
||||
cd komari-web
|
||||
npm install
|
||||
npm run build
|
||||
cd ..
|
||||
mkdir -p public/dist
|
||||
cp -r web/dist/* public/dist/
|
||||
mkdir -p web/public/defaultTheme/dist
|
||||
rm -rf web/public/defaultTheme/dist/*
|
||||
cp -r komari-web/dist/* web/public/defaultTheme/dist/
|
||||
cp -f komari-web/komari-theme.json web/public/defaultTheme/
|
||||
if [ -f komari-web/preview.png ]; then cp -f komari-web/preview.png web/public/defaultTheme/; fi
|
||||
if [ -f komari-web/perview.png ]; then cp -f komari-web/perview.png web/public/defaultTheme/; fi
|
||||
# Compatibility: ensure both filenames exist if only one is present
|
||||
if [ -f web/public/defaultTheme/preview.png ] && [ ! -f web/public/defaultTheme/perview.png ]; then cp -f web/public/defaultTheme/preview.png web/public/defaultTheme/perview.png; fi
|
||||
if [ -f web/public/defaultTheme/perview.png ] && [ ! -f web/public/defaultTheme/preview.png ]; then cp -f web/public/defaultTheme/perview.png web/public/defaultTheme/preview.png; fi
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v5
|
||||
|
||||
@@ -0,0 +1,208 @@
|
||||
name: Generate Release Notes
|
||||
|
||||
on:
|
||||
release:
|
||||
types: [published]
|
||||
repository_dispatch:
|
||||
types:
|
||||
- komari-web-release-published
|
||||
schedule:
|
||||
- cron: "41 * * * *"
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
tag:
|
||||
description: "Release tag to regenerate notes for. Leave empty to use the latest release."
|
||||
required: false
|
||||
type: string
|
||||
|
||||
permissions:
|
||||
contents: write
|
||||
|
||||
jobs:
|
||||
update-release-notes:
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
steps:
|
||||
- name: Generate and apply release notes
|
||||
uses: actions/github-script@v9
|
||||
with:
|
||||
script: |
|
||||
const { owner, repo } = context.repo;
|
||||
const webRepo = { owner: "komari-monitor", repo: "komari-web" };
|
||||
|
||||
const clean = (value) => (typeof value === "string" ? value.trim() : "");
|
||||
const releaseTimestamp = (release) => Date.parse(release.published_at || release.created_at || "");
|
||||
|
||||
async function resolveRelease() {
|
||||
const requestedTag = clean(context.payload.release?.tag_name)
|
||||
|| clean(context.payload.client_payload?.tag)
|
||||
|| clean(core.getInput("tag"));
|
||||
|
||||
if (requestedTag) {
|
||||
const release = context.payload.release?.tag_name === requestedTag
|
||||
? context.payload.release
|
||||
: (await github.rest.repos.getReleaseByTag({ owner, repo, tag: requestedTag })).data;
|
||||
return { tag: requestedTag, release };
|
||||
}
|
||||
|
||||
const release = (await github.rest.repos.getLatestRelease({ owner, repo })).data;
|
||||
return { tag: release.tag_name, release };
|
||||
}
|
||||
|
||||
async function getPreviousReleaseTag(tag, release) {
|
||||
const releases = await github.paginate(github.rest.repos.listReleases, {
|
||||
owner,
|
||||
repo,
|
||||
per_page: 100,
|
||||
});
|
||||
|
||||
const currentTime = releaseTimestamp(release);
|
||||
const candidates = releases
|
||||
.filter((item) => !item.draft && item.tag_name !== tag)
|
||||
.filter((item) => {
|
||||
if (!Number.isFinite(currentTime)) {
|
||||
return true;
|
||||
}
|
||||
|
||||
const itemTime = releaseTimestamp(item);
|
||||
return Number.isFinite(itemTime) && itemTime < currentTime;
|
||||
})
|
||||
.sort((left, right) => releaseTimestamp(right) - releaseTimestamp(left));
|
||||
|
||||
return candidates[0]?.tag_name || "";
|
||||
}
|
||||
|
||||
async function getGeneratedNotes(tag, release, previousTag) {
|
||||
const params = {
|
||||
owner,
|
||||
repo,
|
||||
tag_name: tag,
|
||||
target_commitish: release.target_commitish || undefined,
|
||||
};
|
||||
|
||||
if (previousTag) {
|
||||
params.previous_tag_name = previousTag;
|
||||
}
|
||||
|
||||
const generated = await github.request(
|
||||
"POST /repos/{owner}/{repo}/releases/generate-notes",
|
||||
params,
|
||||
);
|
||||
|
||||
return clean(generated.data.body);
|
||||
}
|
||||
|
||||
async function getComparedCommits(previousTag, tag) {
|
||||
if (!previousTag) {
|
||||
return [];
|
||||
}
|
||||
|
||||
const commits = [];
|
||||
for (let page = 1; page <= 20; page += 1) {
|
||||
const response = await github.rest.repos.compareCommitsWithBasehead({
|
||||
owner,
|
||||
repo,
|
||||
basehead: `${previousTag}...${tag}`,
|
||||
per_page: 100,
|
||||
page,
|
||||
});
|
||||
|
||||
commits.push(...response.data.commits);
|
||||
|
||||
if (!response.headers.link?.includes('rel="next"')) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
return commits;
|
||||
}
|
||||
|
||||
function renderCommitLine(commit) {
|
||||
const subject = clean(commit.commit.message.split("\n")[0]) || "(no subject)";
|
||||
const shortSha = commit.sha.slice(0, 7);
|
||||
const author = commit.author?.login
|
||||
? ` @${commit.author.login}`
|
||||
: (commit.commit.author?.name ? ` ${commit.commit.author.name}` : "");
|
||||
|
||||
return `- ${subject} ([${shortSha}](${commit.html_url}))${author}`;
|
||||
}
|
||||
|
||||
async function buildCommitSection(previousTag, tag) {
|
||||
if (!previousTag) {
|
||||
return "## Commits\n\nNo previous release tag was found; commit comparison is unavailable.";
|
||||
}
|
||||
|
||||
const commits = await getComparedCommits(previousTag, tag);
|
||||
if (commits.length === 0) {
|
||||
return `## Commits\n\nNo commits found between \`${previousTag}\` and \`${tag}\`.`;
|
||||
}
|
||||
|
||||
return ["## Commits", "", ...commits.map(renderCommitLine)].join("\n");
|
||||
}
|
||||
|
||||
async function buildWebReleaseSection(tag) {
|
||||
const demoteMarkdownHeadings = (markdown) => markdown.replace(/^(#{1,5})(\s+)/gm, "#$1$2");
|
||||
|
||||
try {
|
||||
const webRelease = (await github.rest.repos.getReleaseByTag({
|
||||
owner: webRepo.owner,
|
||||
repo: webRepo.repo,
|
||||
tag,
|
||||
})).data;
|
||||
|
||||
const webBody = clean(webRelease.body);
|
||||
if (!webBody) {
|
||||
return `## Komari Web Release Notes\n\nKomari Web release [${tag}](${webRelease.html_url}) exists, but its release notes are empty.`;
|
||||
}
|
||||
|
||||
return [
|
||||
"## Komari Web Release Notes",
|
||||
"",
|
||||
`Source: [komari-web ${tag}](${webRelease.html_url})`,
|
||||
"",
|
||||
demoteMarkdownHeadings(webBody),
|
||||
].join("\n");
|
||||
} catch (error) {
|
||||
if (error.status === 404) {
|
||||
core.info(`Komari Web release ${tag} does not exist yet. Skipping web release notes for now.`);
|
||||
return "";
|
||||
}
|
||||
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
const { tag, release } = await resolveRelease();
|
||||
if (!tag) {
|
||||
core.setFailed("No release tag was provided or found.");
|
||||
return;
|
||||
}
|
||||
|
||||
const previousTag = await getPreviousReleaseTag(tag, release);
|
||||
const generatedBody = await getGeneratedNotes(tag, release, previousTag);
|
||||
if (!generatedBody) {
|
||||
core.setFailed(`GitHub returned empty release notes for ${tag}.`);
|
||||
return;
|
||||
}
|
||||
|
||||
const body = [
|
||||
generatedBody,
|
||||
await buildCommitSection(previousTag, tag),
|
||||
await buildWebReleaseSection(tag),
|
||||
].filter(Boolean).join("\n\n");
|
||||
|
||||
if (clean(release.body) === clean(body)) {
|
||||
core.info(`Release notes for ${tag} are already up to date.`);
|
||||
} else {
|
||||
await github.rest.repos.updateRelease({
|
||||
owner,
|
||||
repo,
|
||||
release_id: release.id,
|
||||
body,
|
||||
});
|
||||
}
|
||||
|
||||
await core.summary
|
||||
.addHeading("Release Notes")
|
||||
.addRaw(`Updated release notes for ${tag}.`)
|
||||
.write();
|
||||
@@ -31,13 +31,20 @@ jobs:
|
||||
|
||||
- name: Clone and build frontend
|
||||
run: |
|
||||
git clone https://github.com/komari-monitor/komari-web web
|
||||
cd web
|
||||
git clone https://github.com/komari-monitor/komari-web komari-web
|
||||
cd komari-web
|
||||
npm install
|
||||
npm run build
|
||||
cd ..
|
||||
mkdir -p public/dist
|
||||
cp -r web/dist/* public/dist/
|
||||
mkdir -p web/public/defaultTheme/dist
|
||||
rm -rf web/public/defaultTheme/dist/*
|
||||
cp -r komari-web/dist/* web/public/defaultTheme/dist/
|
||||
cp -f komari-web/komari-theme.json web/public/defaultTheme/
|
||||
if [ -f komari-web/preview.png ]; then cp -f komari-web/preview.png web/public/defaultTheme/; fi
|
||||
if [ -f komari-web/perview.png ]; then cp -f komari-web/perview.png web/public/defaultTheme/; fi
|
||||
# Compatibility: ensure both filenames exist if only one is present
|
||||
if [ -f web/public/defaultTheme/preview.png ] && [ ! -f web/public/defaultTheme/perview.png ]; then cp -f web/public/defaultTheme/preview.png web/public/defaultTheme/perview.png; fi
|
||||
if [ -f web/public/defaultTheme/perview.png ] && [ ! -f web/public/defaultTheme/preview.png ]; then cp -f web/public/defaultTheme/perview.png web/public/defaultTheme/preview.png; fi
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v5
|
||||
|
||||
@@ -68,13 +68,20 @@ jobs:
|
||||
|
||||
- name: Clone and build frontend
|
||||
run: |
|
||||
git clone https://github.com/komari-monitor/komari-web web
|
||||
cd web
|
||||
git clone https://github.com/komari-monitor/komari-web komari-web
|
||||
cd komari-web
|
||||
npm install
|
||||
npm run build
|
||||
cd ..
|
||||
mkdir -p public/dist
|
||||
cp -r web/dist/* public/dist/
|
||||
mkdir -p web/public/defaultTheme/dist
|
||||
rm -rf web/public/defaultTheme/dist/*
|
||||
cp -r komari-web/dist/* web/public/defaultTheme/dist/
|
||||
cp -f komari-web/komari-theme.json web/public/defaultTheme/
|
||||
if [ -f komari-web/preview.png ]; then cp -f komari-web/preview.png web/public/defaultTheme/; fi
|
||||
if [ -f komari-web/perview.png ]; then cp -f komari-web/perview.png web/public/defaultTheme/; fi
|
||||
# Compatibility: ensure both filenames exist if only one is present
|
||||
if [ -f web/public/defaultTheme/preview.png ] && [ ! -f web/public/defaultTheme/perview.png ]; then cp -f web/public/defaultTheme/preview.png web/public/defaultTheme/perview.png; fi
|
||||
if [ -f web/public/defaultTheme/perview.png ] && [ ! -f web/public/defaultTheme/preview.png ]; then cp -f web/public/defaultTheme/perview.png web/public/defaultTheme/preview.png; fi
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v5
|
||||
|
||||
+3
-1
@@ -14,6 +14,7 @@ komari.db
|
||||
/data
|
||||
/utils/geoip/data
|
||||
.vscode/
|
||||
/backup
|
||||
|
||||
# Test binary, built with `go test -c`
|
||||
*.test
|
||||
@@ -32,4 +33,5 @@ go.work.sum
|
||||
.env
|
||||
|
||||
# build
|
||||
komari
|
||||
komari
|
||||
.codex
|
||||
+13
-2
@@ -6,7 +6,18 @@ WORKDIR /app
|
||||
ARG TARGETOS
|
||||
ARG TARGETARCH
|
||||
|
||||
RUN apk add --no-cache tzdata
|
||||
RUN apk add --no-cache ca-certificates curl tzdata
|
||||
|
||||
RUN set -eux; \
|
||||
case "${TARGETARCH}" in \
|
||||
amd64) cloudflared_arch="amd64" ;; \
|
||||
386) cloudflared_arch="386" ;; \
|
||||
arm64) cloudflared_arch="arm64" ;; \
|
||||
arm) cloudflared_arch="arm" ;; \
|
||||
*) echo "Unsupported TARGETARCH: ${TARGETARCH}" >&2; exit 1 ;; \
|
||||
esac; \
|
||||
curl -fsSL "https://github.com/cloudflare/cloudflared/releases/latest/download/cloudflared-linux-${cloudflared_arch}" -o /usr/local/bin/cloudflared; \
|
||||
chmod +x /usr/local/bin/cloudflared
|
||||
|
||||
COPY komari-${TARGETOS}-${TARGETARCH} /app/komari
|
||||
|
||||
@@ -24,4 +35,4 @@ ENV KOMARI_LISTEN=0.0.0.0:25774
|
||||
|
||||
EXPOSE 25774
|
||||
|
||||
CMD ["/app/komari", "server"]
|
||||
CMD ["/app/komari", "server"]
|
||||
@@ -0,0 +1,27 @@
|
||||
CoreOS Project
|
||||
Copyright 2014 CoreOS, Inc
|
||||
|
||||
This product includes software developed at CoreOS, Inc.
|
||||
(http://www.coreos.com/).
|
||||
|
||||
|
||||
otp
|
||||
Copyright (c) 2014, Paul Querna
|
||||
|
||||
This product includes software developed by
|
||||
Paul Querna (http://paul.querna.org/).
|
||||
|
||||
|
||||
Copyright 2014 gRPC authors.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
@@ -1,6 +1,7 @@
|
||||
# Komari
|
||||
|
||||
# Komari
|
||||

|
||||
[](https://deepwiki.com/komari-monitor/komari)
|
||||
|
||||

|
||||
|
||||
@@ -8,9 +9,15 @@
|
||||
|
||||
Komari is a lightweight, self-hosted server monitoring tool designed to provide a simple and efficient solution for monitoring server performance. It supports viewing server status through a web interface and collects data through a lightweight agent.
|
||||
|
||||
[Documentation](https://komari-document.pages.dev/) | [Telegram Group](https://t.me/komari_monitor)
|
||||
> [!WARNING]
|
||||
> Komari is a self-hosted monitoring and control program intended only for systems you own or are authorized to administer. Do not weaponize Komari, deploy it without consent, or use it for unauthorized access, persistence, command execution, or other abusive activity. For context on real-world abuse risks, see Huntress' analysis: [Komari C2 agent abuse](https://www.huntress.com/blog/komari-c2-agent-abuse).
|
||||
> Users are solely responsible for how they deploy and operate Komari. The developers do not accept responsibility for unauthorized or abusive use, or for any resulting consequences.
|
||||
> On Windows, when remote control is enabled, the client displays a Windows notification at each user login to remind the user that Komari is remote control software.
|
||||
|
||||
[Documentation](https://komari-document.pages.dev/) | [文档(镜像站 By Geekertao)](https://www.komari.wiki) | [Telegram Group](https://t.me/komari_monitor)
|
||||
|
||||
## Features
|
||||
|
||||
- **Lightweight and Efficient**: Low resource consumption, suitable for servers of all sizes.
|
||||
- **Self-hosted**: Complete control over data privacy, easy to deploy.
|
||||
- **Web Interface**: Intuitive monitoring dashboard, easy to use.
|
||||
@@ -18,16 +25,19 @@ Komari is a lightweight, self-hosted server monitoring tool designed to provide
|
||||
## Quick Start
|
||||
|
||||
### 0. One-click Deployment with Cloud Hosting
|
||||
- Rainyun - CNY 4.5/month
|
||||
|
||||
[](https://app.rainyun.com/apps/rca/store/6780/NzYxNzAz_)
|
||||
- Rainyun - CNY 4.5/month
|
||||
|
||||
- Claw Run - USD 1.5/month
|
||||
[](https://app.rainyun.com/apps/rca/store/6780/NzYxNzAz_)
|
||||
|
||||
[](https://template.run.claw.cloud/?openapp=system-fastdeploy%3FtemplateName%3Dkomari)
|
||||
- 1Panel App Store
|
||||
|
||||
Available on 1Panel App Store. Install via **App Store > Utilities > Komari**.
|
||||
|
||||
### 1. Use the One-click Install Script
|
||||
|
||||
Suitable for distributions using systemd (Ubuntu, Debian...).
|
||||
|
||||
```bash
|
||||
curl -fsSL https://raw.githubusercontent.com/komari-monitor/komari/main/install-komari.sh -o install-komari.sh
|
||||
chmod +x install-komari.sh
|
||||
@@ -35,6 +45,7 @@ sudo ./install-komari.sh
|
||||
```
|
||||
|
||||
### 2. Docker Deployment
|
||||
|
||||
1. Create a data directory:
|
||||
```bash
|
||||
mkdir -p ./data
|
||||
@@ -57,6 +68,7 @@ sudo ./install-komari.sh
|
||||
> You can also customize the initial username and password through the environment variables `ADMIN_USERNAME` and `ADMIN_PASSWORD`.
|
||||
|
||||
### 3. Binary File Deployment
|
||||
|
||||
1. Visit Komari's [GitHub Release page](https://github.com/komari-monitor/komari/releases) to download the latest binary for your operating system.
|
||||
2. Run Komari:
|
||||
```bash
|
||||
@@ -68,7 +80,6 @@ sudo ./install-komari.sh
|
||||
> [!NOTE]
|
||||
> Ensure the binary has execute permissions (`chmod +x komari`). Data will be saved in the `data` folder in the running directory.
|
||||
|
||||
|
||||
### Manual Build
|
||||
|
||||
#### Dependencies
|
||||
@@ -87,36 +98,41 @@ sudo ./install-komari.sh
|
||||
git clone https://github.com/komari-monitor/komari
|
||||
cd komari
|
||||
```
|
||||
Copy the static files generated in step 1 to the `/public/dist` folder in the root of the `komari` project.
|
||||
```bash
|
||||
Copy the static files generated in step 1 to the `/web/public/defaultTheme/dist` folder in the root of the `komari` project, and copy `komari-theme.json` + `preview.png`/`perview.png` to `/web/public/defaultTheme`.
|
||||
```bash
|
||||
go build -o komari
|
||||
```
|
||||
4. Run:
|
||||
3. Run:
|
||||
```bash
|
||||
./komari server -l 0.0.0.0:25774
|
||||
```
|
||||
The default listening port is `25774`. Access `http://localhost:25774`.
|
||||
|
||||
## Frontend Development Guide
|
||||
|
||||
[Komari Theme Development Guide | Komari](https://komari-document.pages.dev/dev/theme.html)
|
||||
|
||||
[Translate Komari on Crowdin](https://crowdin.com/project/komari/invite?h=cd051bf172c9a9f7f1360e87ffb521692507706)
|
||||
|
||||
## Client Agent Development Guide
|
||||
|
||||
[Komari Agent Information Reporting and Event Handling Documentation](https://komari-document.pages.dev/dev/agent.html)
|
||||
|
||||
## Contributing
|
||||
|
||||
Issues and Pull Requests are welcome!
|
||||
|
||||
## Acknowledgements
|
||||
|
||||
### 破碎工坊云
|
||||
|
||||
[破碎工坊云 - 专业云计算服务平台,提供高效、稳定、安全的高防服务器与CDN解决方案](https://www.crash.work/)
|
||||
|
||||
### DreamCloud
|
||||
|
||||
[DreamCloud - 极高性价比解锁直连亚太高防](https://as211392.com/)
|
||||
|
||||
### 🚀 Sponsored by SharonNetworks
|
||||
|
||||
[](https://sharon.io)
|
||||
[](https://sharon.io)
|
||||
|
||||
SharonNetworks 为您的业务起飞保驾护航!
|
||||
|
||||
@@ -129,4 +145,3 @@ All the developers who submitted PRs and created themes
|
||||
## Star History
|
||||
|
||||
[](https://www.star-history.com/#komari-monitor/komari&Date)
|
||||
|
||||
@@ -1,42 +0,0 @@
|
||||
package api
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
|
||||
"github.com/komari-monitor/komari/database/accounts"
|
||||
|
||||
"github.com/gin-gonic/gin"
|
||||
)
|
||||
|
||||
func AdminAuthMiddleware() gin.HandlerFunc {
|
||||
return func(c *gin.Context) {
|
||||
// API key authentication
|
||||
apiKey := c.GetHeader("Authorization")
|
||||
if isApiKeyValid(apiKey) {
|
||||
c.Set("api_key", apiKey)
|
||||
c.Next()
|
||||
return
|
||||
}
|
||||
// session-based authentication
|
||||
session, err := c.Cookie("session_token")
|
||||
if err != nil {
|
||||
RespondError(c, http.StatusUnauthorized, "Unauthorized.")
|
||||
c.Abort()
|
||||
return
|
||||
}
|
||||
|
||||
// Komari is a single user system
|
||||
uuid, err := accounts.GetSession(session)
|
||||
if err != nil {
|
||||
RespondError(c, http.StatusUnauthorized, "Unauthorized.")
|
||||
c.Abort()
|
||||
return
|
||||
}
|
||||
accounts.UpdateLatest(session, c.Request.UserAgent(), c.ClientIP())
|
||||
// 将 session 和 用户 UUID 传递到后续处理器
|
||||
c.Set("session", session)
|
||||
c.Set("uuid", uuid)
|
||||
|
||||
c.Next()
|
||||
}
|
||||
}
|
||||
-157
@@ -1,157 +0,0 @@
|
||||
package api
|
||||
|
||||
import (
|
||||
"log"
|
||||
"net/http"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/gin-gonic/gin"
|
||||
"github.com/gorilla/websocket"
|
||||
"github.com/komari-monitor/komari/common"
|
||||
"github.com/patrickmn/go-cache"
|
||||
|
||||
"strconv"
|
||||
|
||||
"github.com/komari-monitor/komari/database/config"
|
||||
"github.com/komari-monitor/komari/database/dbcore"
|
||||
"github.com/komari-monitor/komari/database/models"
|
||||
"github.com/komari-monitor/komari/utils"
|
||||
)
|
||||
|
||||
var (
|
||||
Records = cache.New(1*time.Minute, 1*time.Minute)
|
||||
)
|
||||
|
||||
type TerminalSession struct {
|
||||
UUID string
|
||||
UserUUID string
|
||||
Browser *websocket.Conn
|
||||
Agent *websocket.Conn
|
||||
RequesterIp string
|
||||
}
|
||||
|
||||
var TerminalSessionsMutex = &sync.Mutex{}
|
||||
var TerminalSessions = make(map[string]*TerminalSession)
|
||||
|
||||
func SaveClientReportToDB() error {
|
||||
lastMinute := time.Now().Add(-time.Minute).Unix()
|
||||
var records []models.Record
|
||||
var gpuRecords []models.GPURecord
|
||||
|
||||
// 遍历所有客户端记录
|
||||
for uuid, x := range Records.Items() {
|
||||
if uuid == "" {
|
||||
continue
|
||||
}
|
||||
|
||||
reports, ok := x.Object.([]common.Report)
|
||||
if !ok {
|
||||
log.Printf("Invalid report type for UUID %s", uuid)
|
||||
continue
|
||||
}
|
||||
|
||||
// 过滤一分钟前的记录
|
||||
var filtered []common.Report
|
||||
for _, r := range reports {
|
||||
if r.UpdatedAt.Unix() >= lastMinute {
|
||||
filtered = append(filtered, r)
|
||||
}
|
||||
}
|
||||
|
||||
// 更新缓存
|
||||
Records.Set(uuid, filtered, cache.DefaultExpiration)
|
||||
|
||||
// 计算平均报告并添加到记录列表
|
||||
if len(filtered) > 0 {
|
||||
r := utils.AverageReport(uuid, time.Now(), filtered, 0.3)
|
||||
records = append(records, r)
|
||||
|
||||
// 使用与其他数据相同的聚合逻辑处理GPU数据
|
||||
gpuAggregated := utils.AverageGPUReports(uuid, time.Now(), filtered, 0.3)
|
||||
gpuRecords = append(gpuRecords, gpuAggregated...)
|
||||
}
|
||||
}
|
||||
|
||||
// 批量插入数据库前去重(client与time共同构成唯一键)
|
||||
db := dbcore.GetDBInstance()
|
||||
|
||||
if len(records) > 0 {
|
||||
unique := make(map[string]models.Record)
|
||||
for _, rec := range records {
|
||||
key := rec.Client + "_" + strconv.FormatInt(rec.Time.ToTime().Unix(), 10)
|
||||
unique[key] = rec
|
||||
}
|
||||
var deduped []models.Record
|
||||
for _, rec := range unique {
|
||||
deduped = append(deduped, rec)
|
||||
}
|
||||
if err := db.Model(&models.Record{}).Create(&deduped).Error; err != nil {
|
||||
log.Printf("Failed to save records to database: %v", err)
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
// 批量插入GPU记录
|
||||
if len(gpuRecords) > 0 {
|
||||
// GPU记录也需要去重,防止重复插入
|
||||
gpuUnique := make(map[string]models.GPURecord)
|
||||
for _, rec := range gpuRecords {
|
||||
key := rec.Client + "_" + strconv.Itoa(rec.DeviceIndex) + "_" + strconv.FormatInt(rec.Time.ToTime().Unix(), 10)
|
||||
gpuUnique[key] = rec
|
||||
}
|
||||
var gpuDeduped []models.GPURecord
|
||||
for _, rec := range gpuUnique {
|
||||
gpuDeduped = append(gpuDeduped, rec)
|
||||
}
|
||||
if err := db.Model(&models.GPURecord{}).Create(&gpuDeduped).Error; err != nil {
|
||||
log.Printf("Failed to save GPU records to database: %v", err)
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
type Response struct {
|
||||
Status string `json:"status"`
|
||||
Message string `json:"message"`
|
||||
Data interface{} `json:"data,omitempty"`
|
||||
}
|
||||
|
||||
// Respond sends a standardized JSON response.
|
||||
func Respond(c *gin.Context, httpStatus int, status string, message string, data interface{}) {
|
||||
c.JSON(httpStatus, Response{Status: status, Message: message, Data: data})
|
||||
}
|
||||
|
||||
// RespondSuccess sends a success response with data.
|
||||
func RespondSuccess(c *gin.Context, data interface{}) {
|
||||
Respond(c, http.StatusOK, "success", "", data)
|
||||
}
|
||||
|
||||
// RespondSuccessMessage sends a success response with message and data.
|
||||
func RespondSuccessMessage(c *gin.Context, message string, data interface{}) {
|
||||
Respond(c, http.StatusOK, "success", message, data)
|
||||
}
|
||||
|
||||
// RespondError sends an error response with message.
|
||||
func RespondError(c *gin.Context, httpStatus int, message string) {
|
||||
Respond(c, httpStatus, "error", message, nil)
|
||||
}
|
||||
func GetVersion(c *gin.Context) {
|
||||
RespondSuccess(c, gin.H{
|
||||
"version": utils.CurrentVersion,
|
||||
"hash": utils.VersionHash,
|
||||
})
|
||||
}
|
||||
|
||||
func isApiKeyValid(apiKey string) bool {
|
||||
cfg, err := config.Get()
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
if cfg.ApiKey == "" || len(cfg.ApiKey) < 12 {
|
||||
return false
|
||||
}
|
||||
return apiKey == "Bearer "+cfg.ApiKey
|
||||
}
|
||||
@@ -1,75 +0,0 @@
|
||||
package api
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
|
||||
"github.com/komari-monitor/komari/database/accounts"
|
||||
"github.com/komari-monitor/komari/database/config"
|
||||
|
||||
"github.com/gin-gonic/gin"
|
||||
)
|
||||
|
||||
var (
|
||||
publicPaths = []string{
|
||||
"/ping",
|
||||
"/api/public",
|
||||
"/api/login",
|
||||
"/api/me",
|
||||
"/api/oauth",
|
||||
"/api/oauth_callback",
|
||||
"/api/version",
|
||||
"/api/recent",
|
||||
"/api/admin", // 由AdminAuthMiddleware处理
|
||||
"/api/clients/", // 由TokenAuthMiddleware处理
|
||||
}
|
||||
)
|
||||
|
||||
func PrivateSiteMiddleware() gin.HandlerFunc {
|
||||
return func(c *gin.Context) {
|
||||
// API key authentication
|
||||
apiKey := c.GetHeader("Authorization")
|
||||
if isApiKeyValid(apiKey) {
|
||||
c.Set("api_key", apiKey)
|
||||
c.Next()
|
||||
return
|
||||
}
|
||||
// 如果是公开的路径,直接放行
|
||||
for _, path := range publicPaths {
|
||||
if len(c.Request.URL.Path) >= len(path) && c.Request.URL.Path[:len(path)] == path {
|
||||
c.Next()
|
||||
return
|
||||
}
|
||||
}
|
||||
// 如果不是 /api,直接放行
|
||||
if len(c.Request.URL.Path) < 4 || c.Request.URL.Path[:4] != "/api" {
|
||||
c.Next()
|
||||
return
|
||||
}
|
||||
conf, err := config.Get()
|
||||
if err != nil {
|
||||
RespondError(c, http.StatusInternalServerError, "Failed to get configuration.")
|
||||
c.Abort()
|
||||
return
|
||||
}
|
||||
// 验证私有, 如果不是私有站点,直接放行
|
||||
if !conf.PrivateSite {
|
||||
c.Next()
|
||||
return
|
||||
}
|
||||
// 如果是私有站点,检查是否有 session
|
||||
session, err := c.Cookie("session_token")
|
||||
if err != nil {
|
||||
RespondError(c, http.StatusUnauthorized, "Private site is enabled, please login first.")
|
||||
c.Abort()
|
||||
return
|
||||
}
|
||||
_, err = accounts.GetSession(session)
|
||||
if err != nil {
|
||||
RespondError(c, http.StatusUnauthorized, "Unauthorized.")
|
||||
c.Abort()
|
||||
return
|
||||
}
|
||||
|
||||
c.Next()
|
||||
}
|
||||
}
|
||||
@@ -1,89 +0,0 @@
|
||||
package api
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"database/sql"
|
||||
"encoding/json"
|
||||
"io"
|
||||
"net/http"
|
||||
|
||||
"github.com/komari-monitor/komari/database/clients"
|
||||
"gorm.io/gorm"
|
||||
|
||||
"github.com/gin-gonic/gin"
|
||||
)
|
||||
|
||||
// TokenAuthMiddleware creates a Gin middleware that validates a token from query parameters or request body.
|
||||
func TokenAuthMiddleware() gin.HandlerFunc {
|
||||
return func(c *gin.Context) {
|
||||
// API key authentication
|
||||
apiKey := c.GetHeader("Authorization")
|
||||
if isApiKeyValid(apiKey) {
|
||||
c.Set("api_key", apiKey)
|
||||
c.Next()
|
||||
return
|
||||
}
|
||||
|
||||
var token string
|
||||
|
||||
// Step 1: Check query parameter for token
|
||||
token = c.Query("token")
|
||||
|
||||
// Step 2: If no token in query, check request body for non-GET requests
|
||||
if token == "" && c.Request.Method != http.MethodGet {
|
||||
// Read the body
|
||||
bodyBytes, err := io.ReadAll(c.Request.Body)
|
||||
if err != nil {
|
||||
c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"status": "error", "error": "failed to read request body"})
|
||||
return
|
||||
}
|
||||
|
||||
// Restore the body for downstream handlers
|
||||
c.Request.Body = io.NopCloser(bytes.NewReader(bodyBytes))
|
||||
|
||||
// Try to extract token from JSON body
|
||||
var bodyMap map[string]interface{}
|
||||
if len(bodyBytes) > 0 {
|
||||
if err := json.Unmarshal(bodyBytes, &bodyMap); err == nil {
|
||||
if tokenVal, exists := bodyMap["token"]; exists {
|
||||
if str, ok := tokenVal.(string); ok && str != "" {
|
||||
token = str
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if token == "" {
|
||||
c.AbortWithStatusJSON(http.StatusUnauthorized, gin.H{"status": "error", "error": "token is required"})
|
||||
return
|
||||
}
|
||||
|
||||
exists, err := checkTokenExists(token)
|
||||
if err != nil {
|
||||
c.AbortWithStatusJSON(http.StatusInternalServerError, gin.H{"status": "error", "error": "failed to validate token"})
|
||||
return
|
||||
}
|
||||
if !exists {
|
||||
c.AbortWithStatusJSON(http.StatusUnauthorized, gin.H{"status": "error", "error": "invalid token"})
|
||||
return
|
||||
}
|
||||
|
||||
c.Next()
|
||||
}
|
||||
}
|
||||
|
||||
func checkTokenExists(token string) (bool, error) {
|
||||
_, err := clients.GetClientUUIDByToken(token)
|
||||
|
||||
if err == sql.ErrNoRows {
|
||||
return false, nil
|
||||
}
|
||||
if err == gorm.ErrRecordNotFound {
|
||||
return false, nil
|
||||
}
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
return true, nil
|
||||
}
|
||||
@@ -1,137 +0,0 @@
|
||||
package admin
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
|
||||
"github.com/gin-gonic/gin"
|
||||
"github.com/komari-monitor/komari/database/auditlog"
|
||||
"github.com/komari-monitor/komari/database/clients"
|
||||
"github.com/komari-monitor/komari/database/records"
|
||||
"github.com/komari-monitor/komari/ws"
|
||||
)
|
||||
|
||||
func AddClient(c *gin.Context) {
|
||||
var req struct {
|
||||
Name string `json:"name"`
|
||||
}
|
||||
if err := c.ShouldBindJSON(&req); err != nil || req.Name == "" {
|
||||
uuid, token, err := clients.CreateClient()
|
||||
if err != nil {
|
||||
c.JSON(http.StatusInternalServerError, gin.H{"status": "error", "message": err.Error()})
|
||||
return
|
||||
}
|
||||
c.JSON(http.StatusOK, gin.H{"status": "success", "uuid": uuid, "token": token})
|
||||
return
|
||||
}
|
||||
uuid, token, err := clients.CreateClientWithName(req.Name)
|
||||
if err != nil {
|
||||
c.JSON(http.StatusInternalServerError, gin.H{"status": "error", "message": err.Error()})
|
||||
return
|
||||
}
|
||||
user_uuid, _ := c.Get("uuid")
|
||||
auditlog.Log(c.ClientIP(), user_uuid.(string), "create client:"+uuid, "info")
|
||||
c.JSON(http.StatusOK, gin.H{"status": "success", "uuid": uuid, "token": token, "message": ""})
|
||||
}
|
||||
|
||||
func EditClient(c *gin.Context) {
|
||||
var req = make(map[string]interface{})
|
||||
uuid := c.Param("uuid")
|
||||
if err := c.ShouldBindJSON(&req); err != nil {
|
||||
c.JSON(http.StatusBadRequest, gin.H{"status": "error", "message": err.Error()})
|
||||
return
|
||||
}
|
||||
if uuid == "" {
|
||||
c.JSON(http.StatusBadRequest, gin.H{"status": "error", "message": "Invalid or missing UUID"})
|
||||
return
|
||||
}
|
||||
req["uuid"] = uuid
|
||||
err := clients.SaveClient(req)
|
||||
if err != nil {
|
||||
c.JSON(http.StatusInternalServerError, gin.H{"status": "error", "message": err.Error()})
|
||||
return
|
||||
}
|
||||
user_uuid, _ := c.Get("uuid")
|
||||
auditlog.Log(c.ClientIP(), user_uuid.(string), "edit client:"+uuid, "info")
|
||||
c.JSON(http.StatusOK, gin.H{"status": "success"})
|
||||
}
|
||||
|
||||
func RemoveClient(c *gin.Context) {
|
||||
uuid := c.Param("uuid")
|
||||
err := clients.DeleteClient(uuid)
|
||||
if err != nil {
|
||||
c.JSON(500, gin.H{
|
||||
"status": "error",
|
||||
"error": "Failed to delete client" + err.Error(),
|
||||
})
|
||||
return
|
||||
}
|
||||
user_uuid, _ := c.Get("uuid")
|
||||
auditlog.Log(c.ClientIP(), user_uuid.(string), "delete client:"+uuid, "warn")
|
||||
c.JSON(200, gin.H{"status": "success"})
|
||||
ws.DeleteConnectedClients(uuid)
|
||||
ws.DeleteLatestReport(uuid)
|
||||
}
|
||||
|
||||
func ClearRecord(c *gin.Context) {
|
||||
if err := records.DeleteAll(); err != nil {
|
||||
c.JSON(500, gin.H{
|
||||
"status": "error",
|
||||
"message": "Failed to delete Record" + err.Error(),
|
||||
})
|
||||
return
|
||||
}
|
||||
user_uuid, _ := c.Get("uuid")
|
||||
auditlog.Log(c.ClientIP(), user_uuid.(string), "clear records", "warn")
|
||||
c.JSON(200, gin.H{"status": "success"})
|
||||
}
|
||||
|
||||
func GetClient(c *gin.Context) {
|
||||
uuid := c.Param("uuid")
|
||||
if uuid == "" {
|
||||
c.JSON(400, gin.H{
|
||||
"status": "error",
|
||||
"message": "Invalid or missing UUID",
|
||||
})
|
||||
return
|
||||
}
|
||||
|
||||
result, err := clients.GetClientByUUID(uuid)
|
||||
if err != nil {
|
||||
c.JSON(http.StatusInternalServerError, gin.H{
|
||||
"status": "error",
|
||||
"message": err.Error(),
|
||||
})
|
||||
return
|
||||
}
|
||||
|
||||
c.JSON(http.StatusOK, result)
|
||||
}
|
||||
|
||||
func ListClients(c *gin.Context) {
|
||||
cls, err := clients.GetAllClientBasicInfo()
|
||||
if err != nil {
|
||||
c.JSON(http.StatusInternalServerError, gin.H{"status": "error", "message": err.Error()})
|
||||
return
|
||||
}
|
||||
|
||||
c.JSON(http.StatusOK, cls)
|
||||
}
|
||||
|
||||
func GetClientToken(c *gin.Context) {
|
||||
uuid := c.Param("uuid")
|
||||
if uuid == "" {
|
||||
c.JSON(400, gin.H{
|
||||
"status": "error",
|
||||
"message": "Invalid or missing UUID",
|
||||
})
|
||||
return
|
||||
}
|
||||
|
||||
token, err := clients.GetClientTokenByUUID(uuid)
|
||||
if err != nil {
|
||||
c.JSON(http.StatusInternalServerError, gin.H{"status": "error", "message": err.Error()})
|
||||
return
|
||||
}
|
||||
|
||||
c.JSON(http.StatusOK, gin.H{"status": "success", "token": token, "message:": ""})
|
||||
}
|
||||
@@ -1,28 +0,0 @@
|
||||
package admin
|
||||
|
||||
import (
|
||||
"github.com/gin-gonic/gin"
|
||||
"github.com/komari-monitor/komari/api"
|
||||
"github.com/komari-monitor/komari/database/auditlog"
|
||||
"github.com/komari-monitor/komari/database/dbcore"
|
||||
"github.com/komari-monitor/komari/database/models"
|
||||
)
|
||||
|
||||
func OrderWeight(c *gin.Context) {
|
||||
var req = make(map[string]int)
|
||||
if err := c.ShouldBindJSON(&req); err != nil {
|
||||
api.RespondError(c, 400, "Invalid or missing request body: "+err.Error())
|
||||
return
|
||||
}
|
||||
db := dbcore.GetDBInstance()
|
||||
for uuid, weight := range req {
|
||||
err := db.Model(&models.Client{}).Where("uuid = ?", uuid).Update("weight", weight).Error
|
||||
if err != nil {
|
||||
api.RespondError(c, 500, "Failed to update client weight: "+err.Error())
|
||||
return
|
||||
}
|
||||
}
|
||||
uuid, _ := c.Get("uuid")
|
||||
auditlog.Log(c.ClientIP(), uuid.(string), "order clients", "info")
|
||||
api.RespondSuccess(c, nil)
|
||||
}
|
||||
@@ -1,115 +0,0 @@
|
||||
package clipboard
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
"strconv"
|
||||
|
||||
"github.com/gin-gonic/gin"
|
||||
"github.com/komari-monitor/komari/api"
|
||||
"github.com/komari-monitor/komari/database/auditlog"
|
||||
clipboardDB "github.com/komari-monitor/komari/database/clipboard"
|
||||
"github.com/komari-monitor/komari/database/models"
|
||||
)
|
||||
|
||||
// GetClipboard retrieves a clipboard entry by ID
|
||||
func GetClipboard(c *gin.Context) {
|
||||
idStr := c.Param("id")
|
||||
id, err := strconv.Atoi(idStr)
|
||||
if err != nil {
|
||||
api.RespondError(c, http.StatusBadRequest, "Invalid ID")
|
||||
return
|
||||
}
|
||||
cb, err := clipboardDB.GetClipboardByID(id)
|
||||
if err != nil {
|
||||
api.RespondError(c, http.StatusInternalServerError, "Failed to get clipboard: "+err.Error())
|
||||
return
|
||||
}
|
||||
api.RespondSuccess(c, cb)
|
||||
}
|
||||
|
||||
// ListClipboard lists all clipboard entries
|
||||
func ListClipboard(c *gin.Context) {
|
||||
list, err := clipboardDB.ListClipboard()
|
||||
if err != nil {
|
||||
api.RespondError(c, http.StatusInternalServerError, "Failed to list clipboard: "+err.Error())
|
||||
return
|
||||
}
|
||||
api.RespondSuccess(c, list)
|
||||
}
|
||||
|
||||
// CreateClipboard creates a new clipboard entry
|
||||
func CreateClipboard(c *gin.Context) {
|
||||
var req models.Clipboard
|
||||
if err := c.ShouldBindJSON(&req); err != nil {
|
||||
api.RespondError(c, http.StatusBadRequest, "Invalid request: "+err.Error())
|
||||
return
|
||||
}
|
||||
if err := clipboardDB.CreateClipboard(&req); err != nil {
|
||||
api.RespondError(c, http.StatusInternalServerError, "Failed to create clipboard: "+err.Error())
|
||||
return
|
||||
}
|
||||
userUUID, _ := c.Get("uuid")
|
||||
auditlog.Log(c.ClientIP(), userUUID.(string), "create clipboard:"+strconv.Itoa(req.Id), "info")
|
||||
api.RespondSuccess(c, req)
|
||||
}
|
||||
|
||||
// UpdateClipboard updates an existing clipboard entry
|
||||
func UpdateClipboard(c *gin.Context) {
|
||||
idStr := c.Param("id")
|
||||
id, err := strconv.Atoi(idStr)
|
||||
if err != nil {
|
||||
api.RespondError(c, http.StatusBadRequest, "Invalid ID")
|
||||
return
|
||||
}
|
||||
var fields map[string]interface{}
|
||||
if err := c.ShouldBindJSON(&fields); err != nil {
|
||||
api.RespondError(c, http.StatusBadRequest, "Invalid request: "+err.Error())
|
||||
return
|
||||
}
|
||||
if err := clipboardDB.UpdateClipboardFields(id, fields); err != nil {
|
||||
api.RespondError(c, http.StatusInternalServerError, "Failed to update clipboard: "+err.Error())
|
||||
return
|
||||
}
|
||||
userUUID, _ := c.Get("uuid")
|
||||
auditlog.Log(c.ClientIP(), userUUID.(string), "update clipboard:"+strconv.Itoa(id), "info")
|
||||
api.RespondSuccess(c, nil)
|
||||
}
|
||||
|
||||
// DeleteClipboard deletes a clipboard entry
|
||||
func DeleteClipboard(c *gin.Context) {
|
||||
idStr := c.Param("id")
|
||||
id, err := strconv.Atoi(idStr)
|
||||
if err != nil {
|
||||
api.RespondError(c, http.StatusBadRequest, "Invalid ID")
|
||||
return
|
||||
}
|
||||
if err := clipboardDB.DeleteClipboard(id); err != nil {
|
||||
api.RespondError(c, http.StatusInternalServerError, "Failed to delete clipboard: "+err.Error())
|
||||
return
|
||||
}
|
||||
userUUID, _ := c.Get("uuid")
|
||||
auditlog.Log(c.ClientIP(), userUUID.(string), "delete clipboard:"+strconv.Itoa(id), "warn")
|
||||
api.RespondSuccess(c, nil)
|
||||
}
|
||||
|
||||
// BatchDeleteClipboard deletes multiple clipboard entries
|
||||
func BatchDeleteClipboard(c *gin.Context) {
|
||||
var req struct {
|
||||
IDs []int `json:"ids" binding:"required"`
|
||||
}
|
||||
if err := c.ShouldBindJSON(&req); err != nil {
|
||||
api.RespondError(c, http.StatusBadRequest, "Invalid request: "+err.Error())
|
||||
return
|
||||
}
|
||||
if len(req.IDs) == 0 {
|
||||
api.RespondError(c, http.StatusBadRequest, "IDs cannot be empty")
|
||||
return
|
||||
}
|
||||
if err := clipboardDB.DeleteClipboardBatch(req.IDs); err != nil {
|
||||
api.RespondError(c, http.StatusInternalServerError, "Failed to batch delete clipboard: "+err.Error())
|
||||
return
|
||||
}
|
||||
userUUID, _ := c.Get("uuid")
|
||||
auditlog.Log(c.ClientIP(), userUUID.(string), "batch delete clipboard: "+strconv.Itoa(len(req.IDs))+" items", "warn")
|
||||
api.RespondSuccess(c, nil)
|
||||
}
|
||||
@@ -1,236 +0,0 @@
|
||||
package admin
|
||||
|
||||
import (
|
||||
"archive/zip"
|
||||
"fmt"
|
||||
"io"
|
||||
"log"
|
||||
"net/http"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/gin-gonic/gin"
|
||||
"github.com/komari-monitor/komari/api"
|
||||
"github.com/komari-monitor/komari/cmd/flags"
|
||||
"github.com/komari-monitor/komari/database/dbcore"
|
||||
)
|
||||
|
||||
// copyDatabaseToTemp 安全地复制数据库文件到临时目录
|
||||
// 对于SQLite数据库,会使用BACKUP命令确保数据一致性
|
||||
func copyDatabaseToTemp(dbFilePath, tempDir string) (string, error) {
|
||||
if dbFilePath == "" {
|
||||
return "", fmt.Errorf("database file path is empty")
|
||||
}
|
||||
|
||||
// 检查数据库文件是否存在
|
||||
if _, err := os.Stat(dbFilePath); os.IsNotExist(err) {
|
||||
return "", fmt.Errorf("database file does not exist: %s", dbFilePath)
|
||||
}
|
||||
|
||||
dbFileName := filepath.Base(dbFilePath)
|
||||
tempDbPath := filepath.Join(tempDir, dbFileName)
|
||||
|
||||
// 如果是SQLite数据库,使用SQLite的BACKUP命令来确保一致性
|
||||
if flags.DatabaseType == "sqlite" || flags.DatabaseType == "" {
|
||||
db := dbcore.GetDBInstance()
|
||||
|
||||
// 获取底层的SQLite连接
|
||||
sqlDB, err := db.DB()
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("failed to get underlying database connection: %v", err)
|
||||
}
|
||||
|
||||
// 使用SQLite的BACKUP API来创建一致的备份
|
||||
backupSQL := fmt.Sprintf("BACKUP DATABASE main TO '%s'", tempDbPath)
|
||||
_, err = sqlDB.Exec(backupSQL)
|
||||
if err != nil {
|
||||
// 如果BACKUP命令失败,回退到文件复制方法
|
||||
log.Printf("SQLite BACKUP command failed, falling back to file copy: %v", err)
|
||||
return copyFileToTemp(dbFilePath, tempDbPath)
|
||||
}
|
||||
|
||||
log.Printf("Database backed up using SQLite BACKUP to: %s", tempDbPath)
|
||||
return tempDbPath, nil
|
||||
} else {
|
||||
// 对于非SQLite数据库,直接复制文件
|
||||
return copyFileToTemp(dbFilePath, tempDbPath)
|
||||
}
|
||||
}
|
||||
|
||||
// copyFileToTemp 简单的文件复制到临时目录
|
||||
func copyFileToTemp(srcPath, destPath string) (string, error) {
|
||||
src, err := os.Open(srcPath)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("failed to open source file: %v", err)
|
||||
}
|
||||
defer src.Close()
|
||||
|
||||
dest, err := os.Create(destPath)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("failed to create destination file: %v", err)
|
||||
}
|
||||
defer dest.Close()
|
||||
|
||||
_, err = io.Copy(dest, src)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("failed to copy file: %v", err)
|
||||
}
|
||||
|
||||
log.Printf("Database file copied to: %s", destPath)
|
||||
return destPath, nil
|
||||
}
|
||||
|
||||
// DownloadBackup 用于打包 ./data 目录及数据库文件为 zip 并通过 HTTP 下载
|
||||
func DownloadBackup(c *gin.Context) {
|
||||
backupFileName := fmt.Sprintf("backup-%d.zip", time.Now().UnixMicro())
|
||||
c.Writer.Header().Set("Content-Type", "application/zip")
|
||||
c.Writer.Header().Set("Content-Disposition", fmt.Sprintf("attachment; filename=\"%s\"", backupFileName))
|
||||
|
||||
zipWriter := zip.NewWriter(c.Writer)
|
||||
defer zipWriter.Close()
|
||||
|
||||
// 创建临时目录用于存放数据库备份
|
||||
tempDir, err := os.MkdirTemp("", "komari-backup-*")
|
||||
if err != nil {
|
||||
api.RespondError(c, http.StatusInternalServerError, fmt.Sprintf("Error creating temporary directory: %v", err))
|
||||
return
|
||||
}
|
||||
defer os.RemoveAll(tempDir) // 清理临时目录
|
||||
|
||||
err = filepath.Walk("./data", func(filePath string, info os.FileInfo, err error) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// 获取在 zip 包内的相对路径
|
||||
relPath, err := filepath.Rel("./data", filePath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// 跳过根目录本身,只打包内容
|
||||
if relPath == "." {
|
||||
return nil
|
||||
}
|
||||
|
||||
// zip 内路径统一用正斜杠
|
||||
zipPath := filepath.ToSlash(relPath)
|
||||
|
||||
if info.IsDir() {
|
||||
// 在 zip 中创建目录项
|
||||
_, err = zipWriter.CreateHeader(&zip.FileHeader{
|
||||
Name: zipPath + "/",
|
||||
Method: zip.Deflate, // zip.Store 不压缩
|
||||
Modified: info.ModTime(),
|
||||
})
|
||||
return err
|
||||
}
|
||||
|
||||
// 普通文件,写入 zip
|
||||
file, err := os.Open(filePath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer file.Close()
|
||||
|
||||
writer, err := zipWriter.CreateHeader(&zip.FileHeader{
|
||||
Name: zipPath,
|
||||
Method: zip.Deflate,
|
||||
Modified: info.ModTime(),
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
_, err = io.Copy(writer, file)
|
||||
return err
|
||||
})
|
||||
|
||||
if err != nil {
|
||||
api.RespondError(c, http.StatusInternalServerError, fmt.Sprintf("Error archiving data folder: %v", err))
|
||||
return
|
||||
}
|
||||
|
||||
// 获取 ./data 和数据库文件的绝对路径以进行可靠的比较
|
||||
dataAbsPath, err := filepath.Abs("./data")
|
||||
if err != nil {
|
||||
api.RespondError(c, http.StatusInternalServerError, fmt.Sprintf("Error getting absolute path for data directory: %v", err))
|
||||
return
|
||||
}
|
||||
|
||||
dbFilePath := flags.DatabaseFile
|
||||
dbAbsPath, err := filepath.Abs(dbFilePath)
|
||||
if err != nil {
|
||||
// 如果无法解析数据库路径,可能是一个配置问题,但我们记录并跳过,避免备份失败
|
||||
log.Printf("Could not determine absolute path for database file '%s', skipping explicit addition. Error: %v\n", dbFilePath, err)
|
||||
return
|
||||
}
|
||||
|
||||
// 如果数据库文件的绝对路径不是以 ./data 目录的绝对路径开头,则单独添加它
|
||||
if !strings.HasPrefix(dbAbsPath, dataAbsPath) {
|
||||
// 使用安全的数据库复制方法
|
||||
tempDbPath, err := copyDatabaseToTemp(dbFilePath, tempDir)
|
||||
if err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
log.Printf("Database file '%s' does not exist, skipping.\n", dbFilePath)
|
||||
} else {
|
||||
api.RespondError(c, http.StatusInternalServerError, fmt.Sprintf("Error copying database file: %v", err))
|
||||
return
|
||||
}
|
||||
} else {
|
||||
// 将临时数据库文件添加到zip
|
||||
dbInfo, err := os.Stat(tempDbPath)
|
||||
if err != nil {
|
||||
api.RespondError(c, http.StatusInternalServerError, fmt.Sprintf("Error stating temp database file: %v", err))
|
||||
return
|
||||
}
|
||||
|
||||
file, err := os.Open(tempDbPath)
|
||||
if err != nil {
|
||||
api.RespondError(c, http.StatusInternalServerError, fmt.Sprintf("Error opening temp database file: %v", err))
|
||||
return
|
||||
}
|
||||
defer file.Close()
|
||||
|
||||
writer, err := zipWriter.CreateHeader(&zip.FileHeader{
|
||||
Name: filepath.Base(dbFilePath), // 使用原始数据库文件名
|
||||
Method: zip.Deflate,
|
||||
Modified: dbInfo.ModTime(),
|
||||
})
|
||||
if err != nil {
|
||||
api.RespondError(c, http.StatusInternalServerError, fmt.Sprintf("Error creating zip entry for database file: %v", err))
|
||||
return
|
||||
}
|
||||
|
||||
_, err = io.Copy(writer, file)
|
||||
if err != nil {
|
||||
api.RespondError(c, http.StatusInternalServerError, fmt.Sprintf("Error writing database file to zip: %v", err))
|
||||
return
|
||||
}
|
||||
}
|
||||
} else {
|
||||
log.Printf("Database file '%s' is within './data', skipping explicit addition.\n", dbFilePath)
|
||||
}
|
||||
|
||||
// 添加备份标记文件
|
||||
markupContent := "此文件为 Komari 备份标记文件,请勿删除。\nThis is a Komari backup markup file, please do not delete.\n\n备份时间 / Backup Time: " + time.Now().Format(time.RFC3339)
|
||||
markupWriter, err := zipWriter.CreateHeader(&zip.FileHeader{
|
||||
Name: "komari-backup-markup",
|
||||
Method: zip.Deflate,
|
||||
Modified: time.Now(),
|
||||
})
|
||||
if err != nil {
|
||||
api.RespondError(c, http.StatusInternalServerError, fmt.Sprintf("Error creating backup markup file: %v", err))
|
||||
return
|
||||
}
|
||||
|
||||
_, err = markupWriter.Write([]byte(markupContent))
|
||||
if err != nil {
|
||||
api.RespondError(c, http.StatusInternalServerError, fmt.Sprintf("Error writing backup markup file: %v", err))
|
||||
return
|
||||
}
|
||||
|
||||
// zipWriter.Close() 由 defer 自动调用
|
||||
}
|
||||
@@ -1,98 +0,0 @@
|
||||
package admin
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"time"
|
||||
|
||||
"github.com/gin-gonic/gin"
|
||||
"github.com/gorilla/websocket"
|
||||
"github.com/komari-monitor/komari/api"
|
||||
"github.com/komari-monitor/komari/database/auditlog"
|
||||
"github.com/komari-monitor/komari/database/models"
|
||||
"github.com/komari-monitor/komari/database/tasks"
|
||||
"github.com/komari-monitor/komari/utils"
|
||||
"github.com/komari-monitor/komari/ws"
|
||||
)
|
||||
|
||||
// 接受数据类型:
|
||||
// - command: string
|
||||
// - clients: []string (客户端 UUID 列表)
|
||||
func Exec(c *gin.Context) {
|
||||
var req struct {
|
||||
Command string `json:"command" binding:"required"`
|
||||
Clients []string `json:"clients" binding:"required"`
|
||||
}
|
||||
var onlineClients []string
|
||||
var offlineClients []string
|
||||
if err := c.ShouldBindJSON(&req); err != nil {
|
||||
api.RespondError(c, 400, "Invalid or missing request body: "+err.Error())
|
||||
return
|
||||
}
|
||||
// for uuid := range ws.GetConnectedClients() {
|
||||
// if contain(req.Clients, uuid) {
|
||||
// onlineClients = append(onlineClients, uuid)
|
||||
// }
|
||||
// // else {
|
||||
// // api.RespondError(c, 400, "Client not connected: "+uuid)
|
||||
// // return
|
||||
// // }
|
||||
// }
|
||||
for _, uuid := range req.Clients {
|
||||
if client := ws.GetConnectedClients()[uuid]; client != nil {
|
||||
onlineClients = append(onlineClients, uuid)
|
||||
} else {
|
||||
offlineClients = append(offlineClients, uuid)
|
||||
}
|
||||
}
|
||||
if len(onlineClients) == 0 {
|
||||
api.RespondError(c, 400, "No clients connected")
|
||||
return
|
||||
}
|
||||
taskId := utils.GenerateRandomString(16)
|
||||
if err := tasks.CreateTask(taskId, append(onlineClients, offlineClients...), req.Command); err != nil {
|
||||
api.RespondError(c, 500, "Failed to create task: "+err.Error())
|
||||
return
|
||||
}
|
||||
for _, uuid := range onlineClients {
|
||||
var send struct {
|
||||
Message string `json:"message"`
|
||||
Command string `json:"command"`
|
||||
TaskId string `json:"task_id"`
|
||||
}
|
||||
send.Message = "exec"
|
||||
send.Command = req.Command
|
||||
send.TaskId = taskId
|
||||
|
||||
payload, _ := json.Marshal(send)
|
||||
client := ws.GetConnectedClients()[uuid]
|
||||
if client != nil {
|
||||
if err := client.WriteMessage(websocket.TextMessage, payload); err != nil {
|
||||
api.RespondError(c, 400, "Client connection is broke: "+uuid)
|
||||
return
|
||||
}
|
||||
} else {
|
||||
api.RespondError(c, 400, "Client connection is null: "+uuid)
|
||||
return
|
||||
}
|
||||
}
|
||||
uuid, _ := c.Get("uuid")
|
||||
auditlog.Log(c.ClientIP(), uuid.(string), "REC, task id: "+taskId, "warn")
|
||||
api.RespondSuccess(c, gin.H{
|
||||
"task_id": taskId,
|
||||
"clients": onlineClients,
|
||||
})
|
||||
if len(offlineClients) > 0 {
|
||||
for _, uuid := range offlineClients {
|
||||
tasks.SaveTaskResult(taskId, uuid, "Client offline!", -1, models.FromTime(time.Now()))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// func contain(clients []string, uuid string) bool {
|
||||
// for _, client := range clients {
|
||||
// if client == uuid {
|
||||
// return true
|
||||
// }
|
||||
// }
|
||||
// return false
|
||||
// }
|
||||
@@ -1,49 +0,0 @@
|
||||
package log
|
||||
|
||||
import (
|
||||
"strconv"
|
||||
|
||||
"github.com/gin-gonic/gin"
|
||||
"github.com/komari-monitor/komari/api"
|
||||
"github.com/komari-monitor/komari/database/dbcore"
|
||||
"github.com/komari-monitor/komari/database/models"
|
||||
)
|
||||
|
||||
func GetLogs(c *gin.Context) {
|
||||
limit := c.Query("limit")
|
||||
if limit == "" {
|
||||
limit = "100" // Default to 100 logs if not specified
|
||||
}
|
||||
page := c.Query("page")
|
||||
if page == "" {
|
||||
page = "1" // Default to page 1 if not specified
|
||||
}
|
||||
// Convert limit and page to integers
|
||||
// If conversion fails, return an error
|
||||
limitInt, err := strconv.Atoi(limit)
|
||||
if err != nil || limitInt <= 0 {
|
||||
api.RespondError(c, 400, "Invalid limit: "+limit)
|
||||
return
|
||||
}
|
||||
pageInt, err := strconv.Atoi(page)
|
||||
if err != nil || pageInt <= 0 {
|
||||
api.RespondError(c, 400, "Invalid page: "+page)
|
||||
return
|
||||
}
|
||||
db := dbcore.GetDBInstance()
|
||||
var logs []models.Log
|
||||
// 添加分页:计算偏移量并限制数量
|
||||
offset := (pageInt - 1) * limitInt
|
||||
|
||||
var total int64
|
||||
if err := db.Model(&models.Log{}).Count(&total).Error; err != nil {
|
||||
api.RespondError(c, 500, "Failed to count logs: "+err.Error())
|
||||
return
|
||||
}
|
||||
|
||||
if err := db.Order("time desc").Limit(limitInt).Offset(offset).Find(&logs).Error; err != nil {
|
||||
api.RespondError(c, 500, "Failed to retrieve logs: "+err.Error())
|
||||
return
|
||||
}
|
||||
api.RespondSuccess(c, gin.H{"logs": logs, "total": total})
|
||||
}
|
||||
@@ -1,63 +0,0 @@
|
||||
package admin
|
||||
|
||||
import (
|
||||
"github.com/gin-gonic/gin"
|
||||
"github.com/komari-monitor/komari/api"
|
||||
"github.com/komari-monitor/komari/database"
|
||||
"github.com/komari-monitor/komari/database/config"
|
||||
"github.com/komari-monitor/komari/database/models"
|
||||
"github.com/komari-monitor/komari/utils/messageSender"
|
||||
"github.com/komari-monitor/komari/utils/messageSender/factory"
|
||||
)
|
||||
|
||||
func GetMessageSenderProvider(c *gin.Context) {
|
||||
provider := c.Query("provider")
|
||||
if provider != "" {
|
||||
// 如果指定了provider,返回单个提供者的配置
|
||||
config, err := database.GetMessageSenderConfigByName(provider)
|
||||
if err != nil {
|
||||
api.RespondError(c, 404, "Provider not found: "+err.Error())
|
||||
return
|
||||
}
|
||||
api.RespondSuccess(c, config)
|
||||
return
|
||||
}
|
||||
// 否则返回所有提供者的配置项模板
|
||||
providers := factory.GetSenderConfigs()
|
||||
if len(providers) == 0 {
|
||||
api.RespondError(c, 404, "No message sender providers found")
|
||||
return
|
||||
}
|
||||
api.RespondSuccess(c, providers)
|
||||
}
|
||||
|
||||
func SetMessageSenderProvider(c *gin.Context) {
|
||||
var senderConfig models.MessageSenderProvider
|
||||
if err := c.ShouldBindJSON(&senderConfig); err != nil {
|
||||
api.RespondError(c, 400, "Invalid configuration: "+err.Error())
|
||||
return
|
||||
}
|
||||
if senderConfig.Name == "" {
|
||||
api.RespondError(c, 400, "Provider name is required")
|
||||
return
|
||||
}
|
||||
_, exists := factory.GetConstructor(senderConfig.Name)
|
||||
if !exists {
|
||||
api.RespondError(c, 404, "Provider not found: "+senderConfig.Name)
|
||||
return
|
||||
}
|
||||
if err := database.SaveMessageSenderConfig(&senderConfig); err != nil {
|
||||
api.RespondError(c, 500, "Failed to save message sender provider configuration: "+err.Error())
|
||||
return
|
||||
}
|
||||
cfg, _ := config.Get()
|
||||
// 正在使用,重载
|
||||
if cfg.NotificationMethod == senderConfig.Name {
|
||||
err := messageSender.LoadProvider(senderConfig.Name, senderConfig.Addition)
|
||||
if err != nil {
|
||||
api.RespondError(c, 500, "Failed to load message sender provider: "+err.Error())
|
||||
return
|
||||
}
|
||||
}
|
||||
api.RespondSuccess(c, gin.H{"message": "Message sender provider set successfully"})
|
||||
}
|
||||
@@ -1,90 +0,0 @@
|
||||
package notification
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
|
||||
"github.com/gin-gonic/gin"
|
||||
"github.com/komari-monitor/komari/api"
|
||||
"github.com/komari-monitor/komari/database/models"
|
||||
"github.com/komari-monitor/komari/database/notification"
|
||||
)
|
||||
|
||||
// POST body: clients []string, name string, metric string, threshold float32, ratio float32, interval int
|
||||
func AddLoadNotification(c *gin.Context) {
|
||||
var req struct {
|
||||
Clients []string `json:"clients" binding:"required"`
|
||||
Name string `json:"name"`
|
||||
Metric string `json:"metric" binding:"required"`
|
||||
Threshold float32 `json:"threshold" binding:"required"` // 阈值百分比
|
||||
Ratio float32 `json:"ratio" binding:"required"` // 达标时间比
|
||||
Interval int `json:"interval" binding:"required"` // 间隔时间,单位秒
|
||||
}
|
||||
|
||||
if err := c.ShouldBindJSON(&req); err != nil {
|
||||
api.RespondError(c, http.StatusBadRequest, err.Error())
|
||||
return
|
||||
}
|
||||
|
||||
if req.Interval > 4*60 || req.Interval <= 0 {
|
||||
api.RespondError(c, http.StatusBadRequest, "Interval must be between 1 and 240 minutes")
|
||||
return
|
||||
}
|
||||
if req.Ratio <= 0 || req.Ratio > 1 {
|
||||
api.RespondError(c, http.StatusBadRequest, "Ratio must be between 0 and 1")
|
||||
return
|
||||
}
|
||||
|
||||
if taskID, err := notification.AddLoadNotification(req.Clients, req.Name, req.Metric, req.Threshold, req.Ratio, req.Interval); err != nil {
|
||||
api.RespondError(c, http.StatusInternalServerError, err.Error())
|
||||
} else {
|
||||
api.RespondSuccess(c, gin.H{"task_id": taskID})
|
||||
}
|
||||
}
|
||||
|
||||
// POST body: id []uint
|
||||
func DeleteLoadNotification(c *gin.Context) {
|
||||
var req struct {
|
||||
ID []uint `json:"id" binding:"required"`
|
||||
}
|
||||
if err := c.ShouldBindJSON(&req); err != nil {
|
||||
api.RespondError(c, http.StatusBadRequest, err.Error())
|
||||
return
|
||||
}
|
||||
|
||||
if err := notification.DeleteLoadNotification(req.ID); err != nil {
|
||||
api.RespondError(c, http.StatusInternalServerError, err.Error())
|
||||
} else {
|
||||
api.RespondSuccess(c, nil)
|
||||
}
|
||||
}
|
||||
|
||||
// POST body: id []uint, updates map[string]interface{}
|
||||
func EditLoadNotification(c *gin.Context) {
|
||||
var req struct {
|
||||
Notifications []*models.LoadNotification `json:"notifications" binding:"required"`
|
||||
}
|
||||
|
||||
if err := c.ShouldBindJSON(&req); err != nil {
|
||||
api.RespondError(c, http.StatusBadRequest, "Invalid request data")
|
||||
return
|
||||
}
|
||||
|
||||
if err := notification.EditLoadNotification(req.Notifications); err != nil {
|
||||
api.RespondError(c, http.StatusInternalServerError, err.Error())
|
||||
} else {
|
||||
// for _, notification := range req.Notifications {
|
||||
// notification.DeleteLoadNotification([]uint{notification.Id})
|
||||
// }
|
||||
api.RespondSuccess(c, nil)
|
||||
}
|
||||
}
|
||||
|
||||
func GetAllLoadNotifications(c *gin.Context) {
|
||||
notifications, err := notification.GetAllLoadNotifications()
|
||||
if err != nil {
|
||||
api.RespondError(c, http.StatusInternalServerError, err.Error())
|
||||
return
|
||||
}
|
||||
|
||||
api.RespondSuccess(c, notifications)
|
||||
}
|
||||
@@ -1,109 +0,0 @@
|
||||
package notification
|
||||
|
||||
import (
|
||||
"github.com/gin-gonic/gin"
|
||||
"github.com/komari-monitor/komari/api"
|
||||
"github.com/komari-monitor/komari/database/dbcore"
|
||||
"github.com/komari-monitor/komari/database/models"
|
||||
"gorm.io/gorm/clause"
|
||||
)
|
||||
|
||||
// POST body : []uuid
|
||||
func EnableOfflineNotification(c *gin.Context) {
|
||||
var uuids []string
|
||||
if err := c.ShouldBindJSON(&uuids); err != nil {
|
||||
api.RespondError(c, 400, "Invalid request body: "+err.Error())
|
||||
return
|
||||
}
|
||||
var notifications []models.OfflineNotification
|
||||
for _, uuid := range uuids {
|
||||
notifications = append(notifications, models.OfflineNotification{
|
||||
Client: uuid,
|
||||
Enable: true,
|
||||
})
|
||||
}
|
||||
err := dbcore.GetDBInstance().Model(&models.OfflineNotification{}).
|
||||
Clauses(clause.OnConflict{
|
||||
Columns: []clause.Column{{Name: "client"}},
|
||||
DoUpdates: clause.AssignmentColumns([]string{"enable"}),
|
||||
}).
|
||||
Select("client", "enable").
|
||||
Create(notifications).Error
|
||||
if err != nil {
|
||||
api.RespondError(c, 500, "Failed to enable offline notifications: "+err.Error())
|
||||
return
|
||||
}
|
||||
api.RespondSuccess(c, nil)
|
||||
}
|
||||
|
||||
// POST body : []uuid
|
||||
func DisableOfflineNotification(c *gin.Context) {
|
||||
var uuids []string
|
||||
if err := c.ShouldBindJSON(&uuids); err != nil {
|
||||
api.RespondError(c, 400, "Invalid request body: "+err.Error())
|
||||
return
|
||||
}
|
||||
var notifications []models.OfflineNotification
|
||||
for _, uuid := range uuids {
|
||||
notifications = append(notifications, models.OfflineNotification{
|
||||
Client: uuid,
|
||||
Enable: false,
|
||||
})
|
||||
}
|
||||
err := dbcore.GetDBInstance().Model(&models.OfflineNotification{}).
|
||||
Clauses(clause.OnConflict{
|
||||
Columns: []clause.Column{{Name: "client"}},
|
||||
DoUpdates: clause.AssignmentColumns([]string{"enable"}),
|
||||
}).
|
||||
Select("client", "enable").
|
||||
Create(notifications).Error
|
||||
if err != nil {
|
||||
api.RespondError(c, 500, "Failed to disable offline notifications: "+err.Error())
|
||||
return
|
||||
}
|
||||
api.RespondSuccess(c, nil)
|
||||
}
|
||||
|
||||
func EditOfflineNotification(c *gin.Context) {
|
||||
var notifications []models.OfflineNotification
|
||||
if err := c.ShouldBindJSON(¬ifications); err != nil {
|
||||
api.RespondError(c, 400, "Invalid request body: "+err.Error())
|
||||
return
|
||||
}
|
||||
if len(notifications) == 0 {
|
||||
api.RespondError(c, 400, "At least one notification is required")
|
||||
return
|
||||
}
|
||||
for _, noti := range notifications {
|
||||
if noti.Client == "" {
|
||||
api.RespondError(c, 400, "Client UUID cannot be empty")
|
||||
return
|
||||
}
|
||||
if noti.GracePeriod <= 0 {
|
||||
api.RespondError(c, 400, "GracePeriod must be a positive integer")
|
||||
return
|
||||
}
|
||||
}
|
||||
err := dbcore.GetDBInstance().Model(&models.OfflineNotification{}).
|
||||
Clauses(clause.OnConflict{
|
||||
Columns: []clause.Column{{Name: "client"}},
|
||||
DoUpdates: clause.AssignmentColumns([]string{"enable", "grace_period"}),
|
||||
}).
|
||||
Select("*").
|
||||
Create(notifications).Error
|
||||
if err != nil {
|
||||
api.RespondError(c, 500, "Failed to edit offline notifications: "+err.Error())
|
||||
return
|
||||
}
|
||||
api.RespondSuccess(c, nil)
|
||||
}
|
||||
|
||||
func ListOfflineNotifications(c *gin.Context) {
|
||||
var notifications []models.OfflineNotification
|
||||
err := dbcore.GetDBInstance().Model(&models.OfflineNotification{}).Find(¬ifications).Error
|
||||
if err != nil {
|
||||
api.RespondError(c, 500, "Failed to list offline notifications: "+err.Error())
|
||||
return
|
||||
}
|
||||
api.RespondSuccess(c, notifications)
|
||||
}
|
||||
@@ -1,93 +0,0 @@
|
||||
package admin
|
||||
|
||||
import (
|
||||
"github.com/gin-gonic/gin"
|
||||
"github.com/komari-monitor/komari/api"
|
||||
"github.com/komari-monitor/komari/database"
|
||||
"github.com/komari-monitor/komari/database/accounts"
|
||||
"github.com/komari-monitor/komari/database/config"
|
||||
"github.com/komari-monitor/komari/database/models"
|
||||
"github.com/komari-monitor/komari/utils/oauth"
|
||||
"github.com/komari-monitor/komari/utils/oauth/factory"
|
||||
)
|
||||
|
||||
func BindingExternalAccount(c *gin.Context) {
|
||||
session, _ := c.Cookie("session_token")
|
||||
user, err := accounts.GetUserBySession(session)
|
||||
if err != nil {
|
||||
api.RespondError(c, 500, "No user found: "+err.Error())
|
||||
return
|
||||
}
|
||||
|
||||
c.SetCookie("binding_external_account", user.UUID, 3600, "/", "", false, true)
|
||||
c.Redirect(302, "/api/oauth")
|
||||
}
|
||||
func UnbindExternalAccount(c *gin.Context) {
|
||||
session, _ := c.Cookie("session_token")
|
||||
user, err := accounts.GetUserBySession(session)
|
||||
if err != nil {
|
||||
api.RespondError(c, 500, "No user found: "+err.Error())
|
||||
return
|
||||
}
|
||||
|
||||
err = accounts.UnbindExternalAccount(user.UUID)
|
||||
if err != nil {
|
||||
api.RespondError(c, 500, "Failed to unbind external account: "+err.Error())
|
||||
return
|
||||
}
|
||||
|
||||
api.RespondSuccess(c, nil)
|
||||
}
|
||||
|
||||
func GetOidcProvider(c *gin.Context) {
|
||||
provider := c.Query("provider")
|
||||
if provider != "" {
|
||||
// 如果指定了provider,返回单个提供者的配置
|
||||
config, err := database.GetOidcConfigByName(provider)
|
||||
if err != nil {
|
||||
api.RespondError(c, 404, "Provider not found: "+err.Error())
|
||||
return
|
||||
}
|
||||
api.RespondSuccess(c, config)
|
||||
return
|
||||
}
|
||||
// 否则返回所有提供者的配置
|
||||
providers := factory.GetProviderConfigs()
|
||||
if len(providers) == 0 {
|
||||
api.RespondError(c, 404, "No OIDC providers found")
|
||||
return
|
||||
}
|
||||
api.RespondSuccess(c, providers)
|
||||
}
|
||||
|
||||
func SetOidcProvider(c *gin.Context) {
|
||||
var oidcConfig models.OidcProvider
|
||||
if err := c.ShouldBindJSON(&oidcConfig); err != nil {
|
||||
api.RespondError(c, 400, "Invalid configuration: "+err.Error())
|
||||
return
|
||||
}
|
||||
if oidcConfig.Name == "" {
|
||||
api.RespondError(c, 400, "Provider name is required")
|
||||
return
|
||||
}
|
||||
_, exists := factory.GetConstructor(oidcConfig.Name)
|
||||
if !exists {
|
||||
api.RespondError(c, 404, "Provider not found: "+oidcConfig.Name)
|
||||
return
|
||||
}
|
||||
|
||||
if err := database.SaveOidcConfig(&oidcConfig); err != nil {
|
||||
api.RespondError(c, 500, "Failed to save OIDC provider configuration: "+err.Error())
|
||||
return
|
||||
}
|
||||
cfg, _ := config.Get()
|
||||
// 正在使用,重载
|
||||
if cfg.OAuthProvider == oidcConfig.Name {
|
||||
err := oauth.LoadProvider(oidcConfig.Name, oidcConfig.Addition)
|
||||
if err != nil {
|
||||
api.RespondError(c, 500, "Failed to load OIDC provider: "+err.Error())
|
||||
return
|
||||
}
|
||||
}
|
||||
api.RespondSuccess(c, gin.H{"message": "OIDC provider set successfully"})
|
||||
}
|
||||
@@ -1,80 +0,0 @@
|
||||
package admin
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
|
||||
"github.com/gin-gonic/gin"
|
||||
"github.com/komari-monitor/komari/api"
|
||||
"github.com/komari-monitor/komari/database/models"
|
||||
"github.com/komari-monitor/komari/database/tasks"
|
||||
)
|
||||
|
||||
// POST body: clients []string, target, task_type string, interval int
|
||||
func AddPingTask(c *gin.Context) {
|
||||
var req struct {
|
||||
Clients []string `json:"clients" binding:"required"`
|
||||
Name string `json:"name" binding:"required"`
|
||||
Target string `json:"target" binding:"required"`
|
||||
TaskType string `json:"type" binding:"required"` // icmp, tcp, http
|
||||
Interval int `json:"interval" binding:"required"` // 间隔时间,单位秒
|
||||
}
|
||||
|
||||
if err := c.ShouldBindJSON(&req); err != nil {
|
||||
api.RespondError(c, http.StatusBadRequest, err.Error())
|
||||
return
|
||||
}
|
||||
|
||||
if taskID, err := tasks.AddPingTask(req.Clients, req.Name, req.Target, req.TaskType, req.Interval); err != nil {
|
||||
api.RespondError(c, http.StatusInternalServerError, err.Error())
|
||||
} else {
|
||||
api.RespondSuccess(c, gin.H{"task_id": taskID})
|
||||
}
|
||||
}
|
||||
|
||||
// POST body: id []uint
|
||||
func DeletePingTask(c *gin.Context) {
|
||||
var req struct {
|
||||
ID []uint `json:"id" binding:"required"`
|
||||
}
|
||||
if err := c.ShouldBindJSON(&req); err != nil {
|
||||
api.RespondError(c, http.StatusBadRequest, err.Error())
|
||||
return
|
||||
}
|
||||
|
||||
if err := tasks.DeletePingTask(req.ID); err != nil {
|
||||
api.RespondError(c, http.StatusInternalServerError, err.Error())
|
||||
} else {
|
||||
api.RespondSuccess(c, nil)
|
||||
}
|
||||
}
|
||||
|
||||
// POST body: id []uint, updates map[string]interface{}
|
||||
func EditPingTask(c *gin.Context) {
|
||||
var req struct {
|
||||
Tasks []*models.PingTask `json:"tasks" binding:"required"`
|
||||
}
|
||||
|
||||
if err := c.ShouldBindJSON(&req); err != nil {
|
||||
api.RespondError(c, http.StatusBadRequest, "Invalid request data")
|
||||
return
|
||||
}
|
||||
|
||||
if err := tasks.EditPingTask(req.Tasks); err != nil {
|
||||
api.RespondError(c, http.StatusInternalServerError, err.Error())
|
||||
} else {
|
||||
// for _, task := range req.Tasks {
|
||||
// tasks.DeletePingRecords([]uint{task.Id})
|
||||
// }
|
||||
api.RespondSuccess(c, nil)
|
||||
}
|
||||
}
|
||||
|
||||
func GetAllPingTasks(c *gin.Context) {
|
||||
tasks, err := tasks.GetAllPingTasks()
|
||||
if err != nil {
|
||||
api.RespondError(c, http.StatusInternalServerError, err.Error())
|
||||
return
|
||||
}
|
||||
|
||||
api.RespondSuccess(c, tasks)
|
||||
}
|
||||
@@ -1,50 +0,0 @@
|
||||
package admin
|
||||
|
||||
import (
|
||||
"github.com/komari-monitor/komari/api"
|
||||
"github.com/komari-monitor/komari/database/accounts"
|
||||
"github.com/komari-monitor/komari/database/auditlog"
|
||||
|
||||
"github.com/gin-gonic/gin"
|
||||
)
|
||||
|
||||
func GetSessions(c *gin.Context) {
|
||||
|
||||
ss, err := accounts.GetAllSessions()
|
||||
if err != nil {
|
||||
api.RespondError(c, 500, "Failed to retrieve sessions: "+err.Error())
|
||||
return
|
||||
}
|
||||
current, _ := c.Cookie("session_token")
|
||||
c.JSON(200, gin.H{"status": "success", "current": current, "data": ss})
|
||||
}
|
||||
|
||||
func DeleteSession(c *gin.Context) {
|
||||
var req struct {
|
||||
Session string `json:"session" binding:"required"`
|
||||
}
|
||||
if err := c.ShouldBindJSON(&req); err != nil {
|
||||
api.RespondError(c, 400, "Invalid request: "+err.Error())
|
||||
return
|
||||
}
|
||||
err := accounts.DeleteSession(req.Session)
|
||||
if err != nil {
|
||||
api.RespondError(c, 500, "Failed to delete session: "+err.Error())
|
||||
return
|
||||
}
|
||||
uuid, _ := c.Get("uuid")
|
||||
auditlog.Log(c.ClientIP(), uuid.(string), "delete session", "info")
|
||||
api.RespondSuccess(c, nil)
|
||||
}
|
||||
|
||||
func DeleteAllSession(c *gin.Context) {
|
||||
|
||||
err := accounts.DeleteAllSessions()
|
||||
if err != nil {
|
||||
api.RespondError(c, 500, "Failed to delete all sessions: "+err.Error())
|
||||
return
|
||||
}
|
||||
uuid, _ := c.Get("uuid")
|
||||
auditlog.Log(c.ClientIP(), uuid.(string), "delete all sessions", "warn")
|
||||
api.RespondSuccess(c, nil)
|
||||
}
|
||||
@@ -1,81 +0,0 @@
|
||||
package admin
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
|
||||
"github.com/komari-monitor/komari/api"
|
||||
"github.com/komari-monitor/komari/database/auditlog"
|
||||
"github.com/komari-monitor/komari/database/config"
|
||||
"github.com/komari-monitor/komari/database/models"
|
||||
"github.com/komari-monitor/komari/database/records"
|
||||
"github.com/komari-monitor/komari/database/tasks"
|
||||
|
||||
"github.com/gin-gonic/gin"
|
||||
)
|
||||
|
||||
// GetSettings 获取自定义配置
|
||||
func GetSettings(c *gin.Context) {
|
||||
cst, err := config.Get()
|
||||
if err != nil {
|
||||
if err == sql.ErrNoRows {
|
||||
//override
|
||||
cst = models.Config{Sitename: "Komari"}
|
||||
cst.ID = 1
|
||||
config.Save(cst)
|
||||
api.RespondSuccess(c, cst)
|
||||
return
|
||||
}
|
||||
c.JSON(500, gin.H{
|
||||
"status": "error",
|
||||
"message": "Internal Server Error: " + err.Error(),
|
||||
})
|
||||
}
|
||||
api.RespondSuccess(c, cst)
|
||||
}
|
||||
|
||||
// EditSettings 更新自定义配置
|
||||
func EditSettings(c *gin.Context) {
|
||||
cfg := make(map[string]interface{})
|
||||
if err := c.ShouldBindJSON(&cfg); err != nil {
|
||||
api.RespondError(c, 400, "Invalid or missing request body: "+err.Error())
|
||||
return
|
||||
}
|
||||
|
||||
cfg["id"] = 1 // Only one record
|
||||
if err := config.Update(cfg); err != nil {
|
||||
api.RespondError(c, 500, "Failed to update settings: "+err.Error())
|
||||
return
|
||||
}
|
||||
|
||||
uuid, _ := c.Get("uuid")
|
||||
message := "update settings: "
|
||||
for key := range cfg {
|
||||
ignoredKeys := []string{"id", "updated_at"}
|
||||
if contains(ignoredKeys, key) {
|
||||
continue
|
||||
}
|
||||
message += key + ", "
|
||||
}
|
||||
if len(message) > 2 {
|
||||
message = message[:len(message)-2]
|
||||
}
|
||||
auditlog.Log(c.ClientIP(), uuid.(string), message, "info")
|
||||
api.RespondSuccess(c, nil)
|
||||
}
|
||||
|
||||
func contains(slice []string, item string) bool {
|
||||
for _, v := range slice {
|
||||
if v == item {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func ClearAllRecords(c *gin.Context) {
|
||||
records.DeleteAll()
|
||||
tasks.DeleteAllPingRecords()
|
||||
uuid, _ := c.Get("uuid")
|
||||
auditlog.Log(c.ClientIP(), uuid.(string), "clear all records", "info")
|
||||
api.RespondSuccess(c, nil)
|
||||
}
|
||||
@@ -1,154 +0,0 @@
|
||||
package admin
|
||||
|
||||
import (
|
||||
"github.com/gin-gonic/gin"
|
||||
"github.com/komari-monitor/komari/api"
|
||||
"github.com/komari-monitor/komari/database/tasks"
|
||||
)
|
||||
|
||||
func GetTasks(c *gin.Context) {
|
||||
dbTasks, err := tasks.GetAllTasks()
|
||||
if err != nil {
|
||||
api.RespondError(c, 500, "Failed to retrieve tasks: "+err.Error())
|
||||
return
|
||||
}
|
||||
var responseTasks []gin.H
|
||||
for _, t := range dbTasks {
|
||||
results, err := tasks.GetTaskResultsByTaskId(t.TaskId)
|
||||
if err != nil {
|
||||
api.RespondError(c, 500, "Failed to retrieve task results: "+err.Error())
|
||||
return
|
||||
}
|
||||
|
||||
var filteredResults []gin.H
|
||||
for _, r := range results {
|
||||
filteredResults = append(filteredResults, gin.H{
|
||||
"client": r.Client,
|
||||
"result": r.Result,
|
||||
"exit_code": r.ExitCode,
|
||||
"finished_at": r.FinishedAt,
|
||||
"created_at": r.CreatedAt,
|
||||
})
|
||||
}
|
||||
|
||||
responseTasks = append(responseTasks, gin.H{
|
||||
"task_id": t.TaskId,
|
||||
"clients": t.Clients,
|
||||
"command": t.Command,
|
||||
"results": filteredResults,
|
||||
})
|
||||
}
|
||||
api.RespondSuccess(c, responseTasks)
|
||||
}
|
||||
|
||||
func GetTaskById(c *gin.Context) {
|
||||
taskId := c.Param("task_id")
|
||||
if taskId == "" {
|
||||
api.RespondError(c, 400, "Task ID is required")
|
||||
return
|
||||
}
|
||||
task, err := tasks.GetTaskByTaskId(taskId)
|
||||
if err != nil {
|
||||
api.RespondError(c, 500, "Failed to retrieve task: "+err.Error())
|
||||
return
|
||||
}
|
||||
if task == nil {
|
||||
api.RespondError(c, 404, "Task not found")
|
||||
return
|
||||
}
|
||||
results, err := tasks.GetTaskResultsByTaskId(taskId)
|
||||
if err != nil {
|
||||
api.RespondError(c, 500, "Failed to retrieve task results: "+err.Error())
|
||||
return
|
||||
}
|
||||
var filteredResults []gin.H
|
||||
for _, r := range results {
|
||||
filteredResults = append(filteredResults, gin.H{
|
||||
"client": r.Client,
|
||||
"result": r.Result,
|
||||
"exit_code": r.ExitCode,
|
||||
"finished_at": r.FinishedAt,
|
||||
"created_at": r.CreatedAt,
|
||||
})
|
||||
}
|
||||
api.RespondSuccess(c, gin.H{
|
||||
"task_id": task.TaskId,
|
||||
"clients": task.Clients,
|
||||
"command": task.Command,
|
||||
"results": filteredResults,
|
||||
})
|
||||
}
|
||||
|
||||
func GetTasksByClientId(c *gin.Context) {
|
||||
clientId := c.Param("uuid")
|
||||
if clientId == "" {
|
||||
api.RespondError(c, 400, "Client ID is required")
|
||||
return
|
||||
}
|
||||
tasks, err := tasks.GetTasksByClientId(clientId)
|
||||
if err != nil {
|
||||
api.RespondError(c, 500, "Failed to retrieve tasks: "+err.Error())
|
||||
return
|
||||
}
|
||||
if len(tasks) == 0 {
|
||||
api.RespondError(c, 404, "No tasks found for this client")
|
||||
return
|
||||
}
|
||||
api.RespondSuccess(c, tasks)
|
||||
}
|
||||
|
||||
func GetSpecificTaskResult(c *gin.Context) {
|
||||
taskId := c.Param("task_id")
|
||||
clientId := c.Param("uuid")
|
||||
if taskId == "" || clientId == "" {
|
||||
api.RespondError(c, 400, "Task ID and Client ID are required")
|
||||
return
|
||||
}
|
||||
result, err := tasks.GetSpecificTaskResult(taskId, clientId)
|
||||
if err != nil {
|
||||
api.RespondError(c, 500, "Failed to retrieve task result: "+err.Error())
|
||||
return
|
||||
}
|
||||
if result == nil {
|
||||
api.RespondError(c, 404, "No result found for this task and client")
|
||||
return
|
||||
}
|
||||
api.RespondSuccess(c, result)
|
||||
}
|
||||
|
||||
// Param: task_id
|
||||
func GetTaskResultsByTaskId(c *gin.Context) {
|
||||
taskId := c.Param("task_id")
|
||||
if taskId == "" {
|
||||
api.RespondError(c, 400, "Task ID is required")
|
||||
return
|
||||
}
|
||||
results, err := tasks.GetTaskResultsByTaskId(taskId)
|
||||
if err != nil {
|
||||
api.RespondError(c, 500, "Failed to retrieve task results: "+err.Error())
|
||||
return
|
||||
}
|
||||
if len(results) == 0 {
|
||||
api.RespondError(c, 404, "No results found for this task")
|
||||
return
|
||||
}
|
||||
api.RespondSuccess(c, results)
|
||||
}
|
||||
|
||||
func GetAllTaskResultByUUID(c *gin.Context) {
|
||||
clientId := c.Param("uuid")
|
||||
if clientId == "" {
|
||||
api.RespondError(c, 400, "Client ID is required")
|
||||
return
|
||||
}
|
||||
results, err := tasks.GetAllTasksResultByUUID(clientId)
|
||||
if err != nil {
|
||||
api.RespondError(c, 500, "Failed to retrieve tasks: "+err.Error())
|
||||
return
|
||||
}
|
||||
if len(results) == 0 {
|
||||
api.RespondError(c, 404, "No tasks found for this client")
|
||||
return
|
||||
}
|
||||
api.RespondSuccess(c, results)
|
||||
}
|
||||
@@ -1,50 +0,0 @@
|
||||
package test
|
||||
|
||||
import (
|
||||
"net"
|
||||
|
||||
"github.com/gin-gonic/gin"
|
||||
"github.com/komari-monitor/komari/api"
|
||||
"github.com/komari-monitor/komari/database/config"
|
||||
"github.com/komari-monitor/komari/database/models"
|
||||
"github.com/komari-monitor/komari/utils/geoip"
|
||||
"github.com/komari-monitor/komari/utils/messageSender"
|
||||
)
|
||||
|
||||
func TestSendMessage(c *gin.Context) {
|
||||
err := messageSender.SendEvent(models.EventMessage{
|
||||
Event: "Test",
|
||||
Message: "This is a test message from Komari.",
|
||||
})
|
||||
if err != nil {
|
||||
api.RespondError(c, 500, "Failed to send message: "+err.Error())
|
||||
return
|
||||
}
|
||||
api.RespondSuccess(c, nil)
|
||||
}
|
||||
|
||||
func TestGeoIp(c *gin.Context) {
|
||||
ip := c.Query("ip")
|
||||
if ip == "" {
|
||||
if cfIP := c.GetHeader("CF-Connecting-IP"); cfIP != "" {
|
||||
ip = cfIP
|
||||
} else {
|
||||
ip = c.ClientIP()
|
||||
}
|
||||
}
|
||||
conf, err := config.Get()
|
||||
if err != nil {
|
||||
api.RespondError(c, 500, "Failed to get configuration: "+err.Error())
|
||||
return
|
||||
}
|
||||
if !conf.GeoIpEnabled {
|
||||
api.RespondError(c, 400, "GeoIP is not enabled in the configuration.")
|
||||
return
|
||||
}
|
||||
GeoIpRecord, err := geoip.GetGeoInfo(net.ParseIP(ip))
|
||||
if err != nil {
|
||||
api.RespondError(c, 500, "Failed to get GeoIP record: "+err.Error())
|
||||
return
|
||||
}
|
||||
api.RespondSuccess(c, GeoIpRecord)
|
||||
}
|
||||
@@ -1,46 +0,0 @@
|
||||
package update
|
||||
|
||||
import (
|
||||
"io"
|
||||
"net/http"
|
||||
"os"
|
||||
"strings"
|
||||
|
||||
"github.com/gin-gonic/gin"
|
||||
"github.com/komari-monitor/komari/api"
|
||||
"github.com/komari-monitor/komari/database/auditlog"
|
||||
)
|
||||
|
||||
func UploadFavicon(c *gin.Context) {
|
||||
c.Request.Body = http.MaxBytesReader(c.Writer, c.Request.Body, 5<<20) // 5MB
|
||||
data, err := io.ReadAll(c.Request.Body)
|
||||
if err != nil {
|
||||
if strings.Contains(err.Error(), "request body too large") {
|
||||
api.RespondError(c, http.StatusRequestEntityTooLarge, "File too large. Maximum size is 5MB")
|
||||
} else {
|
||||
api.RespondError(c, http.StatusBadRequest, err.Error())
|
||||
}
|
||||
return
|
||||
}
|
||||
if err := os.WriteFile("./data/favicon.ico", data, 0644); err != nil {
|
||||
api.RespondError(c, http.StatusInternalServerError, "Failed to save favicon: "+err.Error())
|
||||
return
|
||||
}
|
||||
uuid, _ := c.Get("uuid")
|
||||
auditlog.Log(c.ClientIP(), uuid.(string), "Favicon uploaded", "info")
|
||||
api.RespondSuccess(c, nil)
|
||||
}
|
||||
|
||||
func DeleteFavicon(c *gin.Context) {
|
||||
if err := os.Remove("./data/favicon.ico"); err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
api.RespondError(c, http.StatusNotFound, "Favicon not found")
|
||||
} else {
|
||||
api.RespondError(c, http.StatusInternalServerError, "Failed to delete favicon: "+err.Error())
|
||||
}
|
||||
return
|
||||
}
|
||||
uuid, _ := c.Get("uuid")
|
||||
auditlog.Log(c.ClientIP(), uuid.(string), "Favicon deleted", "info")
|
||||
api.RespondSuccess(c, nil)
|
||||
}
|
||||
@@ -1,18 +0,0 @@
|
||||
package update
|
||||
|
||||
import (
|
||||
"github.com/gin-gonic/gin"
|
||||
"github.com/komari-monitor/komari/api"
|
||||
"github.com/komari-monitor/komari/database/auditlog"
|
||||
"github.com/komari-monitor/komari/utils/geoip"
|
||||
)
|
||||
|
||||
func UpdateMmdbGeoIP(c *gin.Context) {
|
||||
if err := geoip.UpdateDatabase(); err != nil {
|
||||
api.RespondError(c, 500, "Failed to update GeoIP database "+err.Error())
|
||||
return
|
||||
}
|
||||
uuid, _ := c.Get("uuid")
|
||||
auditlog.Log(c.ClientIP(), uuid.(string), "GeoIP database updated", "info")
|
||||
api.RespondSuccess(c, nil)
|
||||
}
|
||||
@@ -1,40 +0,0 @@
|
||||
package update
|
||||
|
||||
import (
|
||||
"github.com/gin-gonic/gin"
|
||||
"github.com/komari-monitor/komari/api"
|
||||
"github.com/komari-monitor/komari/database/accounts"
|
||||
"github.com/komari-monitor/komari/database/auditlog"
|
||||
)
|
||||
|
||||
func UpdateUser(c *gin.Context) {
|
||||
var req struct {
|
||||
Uuid string `json:"uuid" binding:"required"`
|
||||
Name *string `json:"username"`
|
||||
Password *string `json:"password"`
|
||||
SsoType *string `json:"sso_type"`
|
||||
}
|
||||
if err := c.ShouldBindJSON(&req); err != nil {
|
||||
api.RespondError(c, 400, "Invalid or missing request body: "+err.Error())
|
||||
return
|
||||
}
|
||||
if req.Password == nil && req.Name == nil {
|
||||
api.RespondError(c, 400, "At least one field (username or password) must be provided")
|
||||
return
|
||||
}
|
||||
if req.Name != nil && len(*req.Name) < 3 {
|
||||
api.RespondError(c, 400, "Username must be at least 3 characters long")
|
||||
return
|
||||
}
|
||||
if req.Password != nil && len(*req.Password) < 6 {
|
||||
api.RespondError(c, 400, "Password must be at least 6 characters long")
|
||||
return
|
||||
}
|
||||
if err := accounts.UpdateUser(req.Uuid, req.Name, req.Password, req.SsoType); err != nil {
|
||||
api.RespondError(c, 500, "Failed to update user: "+err.Error())
|
||||
return
|
||||
}
|
||||
uuid, _ := c.Get("uuid")
|
||||
auditlog.Log(c.ClientIP(), uuid.(string), "User updated", "warn")
|
||||
api.RespondSuccess(c, gin.H{"uuid": req.Uuid})
|
||||
}
|
||||
@@ -1,173 +0,0 @@
|
||||
package admin
|
||||
|
||||
import (
|
||||
"archive/zip"
|
||||
"fmt"
|
||||
"io"
|
||||
"log"
|
||||
"net/http"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/gin-gonic/gin"
|
||||
"github.com/komari-monitor/komari/api"
|
||||
"github.com/komari-monitor/komari/cmd/flags"
|
||||
)
|
||||
|
||||
// 只有一个备份恢复操作在进行
|
||||
var restoreMutex sync.Mutex
|
||||
|
||||
// UploadBackup 用于接收上传的备份文件并将其内容恢复到原始位置
|
||||
func UploadBackup(c *gin.Context) {
|
||||
// 尝试获取锁,如果已有恢复操作在进行,则立即返回错误
|
||||
if !restoreMutex.TryLock() {
|
||||
api.RespondError(c, http.StatusConflict, "Another restore operation is already in progress")
|
||||
return
|
||||
}
|
||||
defer restoreMutex.Unlock()
|
||||
|
||||
// 获取上传的文件
|
||||
file, header, err := c.Request.FormFile("backup")
|
||||
if err != nil {
|
||||
api.RespondError(c, http.StatusBadRequest, fmt.Sprintf("Error getting uploaded file: %v", err))
|
||||
return
|
||||
}
|
||||
defer file.Close()
|
||||
|
||||
// 检查文件是否为zip格式
|
||||
if !strings.HasSuffix(strings.ToLower(header.Filename), ".zip") {
|
||||
api.RespondError(c, http.StatusBadRequest, "Uploaded file must be a ZIP archive")
|
||||
return
|
||||
}
|
||||
|
||||
// 创建临时文件保存上传的zip
|
||||
tempFile, err := os.CreateTemp("", "backup-*.zip")
|
||||
if err != nil {
|
||||
api.RespondError(c, http.StatusInternalServerError, fmt.Sprintf("Error creating temporary file: %v", err))
|
||||
return
|
||||
}
|
||||
tempFilePath := tempFile.Name()
|
||||
defer os.Remove(tempFilePath) // 确保临时文件最终被删除
|
||||
|
||||
// 将上传的文件内容复制到临时文件
|
||||
_, err = io.Copy(tempFile, file)
|
||||
if err != nil {
|
||||
tempFile.Close()
|
||||
api.RespondError(c, http.StatusInternalServerError, fmt.Sprintf("Error saving uploaded file: %v", err))
|
||||
return
|
||||
}
|
||||
tempFile.Close() // 关闭文件以便后续操作
|
||||
|
||||
// 打开zip文件准备解压
|
||||
zipReader, err := zip.OpenReader(tempFilePath)
|
||||
if err != nil {
|
||||
api.RespondError(c, http.StatusInternalServerError, fmt.Sprintf("Error opening zip file: %v", err))
|
||||
return
|
||||
}
|
||||
defer zipReader.Close()
|
||||
|
||||
// 检查是否包含备份标记文件
|
||||
hasMarkupFile := false
|
||||
for _, zipFile := range zipReader.File {
|
||||
if zipFile.Name == "komari-backup-markup" {
|
||||
hasMarkupFile = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if !hasMarkupFile {
|
||||
api.RespondError(c, http.StatusBadRequest, "Invalid backup file: missing komari-backup-markup file")
|
||||
return
|
||||
}
|
||||
|
||||
// 确保data目录存在
|
||||
if err := os.MkdirAll("./data", 0755); err != nil {
|
||||
api.RespondError(c, http.StatusInternalServerError, fmt.Sprintf("Error creating data directory: %v", err))
|
||||
return
|
||||
}
|
||||
|
||||
// 获取数据库文件名
|
||||
dbFileName := filepath.Base(flags.DatabaseFile)
|
||||
|
||||
// 解压文件
|
||||
for _, zipFile := range zipReader.File {
|
||||
// 检查文件路径是否安全(防止路径遍历攻击)
|
||||
filePath := zipFile.Name
|
||||
if strings.Contains(filePath, "..") {
|
||||
log.Printf("Potentially unsafe path in zip: %s, skipping", filePath)
|
||||
continue
|
||||
}
|
||||
|
||||
// 跳过备份标记文件
|
||||
if filePath == "komari-backup-markup" {
|
||||
continue
|
||||
}
|
||||
|
||||
// 确定目标路径
|
||||
var destPath string
|
||||
if filePath == dbFileName {
|
||||
// 如果是数据库文件,恢复到数据库文件路径
|
||||
destPath = flags.DatabaseFile
|
||||
} else {
|
||||
// 其他文件恢复到data目录
|
||||
destPath = filepath.Join("./data", filePath)
|
||||
}
|
||||
|
||||
// 如果是目录,创建目录
|
||||
if zipFile.FileInfo().IsDir() {
|
||||
if err := os.MkdirAll(destPath, 0755); err != nil {
|
||||
log.Printf("Error creating directory %s: %v", destPath, err)
|
||||
}
|
||||
continue
|
||||
}
|
||||
|
||||
// 确保目标文件的目录存在
|
||||
destDir := filepath.Dir(destPath)
|
||||
if err := os.MkdirAll(destDir, 0755); err != nil {
|
||||
log.Printf("Error creating directory %s: %v", destDir, err)
|
||||
continue
|
||||
}
|
||||
|
||||
// 打开zip中的文件
|
||||
srcFile, err := zipFile.Open()
|
||||
if err != nil {
|
||||
log.Printf("Error opening file from zip %s: %v", filePath, err)
|
||||
continue
|
||||
}
|
||||
|
||||
// 创建目标文件
|
||||
destFile, err := os.Create(destPath)
|
||||
if err != nil {
|
||||
srcFile.Close()
|
||||
log.Printf("Error creating file %s: %v", destPath, err)
|
||||
continue
|
||||
}
|
||||
|
||||
// 复制内容
|
||||
_, err = io.Copy(destFile, srcFile)
|
||||
srcFile.Close()
|
||||
destFile.Close()
|
||||
if err != nil {
|
||||
log.Printf("Error extracting file %s: %v", destPath, err)
|
||||
continue
|
||||
}
|
||||
|
||||
// 保持原始文件的修改时间
|
||||
if err := os.Chtimes(destPath, zipFile.Modified, zipFile.Modified); err != nil {
|
||||
log.Printf("Error setting file time for %s: %v", destPath, err)
|
||||
}
|
||||
}
|
||||
|
||||
c.JSON(http.StatusOK, gin.H{
|
||||
"status": "success",
|
||||
"message": "Backup restored successfully. The service will restart shortly.",
|
||||
})
|
||||
|
||||
go func() {
|
||||
log.Println("Backup restored, restarting service in 2 seconds...")
|
||||
time.Sleep(2 * time.Second)
|
||||
os.Exit(0)
|
||||
}()
|
||||
}
|
||||
@@ -1,36 +0,0 @@
|
||||
package client
|
||||
|
||||
import (
|
||||
"time"
|
||||
|
||||
"github.com/gin-gonic/gin"
|
||||
"github.com/komari-monitor/komari/database/clients"
|
||||
"github.com/komari-monitor/komari/database/models"
|
||||
"github.com/komari-monitor/komari/database/tasks"
|
||||
)
|
||||
|
||||
func TaskResult(c *gin.Context) {
|
||||
token := c.Query("token")
|
||||
clientId, _ := clients.GetClientUUIDByToken(token)
|
||||
if clientId == "" {
|
||||
c.JSON(400, gin.H{"status": "error", "message": "Invalid or missing token"})
|
||||
return
|
||||
}
|
||||
var req struct {
|
||||
TaskId string `json:"task_id" binding:"required"`
|
||||
Result string `json:"result" binding:"required"`
|
||||
ExitCode int `json:"exit_code"`
|
||||
FinishedAt time.Time `json:"finished_at" binding:"required"`
|
||||
}
|
||||
if err := c.ShouldBindJSON(&req); err != nil {
|
||||
c.JSON(400, gin.H{"status": "error", "message": "Invalid request"})
|
||||
return
|
||||
}
|
||||
|
||||
if err := tasks.SaveTaskResult(req.TaskId, clientId, req.Result, req.ExitCode, models.FromTime(req.FinishedAt)); err != nil {
|
||||
c.JSON(500, gin.H{"status": "error", "message": "Failed to update task result: " + err.Error()})
|
||||
return
|
||||
}
|
||||
|
||||
c.JSON(200, gin.H{"status": "success", "message": "Task result updated successfully"})
|
||||
}
|
||||
@@ -1,85 +0,0 @@
|
||||
package client
|
||||
|
||||
import (
|
||||
"net"
|
||||
|
||||
"github.com/komari-monitor/komari/database/clients"
|
||||
"github.com/komari-monitor/komari/database/config"
|
||||
"github.com/komari-monitor/komari/utils/geoip"
|
||||
|
||||
"github.com/gin-gonic/gin"
|
||||
)
|
||||
|
||||
func getClientIPType(ip net.IP) int {
|
||||
// 0:ipv4 1:ipv6 -1:错误的输入
|
||||
if ip == nil {
|
||||
return -1
|
||||
}
|
||||
if ip.To4() == nil {
|
||||
return 1
|
||||
} else {
|
||||
return 0
|
||||
}
|
||||
}
|
||||
|
||||
func UploadBasicInfo(c *gin.Context) {
|
||||
var cbi = map[string]interface{}{}
|
||||
if err := c.ShouldBindJSON(&cbi); err != nil {
|
||||
c.JSON(400, gin.H{"status": "error", "error": "Invalid or missing data"})
|
||||
return
|
||||
}
|
||||
|
||||
token := c.Query("token")
|
||||
uuid, err := clients.GetClientUUIDByToken(token)
|
||||
if uuid == "" || err != nil {
|
||||
c.JSON(400, gin.H{"status": "error", "error": "Invalid token"})
|
||||
return
|
||||
}
|
||||
|
||||
cbi["uuid"] = uuid
|
||||
|
||||
if (func() bool {
|
||||
if v4, ok := cbi["ipv4"].(string); !ok || v4 == "" {
|
||||
if v6, ok := cbi["ipv6"].(string); !ok || v6 == "" {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
})() {
|
||||
ipStr := c.ClientIP()
|
||||
ip := net.ParseIP(ipStr)
|
||||
ipType := getClientIPType(ip)
|
||||
|
||||
switch ipType {
|
||||
case 0:
|
||||
cbi["ipv4"] = ip
|
||||
case 1:
|
||||
cbi["ipv6"] = ip
|
||||
default:
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
if cfg, err := config.Get(); err == nil && cfg.GeoIpEnabled {
|
||||
if ipv4, ok := cbi["ipv4"].(string); ok && ipv4 != "" {
|
||||
ip4 := net.ParseIP(ipv4)
|
||||
ip4_record, _ := geoip.GetGeoInfo(ip4)
|
||||
if ip4_record != nil {
|
||||
cbi["region"] = geoip.GetRegionUnicodeEmoji(ip4_record.ISOCode)
|
||||
}
|
||||
} else if ipv6, ok := cbi["ipv6"].(string); ok && ipv6 != "" {
|
||||
ip6 := net.ParseIP(ipv6)
|
||||
ip6_record, _ := geoip.GetGeoInfo(ip6)
|
||||
if ip6_record != nil {
|
||||
cbi["region"] = geoip.GetRegionUnicodeEmoji(ip6_record.ISOCode)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if err := clients.SaveClientInfo(cbi); err != nil {
|
||||
c.JSON(500, gin.H{"status": "error", "error": err})
|
||||
return
|
||||
}
|
||||
|
||||
c.JSON(200, gin.H{"status": "success"})
|
||||
}
|
||||
@@ -1,227 +0,0 @@
|
||||
package jsonRpc
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"io"
|
||||
"net/http"
|
||||
"reflect"
|
||||
"strings"
|
||||
"sync"
|
||||
|
||||
"github.com/gin-gonic/gin"
|
||||
"github.com/komari-monitor/komari/database/accounts"
|
||||
"github.com/komari-monitor/komari/database/clients"
|
||||
"github.com/komari-monitor/komari/database/config"
|
||||
"github.com/komari-monitor/komari/database/models"
|
||||
"github.com/komari-monitor/komari/utils/rpc"
|
||||
"github.com/komari-monitor/komari/ws"
|
||||
)
|
||||
|
||||
// Json Rpc2 over websocket, /api/rpc2
|
||||
func OnRpcRequest(c *gin.Context) {
|
||||
cfg, _ := config.Get()
|
||||
|
||||
// GET -> WebSocket
|
||||
if c.Request.Method == http.MethodGet {
|
||||
_conn, err := ws.UpgradeRequest(c, func(r *http.Request) bool {
|
||||
if cfg.AllowCors {
|
||||
return true
|
||||
}
|
||||
return ws.CheckOrigin(r)
|
||||
})
|
||||
conn := ws.NewSafeConn(_conn)
|
||||
if err != nil {
|
||||
c.JSON(http.StatusBadRequest, gin.H{"status": "error", "error": "Failed to upgrade to WebSocket"})
|
||||
return
|
||||
}
|
||||
permissionGroup := detectPermissionGroup(c, cfg)
|
||||
meta := buildContextMeta(c, permissionGroup)
|
||||
defer conn.Close()
|
||||
for {
|
||||
var req rpc.JsonRpcRequest
|
||||
err := conn.ReadJSON(&req)
|
||||
if err != nil {
|
||||
// JSON 解析
|
||||
var se *json.SyntaxError
|
||||
var ute *json.UnmarshalTypeError
|
||||
if errors.As(err, &se) || errors.As(err, &ute) {
|
||||
conn.WriteJSON(rpc.ErrorResponse(nil, rpc.InvalidRequest, "bad request: "+err.Error(), nil))
|
||||
continue
|
||||
}
|
||||
// 其它视为连接/IO 错误
|
||||
break
|
||||
}
|
||||
if jerr := req.Validate(); jerr != nil {
|
||||
conn.WriteJSON(jerr.ResponseWithID(req.ID))
|
||||
continue
|
||||
}
|
||||
dispatchByPermissionWithMeta(conn, permissionGroup, meta, &req)
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
// 否则按 HTTP POST JSON-RPC 处理 (支持单个或批量)
|
||||
if c.Request.Method != http.MethodPost {
|
||||
c.JSON(http.StatusMethodNotAllowed, gin.H{"error": "method not allowed"})
|
||||
return
|
||||
}
|
||||
body, err := io.ReadAll(c.Request.Body)
|
||||
if err != nil {
|
||||
c.JSON(http.StatusBadRequest, rpc.ErrorResponse(nil, rpc.ParseError, "read body error", err.Error()))
|
||||
return
|
||||
}
|
||||
requests, jerr := rpc.ParseRequests(body)
|
||||
if jerr != nil {
|
||||
c.JSON(http.StatusBadRequest, jerr.Response())
|
||||
return
|
||||
}
|
||||
permissionGroup := detectPermissionGroup(c, cfg)
|
||||
meta := buildContextMeta(c, permissionGroup)
|
||||
// 批量
|
||||
responses := make([]*rpc.JsonRpcResponse, 0, len(requests))
|
||||
for _, rreq := range requests {
|
||||
// 权限与命名空间
|
||||
fc := strings.Split(rreq.Method, ":")
|
||||
if len(fc) == 1 {
|
||||
fc[0] = "common"
|
||||
}
|
||||
allowed := false
|
||||
switch fc[0] {
|
||||
case "guest", "", "rpc", "common":
|
||||
allowed = true
|
||||
case "client":
|
||||
if permissionGroup == "client" || permissionGroup == "admin" {
|
||||
allowed = true
|
||||
}
|
||||
case "admin":
|
||||
if permissionGroup == "admin" {
|
||||
allowed = true
|
||||
}
|
||||
default:
|
||||
responses = append(responses, rpc.ErrorResponse(rreq.ID, 401, "Unauthorized", nil))
|
||||
continue
|
||||
}
|
||||
if !allowed {
|
||||
responses = append(responses, rpc.ErrorResponse(rreq.ID, 401, "Unauthorized", nil))
|
||||
continue
|
||||
}
|
||||
responses = append(responses, rpc.CallWithContext(rpc.NewContextWithMeta(context.TODO(), meta), rreq.ID, rreq.Method, rreq.Params))
|
||||
}
|
||||
// 单个请求直接对象,批量请求数组 (符合 JSON-RPC 2.0)
|
||||
if len(responses) == 1 {
|
||||
c.JSON(http.StatusOK, responses[0])
|
||||
} else {
|
||||
c.JSON(http.StatusOK, responses)
|
||||
}
|
||||
}
|
||||
|
||||
// detectPermissionGroup 提取权限分组,与原逻辑保持一致
|
||||
func detectPermissionGroup(c *gin.Context, cfg models.Config) string {
|
||||
permissionGroup := "guest"
|
||||
token := c.Query("Authorization")
|
||||
if _, err := clients.GetClientUUIDByToken(token); err == nil {
|
||||
permissionGroup = "client"
|
||||
}
|
||||
if session_token, _ := c.Cookie("session_token"); session_token != "" {
|
||||
if _, err := accounts.GetUserBySession(session_token); err == nil {
|
||||
permissionGroup = "admin"
|
||||
}
|
||||
}
|
||||
apiKey := c.GetHeader("Authorization")
|
||||
if apiKey == "Bearer "+cfg.ApiKey {
|
||||
permissionGroup = "admin"
|
||||
}
|
||||
return permissionGroup
|
||||
}
|
||||
|
||||
// buildContextMeta 从 gin.Context 构建 *rpc.ContextMeta
|
||||
func buildContextMeta(c *gin.Context, permissionGroup string) *rpc.ContextMeta {
|
||||
meta := &rpc.ContextMeta{Permission: permissionGroup}
|
||||
// 提取客户端 token (query Authorization / Header Authorization Bearer token)
|
||||
token := c.Query("Authorization")
|
||||
if token == "" {
|
||||
// 兼容 header Bearer <token>
|
||||
hAuth := c.GetHeader("Authorization")
|
||||
if strings.HasPrefix(hAuth, "Bearer ") {
|
||||
token = strings.TrimPrefix(hAuth, "Bearer ")
|
||||
}
|
||||
}
|
||||
if token != "" {
|
||||
if uuid, err := clients.GetClientUUIDByToken(token); err == nil {
|
||||
meta.ClientToken = token
|
||||
meta.ClientUUID = uuid
|
||||
}
|
||||
}
|
||||
// 提取用户 (session cookie)
|
||||
if session_token, _ := c.Cookie("session_token"); session_token != "" {
|
||||
if user, err := accounts.GetUserBySession(session_token); err == nil {
|
||||
meta.User = &user
|
||||
meta.UserUUID = user.UUID
|
||||
}
|
||||
}
|
||||
meta.RemoteIP = c.ClientIP()
|
||||
meta.UserAgent = c.GetHeader("User-Agent")
|
||||
return meta
|
||||
}
|
||||
|
||||
// dispatchByPermissionWithMeta 与原函数类似,但会携带 meta 上下文给 handler
|
||||
func dispatchByPermissionWithMeta(conn *ws.SafeConn, permissionGroup string, meta *rpc.ContextMeta, req *rpc.JsonRpcRequest) {
|
||||
fc := strings.Split(req.Method, ":")
|
||||
if len(fc) == 1 {
|
||||
fc[0] = "common"
|
||||
}
|
||||
ctx := rpc.NewContextWithMeta(context.TODO(), meta)
|
||||
switch fc[0] {
|
||||
case "guest", "", "rpc", "common":
|
||||
go conn.WriteJSON(rpc.CallWithContext(ctx, req.ID, req.Method, req.Params))
|
||||
case "client":
|
||||
if permissionGroup == "client" || permissionGroup == "admin" {
|
||||
go conn.WriteJSON(rpc.CallWithContext(ctx, req.ID, req.Method, req.Params))
|
||||
} else {
|
||||
conn.WriteJSON(rpc.ErrorResponse(req.ID, 401, "Unauthorized", nil))
|
||||
}
|
||||
case "admin":
|
||||
if permissionGroup == "admin" {
|
||||
go conn.WriteJSON(rpc.CallWithContext(ctx, req.ID, req.Method, req.Params))
|
||||
} else {
|
||||
conn.WriteJSON(rpc.ErrorResponse(req.ID, 401, "Unauthorized", nil))
|
||||
}
|
||||
default:
|
||||
conn.WriteJSON(rpc.ErrorResponse(req.ID, 401, "Unauthorized", nil))
|
||||
}
|
||||
}
|
||||
|
||||
// registry holds method handlers keyed by "namespace:MethodName"
|
||||
var (
|
||||
registryMu sync.RWMutex
|
||||
registry = make(map[string]reflect.Value)
|
||||
)
|
||||
|
||||
// Register 将以分组common注册
|
||||
func Register(name string, cb rpc.Handler) error {
|
||||
return RegisterWithGroupAndMeta(name, "common", cb, &rpc.MethodMeta{
|
||||
Name: name,
|
||||
Summary: "This method does not provide a summary",
|
||||
Description: "This method does not provide a description",
|
||||
})
|
||||
}
|
||||
|
||||
// RegisterWithGroup 将回调按分组注册。
|
||||
// 传入函数: Register("ns", someFunc) -> 注册 ns:SomeFunc
|
||||
// group 为空时使用默认分组 "common"。
|
||||
func RegisterWithGroupAndMeta(name, group string, cb rpc.Handler, meta *rpc.MethodMeta) error {
|
||||
return rpc.RegisterWithMeta(group+":"+name, cb, meta)
|
||||
}
|
||||
|
||||
// GetRegisteredKeys 返回已注册的方法 key 列表 (调试用)。
|
||||
func GetRegisteredKeys() []string {
|
||||
registryMu.RLock()
|
||||
defer registryMu.RUnlock()
|
||||
keys := make([]string, 0, len(registry))
|
||||
for k := range registry {
|
||||
keys = append(keys, k)
|
||||
}
|
||||
return keys
|
||||
}
|
||||
@@ -1,44 +0,0 @@
|
||||
package api
|
||||
|
||||
import (
|
||||
"github.com/gin-gonic/gin"
|
||||
"github.com/komari-monitor/komari/database/accounts"
|
||||
"github.com/komari-monitor/komari/database/dbcore"
|
||||
"github.com/komari-monitor/komari/database/models"
|
||||
)
|
||||
|
||||
func GetClientRecentRecords(c *gin.Context) {
|
||||
uuid := c.Param("uuid")
|
||||
|
||||
if uuid == "" {
|
||||
RespondError(c, 400, "UUID is required")
|
||||
return
|
||||
}
|
||||
|
||||
// 登录状态检查
|
||||
isLogin := false
|
||||
session, _ := c.Cookie("session_token")
|
||||
_, err := accounts.GetUserBySession(session)
|
||||
if err == nil {
|
||||
isLogin = true
|
||||
}
|
||||
|
||||
// 仅在未登录时需要 Hidden 信息做过滤
|
||||
hiddenMap := map[string]bool{}
|
||||
if !isLogin {
|
||||
var hiddenClients []models.Client
|
||||
db := dbcore.GetDBInstance()
|
||||
_ = db.Select("uuid").Where("hidden = ?", true).Find(&hiddenClients).Error
|
||||
for _, cli := range hiddenClients {
|
||||
hiddenMap[cli.UUID] = true
|
||||
}
|
||||
|
||||
if hiddenMap[uuid] {
|
||||
RespondError(c, 400, "UUID is required") //防止未登录用户获取隐藏客户端数据
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
records, _ := Records.Get(uuid)
|
||||
RespondSuccess(c, records)
|
||||
}
|
||||
@@ -1,77 +0,0 @@
|
||||
package api
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"io"
|
||||
"net/http"
|
||||
|
||||
"github.com/komari-monitor/komari/database/accounts"
|
||||
"github.com/komari-monitor/komari/database/auditlog"
|
||||
"github.com/komari-monitor/komari/database/config"
|
||||
|
||||
"github.com/gin-gonic/gin"
|
||||
)
|
||||
|
||||
type LoginRequest struct {
|
||||
Username string `json:"username"`
|
||||
Password string `json:"password"`
|
||||
TwoFa string `json:"2fa_code"`
|
||||
}
|
||||
|
||||
func Login(c *gin.Context) {
|
||||
conf, _ := config.Get()
|
||||
if conf.DisablePasswordLogin {
|
||||
RespondError(c, http.StatusForbidden, "Password login is disabled")
|
||||
return
|
||||
}
|
||||
|
||||
bodyBytes, err := io.ReadAll(c.Request.Body)
|
||||
if err != nil {
|
||||
RespondError(c, http.StatusBadRequest, "Invalid request body: "+err.Error())
|
||||
return
|
||||
}
|
||||
var data LoginRequest
|
||||
err = json.Unmarshal(bodyBytes, &data)
|
||||
if err != nil {
|
||||
RespondError(c, http.StatusBadRequest, "Invalid request body: "+err.Error())
|
||||
return
|
||||
}
|
||||
if data.Username == "" || data.Password == "" {
|
||||
RespondError(c, http.StatusBadRequest, "Invalid request body: Username and password are required")
|
||||
return
|
||||
}
|
||||
|
||||
uuid, success := accounts.CheckPassword(data.Username, data.Password)
|
||||
if !success {
|
||||
RespondError(c, http.StatusUnauthorized, "Invalid credentials")
|
||||
return
|
||||
}
|
||||
// 2FA
|
||||
user, _ := accounts.GetUserByUUID(uuid)
|
||||
if user.TwoFactor != "" { // 开启了2FA
|
||||
if data.TwoFa == "" {
|
||||
RespondError(c, http.StatusUnauthorized, "2FA code is required")
|
||||
return
|
||||
}
|
||||
if ok, err := accounts.Verify2Fa(uuid, data.TwoFa); err != nil || !ok {
|
||||
RespondError(c, http.StatusUnauthorized, "Invalid 2FA code")
|
||||
return
|
||||
}
|
||||
}
|
||||
// Create session
|
||||
session, err := accounts.CreateSession(uuid, 2592000, c.Request.UserAgent(), c.ClientIP(), "password")
|
||||
if err != nil {
|
||||
RespondError(c, http.StatusInternalServerError, "Failed to create session: "+err.Error())
|
||||
return
|
||||
}
|
||||
c.SetCookie("session_token", session, 2592000, "/", "", false, true)
|
||||
auditlog.Log(c.ClientIP(), uuid, "logged in (password)", "login")
|
||||
RespondSuccess(c, gin.H{"set-cookie": gin.H{"session_token": session}})
|
||||
}
|
||||
func Logout(c *gin.Context) {
|
||||
session, _ := c.Cookie("session_token")
|
||||
accounts.DeleteSession(session)
|
||||
c.SetCookie("session_token", "", -1, "/", "", false, true)
|
||||
auditlog.Log(c.ClientIP(), "", "logged out", "logout")
|
||||
c.Redirect(302, "/")
|
||||
}
|
||||
@@ -1,27 +0,0 @@
|
||||
package api
|
||||
|
||||
import (
|
||||
"github.com/komari-monitor/komari/database/accounts"
|
||||
|
||||
"github.com/gin-gonic/gin"
|
||||
)
|
||||
|
||||
func GetMe(c *gin.Context) {
|
||||
session, err := c.Cookie("session_token")
|
||||
if err != nil {
|
||||
c.JSON(200, gin.H{"username": "Guest", "logged_in": false})
|
||||
return
|
||||
}
|
||||
uuid, err := accounts.GetSession(session)
|
||||
if err != nil {
|
||||
c.JSON(200, gin.H{"username": "Guest", "logged_in": false})
|
||||
return
|
||||
}
|
||||
user, err := accounts.GetUserByUUID(uuid)
|
||||
if err != nil {
|
||||
c.JSON(200, gin.H{"username": "Guest", "logged_in": false})
|
||||
return
|
||||
}
|
||||
c.JSON(200, gin.H{"username": user.Username, "logged_in": true, "uuid": user.UUID, "sso_type": user.SSOType, "sso_id": user.SSOID, "2fa_enabled": user.TwoFactor != ""})
|
||||
|
||||
}
|
||||
@@ -1,95 +0,0 @@
|
||||
package api
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
|
||||
"github.com/gin-gonic/gin"
|
||||
"github.com/komari-monitor/komari/database/accounts"
|
||||
"github.com/stretchr/testify/assert"
|
||||
)
|
||||
|
||||
func TestGetMe(t *testing.T) {
|
||||
// 设置测试模式
|
||||
gin.SetMode(gin.TestMode)
|
||||
|
||||
// 创建测试用户
|
||||
user, err := accounts.CreateAccount("testuser", "password")
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to create test user: %v", err)
|
||||
}
|
||||
uuid := user.UUID
|
||||
sessionToken, _ := accounts.CreateSession(uuid, 2592000, "test_user_agent", "127.0.0.1", "oauth")
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
sessionToken string
|
||||
expectedStatus int
|
||||
expectedBody map[string]interface{}
|
||||
}{
|
||||
{
|
||||
name: "有效的会话",
|
||||
sessionToken: sessionToken,
|
||||
expectedStatus: http.StatusOK,
|
||||
expectedBody: map[string]interface{}{
|
||||
"username": "testuser",
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "无效的会话",
|
||||
sessionToken: "invalid_token",
|
||||
expectedStatus: http.StatusOK,
|
||||
expectedBody: map[string]interface{}{
|
||||
"username": "Guest",
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "无会话",
|
||||
sessionToken: "",
|
||||
expectedStatus: http.StatusOK,
|
||||
expectedBody: map[string]interface{}{
|
||||
"username": "Guest",
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
// 创建测试路由
|
||||
router := gin.New()
|
||||
router.GET("/me", GetMe)
|
||||
|
||||
// 创建测试请求
|
||||
req, _ := http.NewRequest("GET", "/me", nil)
|
||||
if tt.sessionToken != "" {
|
||||
req.AddCookie(&http.Cookie{
|
||||
Name: "session_token",
|
||||
Value: tt.sessionToken,
|
||||
})
|
||||
}
|
||||
|
||||
// 创建响应记录器
|
||||
w := httptest.NewRecorder()
|
||||
|
||||
// 执行请求
|
||||
router.ServeHTTP(w, req)
|
||||
|
||||
// 断言状态码
|
||||
assert.Equal(t, tt.expectedStatus, w.Code)
|
||||
|
||||
// 解析响应体
|
||||
var response map[string]interface{}
|
||||
err := json.Unmarshal(w.Body.Bytes(), &response)
|
||||
assert.NoError(t, err)
|
||||
|
||||
// 断言响应体
|
||||
assert.Equal(t, tt.expectedBody, response)
|
||||
})
|
||||
}
|
||||
|
||||
// 清除测试数据
|
||||
accounts.DeleteAccountByUsername("testuser")
|
||||
accounts.DeleteAllSessions()
|
||||
}
|
||||
@@ -1,39 +0,0 @@
|
||||
package api
|
||||
|
||||
import (
|
||||
"github.com/gin-gonic/gin"
|
||||
"github.com/komari-monitor/komari/database/accounts"
|
||||
"github.com/komari-monitor/komari/database/clients"
|
||||
)
|
||||
|
||||
func GetNodesInformation(c *gin.Context) {
|
||||
clientList, err := clients.GetAllClientBasicInfo()
|
||||
if err != nil {
|
||||
RespondError(c, 500, "Failed to retrieve client information: "+err.Error())
|
||||
return
|
||||
}
|
||||
isLogin := false
|
||||
session, _ := c.Cookie("session_token")
|
||||
_, err = accounts.GetUserBySession(session)
|
||||
if err == nil {
|
||||
isLogin = true
|
||||
}
|
||||
|
||||
// 过滤掉 Hidden 的客户端,并清理需要隐藏的字段
|
||||
j := 0
|
||||
for i := 0; i < len(clientList); i++ {
|
||||
if clientList[i].Hidden && !isLogin { // 不返回 Hidden 客户端
|
||||
continue
|
||||
}
|
||||
clientList[i].IPv4 = ""
|
||||
clientList[i].IPv6 = ""
|
||||
clientList[i].Remark = "" // 私有备注不展示
|
||||
clientList[i].Version = ""
|
||||
clientList[i].Token = ""
|
||||
clientList[j] = clientList[i]
|
||||
j++
|
||||
}
|
||||
clientList = clientList[:j]
|
||||
|
||||
RespondSuccess(c, clientList)
|
||||
}
|
||||
@@ -1,15 +0,0 @@
|
||||
package api
|
||||
|
||||
import (
|
||||
"github.com/gin-gonic/gin"
|
||||
"github.com/komari-monitor/komari/database"
|
||||
)
|
||||
|
||||
func GetPublicSettings(c *gin.Context) {
|
||||
p, e := database.GetPublicInfo()
|
||||
if e != nil {
|
||||
RespondError(c, 500, e.Error())
|
||||
return
|
||||
}
|
||||
RespondSuccess(c, p)
|
||||
}
|
||||
@@ -1,447 +0,0 @@
|
||||
package record
|
||||
|
||||
import (
|
||||
"slices"
|
||||
"strconv"
|
||||
"time"
|
||||
|
||||
"github.com/gin-gonic/gin"
|
||||
"github.com/komari-monitor/komari/api"
|
||||
"github.com/komari-monitor/komari/database/accounts"
|
||||
"github.com/komari-monitor/komari/database/dbcore"
|
||||
"github.com/komari-monitor/komari/database/models"
|
||||
records "github.com/komari-monitor/komari/database/records"
|
||||
"github.com/komari-monitor/komari/database/tasks"
|
||||
)
|
||||
|
||||
func GetRecordsByUUID(c *gin.Context) {
|
||||
uuid := c.Query("uuid")
|
||||
loadType := c.Query("load_type")
|
||||
|
||||
// 登录状态检查
|
||||
isLogin := false
|
||||
session, _ := c.Cookie("session_token")
|
||||
_, err := accounts.GetUserBySession(session)
|
||||
if err == nil {
|
||||
isLogin = true
|
||||
}
|
||||
|
||||
// 仅在未登录时需要 Hidden 信息做过滤
|
||||
hiddenMap := map[string]bool{}
|
||||
if !isLogin {
|
||||
var hiddenClients []models.Client
|
||||
db := dbcore.GetDBInstance()
|
||||
_ = db.Select("uuid").Where("hidden = ?", true).Find(&hiddenClients).Error
|
||||
for _, cli := range hiddenClients {
|
||||
hiddenMap[cli.UUID] = true
|
||||
}
|
||||
|
||||
if hiddenMap[uuid] {
|
||||
api.RespondError(c, 400, "UUID is required") //防止未登录用户获取隐藏客户端数据
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
hours := c.Query("hours")
|
||||
if uuid == "" {
|
||||
api.RespondError(c, 400, "UUID is required")
|
||||
return
|
||||
}
|
||||
if hours == "" {
|
||||
hours = "4"
|
||||
}
|
||||
|
||||
hoursInt, err := strconv.Atoi(hours)
|
||||
if err != nil {
|
||||
api.RespondError(c, 400, "Invalid hours parameter")
|
||||
return
|
||||
}
|
||||
|
||||
// 验证 load_type 参数
|
||||
validLoadTypes := map[string]bool{
|
||||
"cpu": true, "ram": true, "swap": true,
|
||||
"load": true, "temp": true, "disk": true, "network": true,
|
||||
"process": true, "connections": true, "all": true, "": true,
|
||||
}
|
||||
|
||||
if !validLoadTypes[loadType] {
|
||||
api.RespondError(c, 400, "Invalid load_type parameter")
|
||||
return
|
||||
}
|
||||
|
||||
clientRecords, err := records.GetRecordsByClientAndTime(uuid, time.Now().Add(-time.Duration(hoursInt)*time.Hour), time.Now())
|
||||
if err != nil {
|
||||
api.RespondError(c, 500, "Failed to fetch records: "+err.Error())
|
||||
return
|
||||
}
|
||||
|
||||
// 准备基本响应
|
||||
response := gin.H{
|
||||
"records": clientRecords,
|
||||
"count": len(clientRecords),
|
||||
}
|
||||
|
||||
// 如果有load_type过滤,应用过滤
|
||||
if loadType != "" && loadType != "all" {
|
||||
filteredRecords := filterRecordsByLoadType(clientRecords, loadType)
|
||||
response = gin.H{
|
||||
"records": filteredRecords,
|
||||
"count": len(filteredRecords),
|
||||
"load_type": loadType,
|
||||
}
|
||||
}
|
||||
|
||||
// 自动检测是否有GPU数据并附加到响应中
|
||||
if loadType == "" || loadType == "all" || loadType == "gpu" {
|
||||
gpuRecords, err := records.GetGPURecordsByClientAndTime(uuid, time.Now().Add(-time.Duration(hoursInt)*time.Hour), time.Now())
|
||||
if err == nil && len(gpuRecords) > 0 {
|
||||
// 按设备索引分组数据,构建简化的设备结构
|
||||
gpuDevices := make(map[string]interface{})
|
||||
|
||||
for _, record := range gpuRecords {
|
||||
deviceKey := strconv.Itoa(record.DeviceIndex)
|
||||
|
||||
// 如果设备还没有初始化,创建设备信息
|
||||
if gpuDevices[deviceKey] == nil {
|
||||
gpuDevices[deviceKey] = gin.H{
|
||||
"device_index": record.DeviceIndex,
|
||||
"device_name": record.DeviceName,
|
||||
"records": []models.GPURecord{},
|
||||
}
|
||||
}
|
||||
|
||||
// 添加记录到设备
|
||||
device := gpuDevices[deviceKey].(gin.H)
|
||||
records := device["records"].([]models.GPURecord)
|
||||
device["records"] = append(records, record)
|
||||
gpuDevices[deviceKey] = device
|
||||
}
|
||||
|
||||
// 添加优化后的GPU数据结构到响应
|
||||
response["gpu_devices"] = gpuDevices
|
||||
response["has_gpu_data"] = true
|
||||
} else {
|
||||
response["has_gpu_data"] = false
|
||||
}
|
||||
}
|
||||
|
||||
api.RespondSuccess(c, response)
|
||||
}
|
||||
|
||||
// filterRecordsByLoadType 根据 load_type 过滤记录,只返回相关字段
|
||||
func filterRecordsByLoadType(records []models.Record, loadType string) []gin.H {
|
||||
filteredRecords := make([]gin.H, 0, len(records))
|
||||
|
||||
for _, r := range records {
|
||||
record := gin.H{
|
||||
"client": r.Client,
|
||||
"time": r.Time,
|
||||
}
|
||||
|
||||
switch loadType {
|
||||
case "cpu":
|
||||
record["cpu"] = r.Cpu
|
||||
case "gpu":
|
||||
record["gpu"] = r.Gpu
|
||||
case "ram":
|
||||
record["ram"] = r.Ram
|
||||
record["ram_total"] = r.RamTotal
|
||||
if r.RamTotal > 0 {
|
||||
record["ram_percent"] = float32(r.Ram) / float32(r.RamTotal) * 100
|
||||
}
|
||||
case "swap":
|
||||
record["swap"] = r.Swap
|
||||
record["swap_total"] = r.SwapTotal
|
||||
if r.SwapTotal > 0 {
|
||||
record["swap_percent"] = float32(r.Swap) / float32(r.SwapTotal) * 100
|
||||
}
|
||||
case "load":
|
||||
record["load"] = r.Load
|
||||
case "temp":
|
||||
record["temp"] = r.Temp
|
||||
case "disk":
|
||||
record["disk"] = r.Disk
|
||||
record["disk_total"] = r.DiskTotal
|
||||
if r.DiskTotal > 0 {
|
||||
record["disk_percent"] = float32(r.Disk) / float32(r.DiskTotal) * 100
|
||||
}
|
||||
case "network":
|
||||
record["net_in"] = r.NetIn
|
||||
record["net_out"] = r.NetOut
|
||||
record["net_total_up"] = r.NetTotalUp
|
||||
record["net_total_down"] = r.NetTotalDown
|
||||
case "process":
|
||||
record["process"] = r.Process
|
||||
case "connections":
|
||||
record["connections"] = r.Connections
|
||||
record["connections_udp"] = r.ConnectionsUdp
|
||||
record["connections_tcp"] = r.Connections - r.ConnectionsUdp
|
||||
}
|
||||
|
||||
filteredRecords = append(filteredRecords, record)
|
||||
}
|
||||
|
||||
return filteredRecords
|
||||
}
|
||||
|
||||
// GET query: uuid string, task_id int, hours int
|
||||
// 支持三种查询模式:
|
||||
// 1. 仅 uuid - 获取该客户端的所有 ping 任务记录
|
||||
// 2. 仅 task_id - 获取该任务的所有客户端记录
|
||||
// 3. uuid + task_id - 获取特定客户端在特定任务的记录
|
||||
func GetPingRecords(c *gin.Context) {
|
||||
uuid := c.Query("uuid")
|
||||
taskIdStr := c.Query("task_id")
|
||||
|
||||
// 必须提供 uuid 或 task_id 其中至少一个
|
||||
if uuid == "" && taskIdStr == "" {
|
||||
api.RespondError(c, 400, "UUID or task_id is required")
|
||||
return
|
||||
}
|
||||
|
||||
// 登录状态检查
|
||||
isLogin := false
|
||||
session, _ := c.Cookie("session_token")
|
||||
_, err := accounts.GetUserBySession(session)
|
||||
if err == nil {
|
||||
isLogin = true
|
||||
}
|
||||
|
||||
type RecordsResp struct {
|
||||
TaskId uint `json:"task_id,omitempty"`
|
||||
Time string `json:"time"`
|
||||
Value int `json:"value"`
|
||||
Client string `json:"client,omitempty"`
|
||||
}
|
||||
type ClientBasicInfo struct {
|
||||
Client string `json:"client"`
|
||||
Loss float64 `json:"loss"`
|
||||
Min int `json:"min"`
|
||||
Max int `json:"max"`
|
||||
}
|
||||
type Resp struct {
|
||||
Count int `json:"count"`
|
||||
BasicInfo []ClientBasicInfo `json:"basic_info,omitempty"`
|
||||
Records []RecordsResp `json:"records"`
|
||||
Tasks []gin.H `json:"tasks,omitempty"`
|
||||
}
|
||||
var records []models.PingRecord
|
||||
hiddenMap := map[string]bool{}
|
||||
response := &Resp{
|
||||
Count: 0,
|
||||
Records: []RecordsResp{},
|
||||
}
|
||||
|
||||
// 仅在未登录时需要 Hidden 信息做过滤
|
||||
if !isLogin {
|
||||
var hiddenClients []models.Client
|
||||
db := dbcore.GetDBInstance()
|
||||
_ = db.Select("uuid").Where("hidden = ?", true).Find(&hiddenClients).Error
|
||||
for _, cli := range hiddenClients {
|
||||
hiddenMap[cli.UUID] = true
|
||||
}
|
||||
if uuid != "" {
|
||||
if hiddenMap[uuid] {
|
||||
api.RespondSuccess(c, response) // 对于尝试获取隐藏uuid一键哈气
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
hours := c.Query("hours")
|
||||
|
||||
if hours == "" {
|
||||
hours = "4"
|
||||
}
|
||||
|
||||
hoursInt, err := strconv.Atoi(hours)
|
||||
if err != nil {
|
||||
hoursInt = 4
|
||||
}
|
||||
|
||||
endTime := time.Now()
|
||||
startTime := endTime.Add(-time.Duration(hoursInt) * time.Hour)
|
||||
|
||||
// 解析 task_id,支持同时传入 uuid 和 task_id
|
||||
taskId := -1
|
||||
if taskIdStr != "" {
|
||||
taskId, err = strconv.Atoi(taskIdStr)
|
||||
if err != nil {
|
||||
api.RespondError(c, 400, "Invalid task_id parameter")
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
// 查询记录,现在支持 uuid + task_id 组合查询
|
||||
records, err = tasks.GetPingRecords(uuid, taskId, startTime, endTime)
|
||||
if err != nil {
|
||||
api.RespondError(c, 500, "Failed to fetch ping records: "+err.Error())
|
||||
return
|
||||
}
|
||||
|
||||
// 用于统计每个客户端的信息(按 task_id 查询时使用)
|
||||
clientStats := make(map[string]struct {
|
||||
total int
|
||||
loss int
|
||||
min int
|
||||
max int
|
||||
})
|
||||
|
||||
for _, r := range records {
|
||||
if r.Client != "" && !isLogin {
|
||||
if hiddenMap[r.Client] {
|
||||
continue // 跳过隐藏的节点
|
||||
}
|
||||
}
|
||||
toTime := r.Time.ToTime()
|
||||
rec := RecordsResp{
|
||||
Time: toTime.Format(time.RFC3339),
|
||||
Value: r.Value,
|
||||
}
|
||||
rec.Client = r.Client
|
||||
stats := clientStats[r.Client]
|
||||
stats.total++
|
||||
|
||||
if r.Value < 0 {
|
||||
stats.loss++
|
||||
} else {
|
||||
if stats.min == 0 || r.Value < stats.min {
|
||||
stats.min = r.Value
|
||||
}
|
||||
if r.Value > stats.max {
|
||||
stats.max = r.Value
|
||||
}
|
||||
}
|
||||
clientStats[r.Client] = stats
|
||||
rec.TaskId = r.TaskId
|
||||
|
||||
response.Records = append(response.Records, rec)
|
||||
}
|
||||
|
||||
// 返回 BasicInfo - 按客户端分组的统计信息
|
||||
// 在以下情况下特别有用:
|
||||
// 1. 仅 task_id 查询 - 查看该任务下所有客户端的表现
|
||||
// 2. uuid + task_id 查询 - 查看特定客户端在特定任务的表现
|
||||
if len(clientStats) > 0 {
|
||||
response.BasicInfo = make([]ClientBasicInfo, 0, len(clientStats))
|
||||
for client, stats := range clientStats {
|
||||
if client != "" && !isLogin {
|
||||
if hiddenMap[client] {
|
||||
continue // 跳过隐藏的节点
|
||||
}
|
||||
}
|
||||
loss := float64(0)
|
||||
if stats.total > 0 {
|
||||
loss = float64(stats.loss) / float64(stats.total) * 100
|
||||
}
|
||||
response.BasicInfo = append(response.BasicInfo, ClientBasicInfo{
|
||||
Client: client,
|
||||
Loss: loss,
|
||||
Min: stats.min,
|
||||
Max: stats.max,
|
||||
})
|
||||
}
|
||||
|
||||
// 如果同时指定了 uuid 和 task_id,BasicInfo 应该只有一条记录
|
||||
// 这种情况下可以在响应中标记查询模式
|
||||
if uuid != "" && taskId != -1 && len(response.BasicInfo) == 1 {
|
||||
// 这是精确查询模式
|
||||
}
|
||||
}
|
||||
|
||||
// 优化后的任务信息返回逻辑
|
||||
// 1. uuid != "" - 返回该客户端参与的所有任务信息
|
||||
// 2. uuid != "" && taskId != -1 - 返回该客户端在指定任务的信息
|
||||
// 3. taskId != -1 && uuid == "" - 返回该任务的所有客户端统计(通过 BasicInfo)
|
||||
if uuid != "" || taskId != -1 {
|
||||
// 获取所有 pingTasks
|
||||
pingTasks, err := tasks.GetAllPingTasks()
|
||||
if err != nil {
|
||||
api.RespondError(c, 500, "Failed to fetch ping tasks: "+err.Error())
|
||||
return
|
||||
}
|
||||
|
||||
tasksList := make([]gin.H, 0, len(pingTasks))
|
||||
for _, t := range pingTasks {
|
||||
// 如果指定了 taskId,只处理该任务
|
||||
if taskId != -1 {
|
||||
if t.Id != uint(taskId) {
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
// 如果指定了 uuid,检查任务是否分配给该客户端
|
||||
if uuid != "" {
|
||||
found := slices.Contains(t.Clients, uuid)
|
||||
if !found {
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
// 计算该任务的丢包率和延迟统计
|
||||
totalCount := 0
|
||||
lossCount := 0
|
||||
minLatency := 0
|
||||
maxLatency := 0
|
||||
avgLatency := 0
|
||||
sumLatency := 0
|
||||
validCount := 0
|
||||
|
||||
for _, r := range records {
|
||||
// 根据查询模式过滤记录
|
||||
if r.TaskId != t.Id {
|
||||
continue
|
||||
}
|
||||
// 如果同时指定了 uuid 和 task_id,只统计该客户端的记录
|
||||
if uuid != "" && r.Client != uuid {
|
||||
continue
|
||||
}
|
||||
|
||||
totalCount++
|
||||
if r.Value < 0 {
|
||||
lossCount++
|
||||
} else {
|
||||
validCount++
|
||||
sumLatency += r.Value
|
||||
if minLatency == 0 || r.Value < minLatency {
|
||||
minLatency = r.Value
|
||||
}
|
||||
if r.Value > maxLatency {
|
||||
maxLatency = r.Value
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
var lossRate float64 = 0
|
||||
if totalCount > 0 {
|
||||
lossRate = float64(lossCount) / float64(totalCount) * 100
|
||||
}
|
||||
if validCount > 0 {
|
||||
avgLatency = sumLatency / validCount
|
||||
}
|
||||
|
||||
taskInfo := gin.H{
|
||||
"id": t.Id,
|
||||
"name": t.Name,
|
||||
"type": t.Type,
|
||||
"interval": t.Interval,
|
||||
"loss": lossRate,
|
||||
"min": minLatency,
|
||||
"max": maxLatency,
|
||||
"avg": avgLatency,
|
||||
"total": totalCount,
|
||||
}
|
||||
|
||||
// 如果是仅 task_id 查询,添加客户端列表信息
|
||||
if uuid == "" && taskId != -1 {
|
||||
taskInfo["clients"] = t.Clients
|
||||
}
|
||||
|
||||
tasksList = append(tasksList, taskInfo)
|
||||
}
|
||||
response.Tasks = tasksList
|
||||
}
|
||||
|
||||
response.Count = len(response.Records) // 计算最后结果保持计数一致
|
||||
api.RespondSuccess(c, response)
|
||||
}
|
||||
@@ -1,38 +0,0 @@
|
||||
package task
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
|
||||
"github.com/gin-gonic/gin"
|
||||
"github.com/komari-monitor/komari/api"
|
||||
"github.com/komari-monitor/komari/database/tasks"
|
||||
)
|
||||
|
||||
type PublicPingTask struct {
|
||||
Id uint `json:"id"`
|
||||
Name string `json:"name"`
|
||||
Clients []string `json:"clients"`
|
||||
Type string `json:"type"`
|
||||
Interval int `json:"interval"`
|
||||
}
|
||||
|
||||
func GetPublicPingTasks(c *gin.Context) {
|
||||
tasks, err := tasks.GetAllPingTasks()
|
||||
if err != nil {
|
||||
api.RespondError(c, http.StatusInternalServerError, err.Error())
|
||||
return
|
||||
}
|
||||
|
||||
publicTasks := make([]PublicPingTask, len(tasks))
|
||||
for i, task := range tasks {
|
||||
publicTasks[i] = PublicPingTask{
|
||||
Id: task.Id,
|
||||
Name: task.Name,
|
||||
Clients: task.Clients,
|
||||
Type: task.Type,
|
||||
Interval: task.Interval,
|
||||
}
|
||||
}
|
||||
|
||||
api.RespondSuccess(c, publicTasks)
|
||||
}
|
||||
-167
@@ -1,167 +0,0 @@
|
||||
package api
|
||||
|
||||
import (
|
||||
"log"
|
||||
"net/http"
|
||||
"time"
|
||||
|
||||
"github.com/gin-gonic/gin"
|
||||
"github.com/gorilla/websocket"
|
||||
"github.com/komari-monitor/komari/database/auditlog"
|
||||
"github.com/komari-monitor/komari/database/clients"
|
||||
"github.com/komari-monitor/komari/utils"
|
||||
"github.com/komari-monitor/komari/ws"
|
||||
)
|
||||
|
||||
func RequestTerminal(c *gin.Context) {
|
||||
uuid := c.Param("uuid")
|
||||
user_uuid, _ := c.Get("uuid")
|
||||
_, err := clients.GetClientByUUID(uuid)
|
||||
if err != nil {
|
||||
c.JSON(400, gin.H{
|
||||
"status": "error",
|
||||
"message": "Client not found",
|
||||
})
|
||||
return
|
||||
}
|
||||
// 建立ws
|
||||
if !websocket.IsWebSocketUpgrade(c.Request) {
|
||||
c.JSON(http.StatusBadRequest, gin.H{"status": "error", "message": "Require WebSocket upgrade"})
|
||||
return
|
||||
}
|
||||
upgrader := websocket.Upgrader{
|
||||
CheckOrigin: ws.CheckOrigin,
|
||||
}
|
||||
conn, err := upgrader.Upgrade(c.Writer, c.Request, nil)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
// 新建一个终端连接
|
||||
id := utils.GenerateRandomString(32)
|
||||
session := &TerminalSession{
|
||||
UserUUID: user_uuid.(string),
|
||||
UUID: uuid,
|
||||
Browser: conn,
|
||||
Agent: nil,
|
||||
RequesterIp: c.ClientIP(),
|
||||
}
|
||||
|
||||
TerminalSessionsMutex.Lock()
|
||||
TerminalSessions[id] = session
|
||||
TerminalSessionsMutex.Unlock()
|
||||
conn.SetCloseHandler(func(code int, text string) error {
|
||||
log.Println("Terminal connection closed:", code, text)
|
||||
TerminalSessionsMutex.Lock()
|
||||
delete(TerminalSessions, id)
|
||||
TerminalSessionsMutex.Unlock()
|
||||
// 通知 Agent 关闭终端连接
|
||||
if session.Agent != nil {
|
||||
session.Agent.Close()
|
||||
}
|
||||
return nil
|
||||
})
|
||||
|
||||
if ws.GetConnectedClients()[uuid] == nil {
|
||||
conn.WriteMessage(1, []byte("Client offline!\n被控端离线!"))
|
||||
conn.Close()
|
||||
TerminalSessionsMutex.Lock()
|
||||
delete(TerminalSessions, id)
|
||||
TerminalSessionsMutex.Unlock()
|
||||
return
|
||||
}
|
||||
err = ws.GetConnectedClients()[uuid].WriteJSON(gin.H{
|
||||
"message": "terminal",
|
||||
"request_id": id,
|
||||
})
|
||||
if err != nil {
|
||||
conn.Close()
|
||||
TerminalSessionsMutex.Lock()
|
||||
delete(TerminalSessions, id)
|
||||
TerminalSessionsMutex.Unlock()
|
||||
return
|
||||
}
|
||||
conn.WriteMessage(1, []byte("等待被控端连接 waiting for agent..."))
|
||||
// 如果没有连接上,则关闭连接
|
||||
time.AfterFunc(30*time.Second, func() {
|
||||
TerminalSessionsMutex.Lock()
|
||||
if session.Agent == nil {
|
||||
if session.Browser != nil {
|
||||
session.Browser.WriteMessage(1, []byte("被控端连接超时 timeout"))
|
||||
session.Browser.Close()
|
||||
}
|
||||
conn.Close()
|
||||
delete(TerminalSessions, id)
|
||||
}
|
||||
TerminalSessionsMutex.Unlock()
|
||||
})
|
||||
//auditlog.Log(c.ClientIP(), user_uuid.(string), "request, terminal id:"+id+",client:"+session.UUID, "terminal")
|
||||
}
|
||||
|
||||
func ForwardTerminal(id string) {
|
||||
session, exists := TerminalSessions[id]
|
||||
|
||||
if !exists || session == nil || session.Agent == nil || session.Browser == nil {
|
||||
return
|
||||
}
|
||||
auditlog.Log(session.RequesterIp, session.UserUUID, "established, terminal id:"+id, "terminal")
|
||||
established_time := time.Now()
|
||||
errChan := make(chan error, 1)
|
||||
|
||||
go func() {
|
||||
for {
|
||||
messageType, data, err := session.Browser.ReadMessage()
|
||||
if err != nil {
|
||||
errChan <- err
|
||||
return
|
||||
}
|
||||
|
||||
if messageType == websocket.TextMessage {
|
||||
if session.Agent != nil && string(data[0:1]) == "{" {
|
||||
err = session.Agent.WriteMessage(websocket.TextMessage, data)
|
||||
} else if session.Agent != nil {
|
||||
err = session.Agent.WriteMessage(websocket.BinaryMessage, data)
|
||||
}
|
||||
} else if session.Agent != nil {
|
||||
// 二进制消息,原样传递
|
||||
err = session.Agent.WriteMessage(websocket.BinaryMessage, data)
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
errChan <- err
|
||||
return
|
||||
}
|
||||
}
|
||||
}()
|
||||
|
||||
go func() {
|
||||
for {
|
||||
_, data, err := session.Agent.ReadMessage()
|
||||
if err != nil {
|
||||
errChan <- err
|
||||
return
|
||||
}
|
||||
if session.Browser != nil {
|
||||
err = session.Browser.WriteMessage(websocket.BinaryMessage, data)
|
||||
if err != nil {
|
||||
errChan <- err
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
}()
|
||||
|
||||
// 等待错误或主动关闭
|
||||
<-errChan
|
||||
// 关闭连接
|
||||
if session.Agent != nil {
|
||||
session.Agent.Close()
|
||||
}
|
||||
if session.Browser != nil {
|
||||
session.Browser.Close()
|
||||
}
|
||||
disconnect_time := time.Now()
|
||||
auditlog.Log(session.RequesterIp, session.UserUUID, "disconnected, terminal id:"+id+", duration:"+disconnect_time.Sub(established_time).String(), "terminal")
|
||||
TerminalSessionsMutex.Lock()
|
||||
delete(TerminalSessions, id)
|
||||
TerminalSessionsMutex.Unlock()
|
||||
}
|
||||
+33
-6
@@ -1,14 +1,41 @@
|
||||
package flags
|
||||
|
||||
import "strings"
|
||||
|
||||
const (
|
||||
DatabaseTypeSQLite = "sqlite"
|
||||
)
|
||||
|
||||
var (
|
||||
// 数据库配置
|
||||
DatabaseType string // 数据库类型:sqlite, mysql
|
||||
DatabaseType string // 数据库类型:sqlite
|
||||
DatabaseFile string // SQLite数据库文件路径
|
||||
DatabaseHost string // MySQL/其他数据库主机地址
|
||||
DatabasePort string // MySQL/其他数据库端口
|
||||
DatabaseUser string // MySQL/其他数据库用户名
|
||||
DatabasePass string // MySQL/其他数据库密码
|
||||
DatabaseName string // MySQL/其他数据库名称
|
||||
DatabaseHost string // 保留的兼容参数,当前未使用
|
||||
DatabasePort string // 保留的兼容参数,当前未使用
|
||||
DatabaseUser string // 保留的兼容参数,当前未使用
|
||||
DatabasePass string // 保留的兼容参数,当前未使用
|
||||
DatabaseName string // 保留的兼容参数,当前未使用
|
||||
|
||||
Listen string
|
||||
)
|
||||
|
||||
func NormalizeDatabaseType(databaseType string) string {
|
||||
databaseType = strings.ToLower(strings.TrimSpace(databaseType))
|
||||
if databaseType == "" {
|
||||
return DatabaseTypeSQLite
|
||||
}
|
||||
return databaseType
|
||||
}
|
||||
|
||||
func ApplyDatabaseTypeNormalization() string {
|
||||
DatabaseType = NormalizeDatabaseType(DatabaseType)
|
||||
return DatabaseType
|
||||
}
|
||||
|
||||
func IsSQLite() bool {
|
||||
return NormalizeDatabaseType(DatabaseType) == DatabaseTypeSQLite
|
||||
}
|
||||
|
||||
func SupportedDatabaseTypes() string {
|
||||
return DatabaseTypeSQLite
|
||||
}
|
||||
@@ -0,0 +1,20 @@
|
||||
package flags
|
||||
|
||||
import "testing"
|
||||
|
||||
func TestNormalizeDatabaseType(t *testing.T) {
|
||||
tests := map[string]string{
|
||||
"": DatabaseTypeSQLite,
|
||||
"sqlite": DatabaseTypeSQLite,
|
||||
" SQLite ": DatabaseTypeSQLite,
|
||||
"SQLITE": DatabaseTypeSQLite,
|
||||
"postgres": "postgres",
|
||||
" postgres": "postgres",
|
||||
}
|
||||
|
||||
for input, want := range tests {
|
||||
if got := NormalizeDatabaseType(input); got != want {
|
||||
t.Fatalf("NormalizeDatabaseType(%q) = %q, want %q", input, got, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -4,9 +4,8 @@ import (
|
||||
"os"
|
||||
|
||||
"github.com/komari-monitor/komari/database/dbcore"
|
||||
"github.com/komari-monitor/komari/database/models"
|
||||
"github.com/komari-monitor/komari/pkg/config"
|
||||
"github.com/spf13/cobra"
|
||||
"gorm.io/gorm"
|
||||
)
|
||||
|
||||
var PermitPasswordLoginCmd = &cobra.Command{
|
||||
@@ -14,12 +13,8 @@ var PermitPasswordLoginCmd = &cobra.Command{
|
||||
Short: "Force permit password login",
|
||||
Long: `Force permit password login`,
|
||||
Run: func(cmd *cobra.Command, args []string) {
|
||||
db := dbcore.GetDBInstance()
|
||||
err := db.Transaction(func(tx *gorm.DB) error {
|
||||
return tx.Model(&models.Config{}).Where("id = ?", 1).
|
||||
Update("disable_password_login", false).Error
|
||||
})
|
||||
if err != nil {
|
||||
dbcore.GetDBInstance()
|
||||
if err := config.Set(config.DisablePasswordLoginKey, false); err != nil {
|
||||
cmd.Println("Error:", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
+9
-6
@@ -48,11 +48,14 @@ func Execute() {
|
||||
|
||||
func init() {
|
||||
// 设置命令行参数,提供环境变量作为默认值
|
||||
RootCmd.PersistentFlags().StringVarP(&flags.DatabaseType, "db-type", "t", dbTypeEnv, "Database type (sqlite, mysql) [env: KOMARI_DB_TYPE]")
|
||||
RootCmd.PersistentFlags().StringVarP(&flags.DatabaseType, "db-type", "t", dbTypeEnv, "Database type (sqlite) [env: KOMARI_DB_TYPE]")
|
||||
RootCmd.PersistentFlags().StringVarP(&flags.DatabaseFile, "database", "d", dbFileEnv, "SQLite database file path [env: KOMARI_DB_FILE]")
|
||||
RootCmd.PersistentFlags().StringVar(&flags.DatabaseHost, "db-host", dbHostEnv, "MySQL/Other database host address [env: KOMARI_DB_HOST]")
|
||||
RootCmd.PersistentFlags().StringVar(&flags.DatabasePort, "db-port", dbPortEnv, "MySQL/Other database port [env: KOMARI_DB_PORT]")
|
||||
RootCmd.PersistentFlags().StringVar(&flags.DatabaseUser, "db-user", dbUserEnv, "MySQL/Other database username [env: KOMARI_DB_USER]")
|
||||
RootCmd.PersistentFlags().StringVar(&flags.DatabasePass, "db-pass", dbPassEnv, "MySQL/Other database password [env: KOMARI_DB_PASS]")
|
||||
RootCmd.PersistentFlags().StringVar(&flags.DatabaseName, "db-name", dbNameEnv, "MySQL/Other database name [env: KOMARI_DB_NAME]")
|
||||
RootCmd.PersistentFlags().StringVar(&flags.DatabaseHost, "db-host", dbHostEnv, "Reserved database host parameter [env: KOMARI_DB_HOST]")
|
||||
RootCmd.PersistentFlags().StringVar(&flags.DatabasePort, "db-port", dbPortEnv, "Reserved database port parameter [env: KOMARI_DB_PORT]")
|
||||
RootCmd.PersistentFlags().StringVar(&flags.DatabaseUser, "db-user", dbUserEnv, "Reserved database username parameter [env: KOMARI_DB_USER]")
|
||||
RootCmd.PersistentFlags().StringVar(&flags.DatabasePass, "db-pass", dbPassEnv, "Reserved database password parameter [env: KOMARI_DB_PASS]")
|
||||
RootCmd.PersistentFlags().StringVar(&flags.DatabaseName, "db-name", dbNameEnv, "Reserved database name parameter [env: KOMARI_DB_NAME]")
|
||||
for _, name := range []string{"db-host", "db-port", "db-user", "db-pass", "db-name"} {
|
||||
_ = RootCmd.PersistentFlags().MarkHidden(name)
|
||||
}
|
||||
}
|
||||
+76
-267
@@ -7,46 +7,37 @@ import (
|
||||
"net/http"
|
||||
"os"
|
||||
"os/signal"
|
||||
"strings"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
"github.com/gin-gonic/gin"
|
||||
"github.com/komari-monitor/komari/api"
|
||||
"github.com/komari-monitor/komari/api/admin"
|
||||
"github.com/komari-monitor/komari/api/admin/clipboard"
|
||||
log_api "github.com/komari-monitor/komari/api/admin/log"
|
||||
"github.com/komari-monitor/komari/api/admin/notification"
|
||||
"github.com/komari-monitor/komari/api/admin/test"
|
||||
"github.com/komari-monitor/komari/api/admin/update"
|
||||
"github.com/komari-monitor/komari/api/client"
|
||||
"github.com/komari-monitor/komari/api/jsonRpc"
|
||||
"github.com/komari-monitor/komari/api/record"
|
||||
"github.com/komari-monitor/komari/api/task"
|
||||
"github.com/komari-monitor/komari/cmd/flags"
|
||||
"github.com/komari-monitor/komari/pkg/corn"
|
||||
"github.com/komari-monitor/komari/web/api"
|
||||
|
||||
"github.com/komari-monitor/komari/database"
|
||||
"github.com/komari-monitor/komari/database/accounts"
|
||||
"github.com/komari-monitor/komari/database/auditlog"
|
||||
"github.com/komari-monitor/komari/database/config"
|
||||
"github.com/komari-monitor/komari/database/dbcore"
|
||||
"github.com/komari-monitor/komari/database/models"
|
||||
d_notification "github.com/komari-monitor/komari/database/notification"
|
||||
"github.com/komari-monitor/komari/database/records"
|
||||
"github.com/komari-monitor/komari/database/tasks"
|
||||
"github.com/komari-monitor/komari/public"
|
||||
"github.com/komari-monitor/komari/pkg/config"
|
||||
"github.com/komari-monitor/komari/utils"
|
||||
"github.com/komari-monitor/komari/utils/cloudflared"
|
||||
"github.com/komari-monitor/komari/utils/geoip"
|
||||
logutil "github.com/komari-monitor/komari/utils/log"
|
||||
"github.com/komari-monitor/komari/utils/messageSender"
|
||||
"github.com/komari-monitor/komari/utils/notifier"
|
||||
"github.com/komari-monitor/komari/utils/oauth"
|
||||
"github.com/komari-monitor/komari/web/nezha"
|
||||
"github.com/komari-monitor/komari/web/oauth"
|
||||
report_cache "github.com/komari-monitor/komari/web/report"
|
||||
"github.com/komari-monitor/komari/web/router"
|
||||
"github.com/komari-monitor/komari/web/security"
|
||||
"github.com/spf13/cobra"
|
||||
)
|
||||
|
||||
var (
|
||||
DynamicCorsEnabled bool = false
|
||||
)
|
||||
|
||||
var ServerCmd = &cobra.Command{
|
||||
Use: "server",
|
||||
Short: "Start the server",
|
||||
@@ -65,10 +56,6 @@ func init() {
|
||||
|
||||
func RunServer() {
|
||||
// #region 初始化
|
||||
if err := os.MkdirAll("./data", os.ModePerm); err != nil {
|
||||
log.Fatalf("Failed to create data directory: %v", err)
|
||||
}
|
||||
// 创建主题目录
|
||||
if err := os.MkdirAll("./data/theme", os.ModePerm); err != nil {
|
||||
log.Fatalf("Failed to create theme directory: %v", err)
|
||||
}
|
||||
@@ -76,7 +63,7 @@ func RunServer() {
|
||||
if utils.VersionHash != "unknown" {
|
||||
gin.SetMode(gin.ReleaseMode)
|
||||
}
|
||||
conf, err := config.Get()
|
||||
conf, err := config.GetManyAs[config.Settings]()
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
@@ -88,7 +75,7 @@ func RunServer() {
|
||||
|
||||
if conf.NezhaCompatEnabled {
|
||||
go func() {
|
||||
if err := StartNezhaCompat(conf.NezhaCompatListen); err != nil {
|
||||
if err := nezha.StartNezhaCompat(conf.NezhaCompatListen); err != nil {
|
||||
log.Printf("Nezha compat server error: %v", err)
|
||||
auditlog.EventLog("error", fmt.Sprintf("Nezha compat server error: %v", err))
|
||||
}
|
||||
@@ -96,8 +83,11 @@ func RunServer() {
|
||||
}
|
||||
|
||||
config.Subscribe(func(event config.ConfigEvent) {
|
||||
if event.New.OAuthProvider != event.Old.OAuthProvider {
|
||||
oidcProvider, err := database.GetOidcConfigByName(event.New.OAuthProvider)
|
||||
if ok, t := config.IsChangedT[string](event, config.OAuthProviderKey); ok {
|
||||
if t == "" || t == "none" {
|
||||
t = "github"
|
||||
}
|
||||
oidcProvider, err := database.GetOidcConfigByName(t)
|
||||
if err != nil {
|
||||
log.Printf("Failed to get OIDC provider config: %v", err)
|
||||
} else {
|
||||
@@ -108,17 +98,16 @@ func RunServer() {
|
||||
auditlog.EventLog("error", fmt.Sprintf("Failed to load OIDC provider: %v", err))
|
||||
}
|
||||
}
|
||||
if event.New.NotificationMethod != event.Old.NotificationMethod {
|
||||
messageSender.Initialize()
|
||||
}
|
||||
if event.New.NezhaCompatEnabled != event.Old.NezhaCompatEnabled {
|
||||
if event.New.NezhaCompatEnabled {
|
||||
if err := StartNezhaCompat(event.New.NezhaCompatListen); err != nil {
|
||||
|
||||
if ok, t := config.IsChangedT[bool](event, config.NezhaCompatEnabledKey); ok {
|
||||
if t {
|
||||
l, _ := config.GetAs[string](config.NezhaCompatListenKey)
|
||||
if err := nezha.StartNezhaCompat(l); err != nil {
|
||||
log.Printf("start Nezha compat server error: %v", err)
|
||||
auditlog.EventLog("error", fmt.Sprintf("start Nezha compat server error: %v", err))
|
||||
}
|
||||
} else {
|
||||
if err := StopNezhaCompat(); err != nil {
|
||||
if err := nezha.StopNezhaCompat(); err != nil {
|
||||
log.Printf("stop Nezha compat server error: %v", err)
|
||||
auditlog.EventLog("error", fmt.Sprintf("stop Nezha compat server error: %v", err))
|
||||
}
|
||||
@@ -127,43 +116,27 @@ func RunServer() {
|
||||
|
||||
})
|
||||
// 初始化 cloudflared
|
||||
if strings.ToLower(GetEnv("KOMARI_ENABLE_CLOUDFLARED", "false")) == "true" {
|
||||
err := cloudflared.RunCloudflared() // 阻塞,确保cloudflared跑起来
|
||||
if err != nil {
|
||||
log.Fatalf("Failed to run cloudflared: %v", err)
|
||||
}
|
||||
if err := cloudflared.AutoStart(GetEnv("KOMARI_CLOUDFLARED_TOKEN", "")); err != nil {
|
||||
log.Printf("failed to auto start cloudflared: %v", err)
|
||||
}
|
||||
|
||||
r := gin.Default()
|
||||
r := gin.New()
|
||||
r.Use(logutil.GinLogger())
|
||||
r.Use(logutil.GinRecovery())
|
||||
|
||||
// 动态 CORS 中间件
|
||||
|
||||
DynamicCorsEnabled = conf.AllowCors
|
||||
config.Subscribe(func(event config.ConfigEvent) {
|
||||
DynamicCorsEnabled = event.New.AllowCors
|
||||
if event.New.GeoIpProvider != event.Old.GeoIpProvider {
|
||||
if event.IsChanged(config.GeoIpProviderKey) {
|
||||
go geoip.InitGeoIp()
|
||||
}
|
||||
if event.New.NotificationMethod != event.Old.NotificationMethod {
|
||||
|
||||
if event.IsChanged(config.NotificationMethodKey) {
|
||||
go messageSender.Initialize()
|
||||
}
|
||||
})
|
||||
r.Use(func(c *gin.Context) {
|
||||
if DynamicCorsEnabled {
|
||||
c.Header("Access-Control-Allow-Origin", "*")
|
||||
c.Header("Access-Control-Allow-Methods", "GET, POST, PUT, PATCH, DELETE, HEAD, OPTIONS")
|
||||
c.Header("Access-Control-Allow-Headers", "Origin, Content-Length, Content-Type, Authorization, Accept, X-CSRF-Token, X-Requested-With, Set-Cookie")
|
||||
c.Header("Access-Control-Expose-Headers", "Content-Length, Authorization, Set-Cookie")
|
||||
c.Header("Access-Control-Allow-Credentials", "true")
|
||||
c.Header("Access-Control-Max-Age", "43200") // 12 hours
|
||||
if c.Request.Method == "OPTIONS" {
|
||||
c.AbortWithStatus(204)
|
||||
return
|
||||
}
|
||||
}
|
||||
c.Next()
|
||||
})
|
||||
|
||||
})
|
||||
r.Use(security.CorsMiddleware(conf.CorsOriginCheckEnabled, conf.CorsAllowedOrigins))
|
||||
|
||||
r.Use(api.IdentityMiddleware())
|
||||
r.Use(api.PrivateSiteMiddleware())
|
||||
|
||||
r.Use(func(c *gin.Context) {
|
||||
@@ -173,172 +146,7 @@ func RunServer() {
|
||||
c.Next()
|
||||
})
|
||||
|
||||
r.Any("/ping", func(c *gin.Context) {
|
||||
c.String(200, "pong")
|
||||
})
|
||||
// #region 公开路由
|
||||
r.POST("/api/login", api.Login)
|
||||
r.GET("/api/me", api.GetMe)
|
||||
r.GET("/api/clients", api.GetClients)
|
||||
r.GET("/api/nodes", api.GetNodesInformation)
|
||||
r.GET("/api/public", api.GetPublicSettings)
|
||||
r.GET("/api/oauth", api.OAuth)
|
||||
r.GET("/api/oauth_callback", api.OAuthCallback)
|
||||
r.GET("/api/logout", api.Logout)
|
||||
r.GET("/api/version", api.GetVersion)
|
||||
r.GET("/api/recent/:uuid", api.GetClientRecentRecords)
|
||||
|
||||
r.GET("/api/records/load", record.GetRecordsByUUID)
|
||||
r.GET("/api/records/ping", record.GetPingRecords)
|
||||
r.GET("/api/task/ping", task.GetPublicPingTasks)
|
||||
r.GET("/api/rpc2", jsonRpc.OnRpcRequest)
|
||||
r.POST("/api/rpc2", jsonRpc.OnRpcRequest)
|
||||
|
||||
// #region Agent
|
||||
r.POST("/api/clients/register", client.RegisterClient)
|
||||
tokenAuthrized := r.Group("/api/clients", api.TokenAuthMiddleware())
|
||||
{
|
||||
tokenAuthrized.GET("/report", client.WebSocketReport) // websocket
|
||||
tokenAuthrized.POST("/uploadBasicInfo", client.UploadBasicInfo)
|
||||
tokenAuthrized.POST("/report", client.UploadReport)
|
||||
tokenAuthrized.GET("/terminal", client.EstablishConnection)
|
||||
tokenAuthrized.POST("/task/result", client.TaskResult)
|
||||
}
|
||||
// #region 管理员
|
||||
adminAuthrized := r.Group("/api/admin", api.AdminAuthMiddleware())
|
||||
{
|
||||
adminAuthrized.GET("/download/backup", admin.DownloadBackup)
|
||||
adminAuthrized.POST("/upload/backup", admin.UploadBackup)
|
||||
// test
|
||||
testGroup := adminAuthrized.Group("/test")
|
||||
{
|
||||
testGroup.GET("/geoip", test.TestGeoIp)
|
||||
testGroup.POST("/sendMessage", test.TestSendMessage)
|
||||
}
|
||||
// update
|
||||
updateGroup := adminAuthrized.Group("/update")
|
||||
{
|
||||
updateGroup.POST("/mmdb", update.UpdateMmdbGeoIP)
|
||||
updateGroup.POST("/user", update.UpdateUser)
|
||||
updateGroup.PUT("/favicon", update.UploadFavicon)
|
||||
updateGroup.POST("/favicon", update.DeleteFavicon)
|
||||
}
|
||||
// tasks
|
||||
taskGroup := adminAuthrized.Group("/task")
|
||||
{
|
||||
taskGroup.GET("/all", admin.GetTasks)
|
||||
taskGroup.POST("/exec", admin.Exec)
|
||||
taskGroup.GET("/:task_id", admin.GetTaskById)
|
||||
taskGroup.GET("/:task_id/result", admin.GetTaskResultsByTaskId)
|
||||
taskGroup.GET("/:task_id/result/:uuid", admin.GetSpecificTaskResult)
|
||||
taskGroup.GET("/client/:uuid", admin.GetTasksByClientId)
|
||||
}
|
||||
// settings
|
||||
settingsGroup := adminAuthrized.Group("/settings")
|
||||
{
|
||||
settingsGroup.GET("/", admin.GetSettings)
|
||||
settingsGroup.POST("/", admin.EditSettings)
|
||||
settingsGroup.POST("/oidc", admin.SetOidcProvider)
|
||||
settingsGroup.GET("/oidc", admin.GetOidcProvider)
|
||||
settingsGroup.POST("/message-sender", admin.SetMessageSenderProvider)
|
||||
settingsGroup.GET("/message-sender", admin.GetMessageSenderProvider)
|
||||
}
|
||||
// themes
|
||||
themeGroup := adminAuthrized.Group("/theme")
|
||||
{
|
||||
themeGroup.PUT("/upload", admin.UploadTheme)
|
||||
themeGroup.GET("/list", admin.ListThemes)
|
||||
themeGroup.POST("/delete", admin.DeleteTheme)
|
||||
themeGroup.GET("/set", admin.SetTheme)
|
||||
themeGroup.POST("/update", admin.UpdateTheme)
|
||||
themeGroup.POST("/settings", admin.UpdateThemeSettings)
|
||||
}
|
||||
// clients
|
||||
clientGroup := adminAuthrized.Group("/client")
|
||||
{
|
||||
clientGroup.POST("/add", admin.AddClient)
|
||||
clientGroup.GET("/list", admin.ListClients)
|
||||
clientGroup.GET("/:uuid", admin.GetClient)
|
||||
clientGroup.POST("/:uuid/edit", admin.EditClient)
|
||||
clientGroup.POST("/:uuid/remove", admin.RemoveClient)
|
||||
clientGroup.GET("/:uuid/token", admin.GetClientToken)
|
||||
clientGroup.POST("/order", admin.OrderWeight)
|
||||
// client terminal
|
||||
clientGroup.GET("/:uuid/terminal", api.RequestTerminal)
|
||||
}
|
||||
|
||||
// records
|
||||
recordGroup := adminAuthrized.Group("/record")
|
||||
{
|
||||
recordGroup.POST("/clear", admin.ClearRecord)
|
||||
recordGroup.POST("/clear/all", admin.ClearAllRecords)
|
||||
}
|
||||
// oauth2
|
||||
oauth2Group := adminAuthrized.Group("/oauth2")
|
||||
{
|
||||
oauth2Group.GET("/bind", admin.BindingExternalAccount)
|
||||
oauth2Group.POST("/unbind", admin.UnbindExternalAccount)
|
||||
}
|
||||
sessionGroup := adminAuthrized.Group("/session")
|
||||
{
|
||||
sessionGroup.GET("/get", admin.GetSessions)
|
||||
sessionGroup.POST("/remove", admin.DeleteSession)
|
||||
sessionGroup.POST("/remove/all", admin.DeleteAllSession)
|
||||
}
|
||||
two_factorGroup := adminAuthrized.Group("/2fa")
|
||||
{
|
||||
two_factorGroup.GET("/generate", admin.Generate2FA)
|
||||
two_factorGroup.POST("/enable", admin.Enable2FA)
|
||||
two_factorGroup.POST("/disable", admin.Disable2FA)
|
||||
}
|
||||
adminAuthrized.GET("/logs", log_api.GetLogs)
|
||||
|
||||
// clipboard
|
||||
clipboardGroup := adminAuthrized.Group("/clipboard")
|
||||
{
|
||||
clipboardGroup.GET("/:id", clipboard.GetClipboard)
|
||||
clipboardGroup.GET("", clipboard.ListClipboard)
|
||||
clipboardGroup.POST("", clipboard.CreateClipboard)
|
||||
clipboardGroup.POST("/:id", clipboard.UpdateClipboard)
|
||||
clipboardGroup.POST("/remove", clipboard.BatchDeleteClipboard)
|
||||
clipboardGroup.POST("/:id/remove", clipboard.DeleteClipboard)
|
||||
}
|
||||
|
||||
notificationGroup := adminAuthrized.Group("/notification")
|
||||
{
|
||||
// offline notifications
|
||||
notificationGroup.GET("/offline", notification.ListOfflineNotifications)
|
||||
notificationGroup.POST("/offline/edit", notification.EditOfflineNotification)
|
||||
notificationGroup.POST("/offline/enable", notification.EnableOfflineNotification)
|
||||
notificationGroup.POST("/offline/disable", notification.DisableOfflineNotification)
|
||||
loadAlertGroup := notificationGroup.Group("/load")
|
||||
{
|
||||
loadAlertGroup.GET("/", notification.GetAllLoadNotifications)
|
||||
loadAlertGroup.POST("/add", notification.AddLoadNotification)
|
||||
loadAlertGroup.POST("/delete", notification.DeleteLoadNotification)
|
||||
loadAlertGroup.POST("/edit", notification.EditLoadNotification)
|
||||
}
|
||||
}
|
||||
|
||||
pingTaskGroup := adminAuthrized.Group("/ping")
|
||||
{
|
||||
pingTaskGroup.GET("/", admin.GetAllPingTasks)
|
||||
pingTaskGroup.POST("/add", admin.AddPingTask)
|
||||
pingTaskGroup.POST("/delete", admin.DeletePingTask)
|
||||
pingTaskGroup.POST("/edit", admin.EditPingTask)
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
public.Static(r.Group("/"), func(handlers ...gin.HandlerFunc) {
|
||||
r.NoRoute(handlers...)
|
||||
})
|
||||
// #region 静态文件服务
|
||||
public.UpdateIndex(conf)
|
||||
config.Subscribe(func(event config.ConfigEvent) {
|
||||
public.UpdateIndex(event.New)
|
||||
})
|
||||
router.Register(r)
|
||||
|
||||
srv := &http.Server{
|
||||
Addr: flags.Listen,
|
||||
@@ -364,18 +172,6 @@ func RunServer() {
|
||||
}
|
||||
|
||||
func InitDatabase() {
|
||||
// // 打印数据库类型和连接信息
|
||||
// if flags.DatabaseType == "mysql" {
|
||||
// log.Printf("使用 MySQL 数据库连接: %s@%s:%s/%s",
|
||||
// flags.DatabaseUser, flags.DatabaseHost, flags.DatabasePort, flags.DatabaseName)
|
||||
// log.Printf("环境变量配置: [KOMARI_DB_TYPE=%s] [KOMARI_DB_HOST=%s] [KOMARI_DB_PORT=%s] [KOMARI_DB_USER=%s] [KOMARI_DB_NAME=%s]",
|
||||
// os.Getenv("KOMARI_DB_TYPE"), os.Getenv("KOMARI_DB_HOST"), os.Getenv("KOMARI_DB_PORT"),
|
||||
// os.Getenv("KOMARI_DB_USER"), os.Getenv("KOMARI_DB_NAME"))
|
||||
// } else {
|
||||
// log.Printf("使用 SQLite 数据库文件: %s", flags.DatabaseFile)
|
||||
// log.Printf("环境变量配置: [KOMARI_DB_TYPE=%s] [KOMARI_DB_FILE=%s]",
|
||||
// os.Getenv("KOMARI_DB_TYPE"), os.Getenv("KOMARI_DB_FILE"))
|
||||
// }
|
||||
var count int64 = 0
|
||||
if dbcore.GetDBInstance().Model(&models.User{}).Count(&count); count == 0 {
|
||||
user, passwd, err := accounts.CreateDefaultAdminAccount()
|
||||
@@ -388,41 +184,54 @@ func InitDatabase() {
|
||||
|
||||
// #region 定时任务
|
||||
func DoScheduledWork() {
|
||||
tasks.ReloadPingSchedule()
|
||||
d_notification.ReloadLoadNotificationSchedule()
|
||||
ticker := time.NewTicker(time.Minute * 30)
|
||||
minute := time.NewTicker(60 * time.Second)
|
||||
//records.DeleteRecordBefore(time.Now().Add(-time.Hour * 24 * 30))
|
||||
records.CompactRecord()
|
||||
cfg, _ := config.Get()
|
||||
go notifier.CheckExpireScheduledWork()
|
||||
for {
|
||||
select {
|
||||
case <-ticker.C:
|
||||
records.DeleteRecordBefore(time.Now().Add(-time.Hour * time.Duration(cfg.RecordPreserveTime)))
|
||||
records.CompactRecord()
|
||||
tasks.ClearTaskResultsByTimeBefore(time.Now().Add(-time.Hour * time.Duration(cfg.RecordPreserveTime)))
|
||||
tasks.DeletePingRecordsBefore(time.Now().Add(-time.Hour * time.Duration(cfg.PingRecordPreserveTime)))
|
||||
auditlog.RemoveOldLogs()
|
||||
case <-minute.C:
|
||||
api.SaveClientReportToDB()
|
||||
if !cfg.RecordEnabled {
|
||||
records.DeleteAll()
|
||||
tasks.DeleteAllPingRecords()
|
||||
}
|
||||
// 每分钟检查一次流量提醒
|
||||
go notifier.CheckTraffic()
|
||||
}
|
||||
if err := tasks.ReloadPingSchedule(); err != nil {
|
||||
log.Println("Failed to reload ping schedule:", err)
|
||||
}
|
||||
if err := d_notification.ReloadLoadNotificationSchedule(); err != nil {
|
||||
log.Println("Failed to reload load notification schedule:", err)
|
||||
}
|
||||
records.CompactRecord()
|
||||
|
||||
if err := corn.AddFunc("records:cleanup", "@every 30m", cleanupScheduledData); err != nil {
|
||||
log.Println("Failed to add cleanup scheduled task:", err)
|
||||
}
|
||||
if err := corn.AddFunc("records:minute", "@every 1m", minuteScheduledWork); err != nil {
|
||||
log.Println("Failed to add minute scheduled task:", err)
|
||||
}
|
||||
if err := corn.AddFunc("notifier:expire", "0 0 9 * * *", notifier.CheckExpireScheduledWork); err != nil {
|
||||
log.Println("Failed to add expire notification scheduled task:", err)
|
||||
}
|
||||
notifier.InitTrafficReportSchedule()
|
||||
}
|
||||
|
||||
func cleanupScheduledData() {
|
||||
cfg, _ := config.GetManyAs[config.Settings]()
|
||||
records.DeleteRecordBefore(time.Now().Add(-time.Hour * time.Duration(cfg.RecordPreserveTime)))
|
||||
records.CompactRecord()
|
||||
tasks.ClearTaskResultsByTimeBefore(time.Now().Add(-time.Hour * time.Duration(cfg.RecordPreserveTime)))
|
||||
tasks.DeletePingRecordsBefore(time.Now().Add(-time.Hour * time.Duration(cfg.PingRecordPreserveTime)))
|
||||
auditlog.RemoveOldLogs()
|
||||
accounts.RemoveExpiredSessions()
|
||||
}
|
||||
|
||||
func minuteScheduledWork() {
|
||||
cfg, _ := config.GetManyAs[config.Settings]()
|
||||
report_cache.SaveClientReportToDB()
|
||||
if !cfg.RecordEnabled {
|
||||
records.DeleteAll()
|
||||
tasks.DeleteAllPingRecords()
|
||||
}
|
||||
// 每分钟检查一次流量提醒
|
||||
notifier.CheckTraffic()
|
||||
}
|
||||
|
||||
func OnShutdown() {
|
||||
auditlog.Log("", "", "server is shutting down", "info")
|
||||
cloudflared.Kill()
|
||||
corn.StopAll()
|
||||
cloudflared.Shutdown()
|
||||
}
|
||||
|
||||
func OnFatal(err error) {
|
||||
auditlog.Log("", "", "server encountered a fatal error: "+err.Error(), "error")
|
||||
cloudflared.Kill()
|
||||
cloudflared.Shutdown()
|
||||
}
|
||||
@@ -1,135 +0,0 @@
|
||||
package common
|
||||
|
||||
import "time"
|
||||
|
||||
type Message struct {
|
||||
Type string `json:"type"`
|
||||
Content string `json:"content"`
|
||||
Sender string `json:"sender"`
|
||||
Timestamp int64 `json:"timestamp"`
|
||||
}
|
||||
|
||||
type ClientConfig struct {
|
||||
ClientUUID string `json:"client_uuid" gorm:"type:uuid;primaryKey;foreignKey:ClientUUID;references:UUID;constraint:OnDelete:CASCADE"`
|
||||
CPU bool `json:"cpu" gorm:"default:true"`
|
||||
GPU bool `json:"gpu" gorm:"default:true"`
|
||||
RAM bool `json:"ram" gorm:"default:true"`
|
||||
SWAP bool `json:"swap" gorm:"default:true"`
|
||||
LOAD bool `json:"load" gorm:"default:true"`
|
||||
UPTIME bool `json:"uptime" gorm:"default:true"`
|
||||
TEMP bool `json:"temp" gorm:"default:true"`
|
||||
OS bool `json:"os" gorm:"default:true"`
|
||||
DISK bool `json:"disk" gorm:"default:true"`
|
||||
NET bool `json:"net" gorm:"default:true"`
|
||||
PROCESS bool `json:"process" gorm:"default:true"`
|
||||
Connections bool `json:"connections" gorm:"default:true"`
|
||||
Interval int `json:"interval" gorm:"default:3;check:Interval >= 1"`
|
||||
CreatedAt time.Time `json:"created_at"`
|
||||
UpdatedAt time.Time `json:"updated_at"`
|
||||
}
|
||||
|
||||
// ClientInfo stores static client information
|
||||
// Deprecated: Use models.Client instead.
|
||||
type ClientInfo struct {
|
||||
UUID string `json:"uuid,omitempty" gorm:"type:varchar(36);primaryKey;foreignKey:ClientUUID;references:UUID;constraint:OnDelete:CASCADE"`
|
||||
Name string `json:"name" gorm:"type:varchar(100);not null"`
|
||||
CpuName string `json:"cpu_name" gorm:"type:varchar(100)"`
|
||||
Virtualization string `json:"virtualization" gorm:"type:varchar(50)"`
|
||||
Arch string `json:"arch" gorm:"type:varchar(50)"`
|
||||
CpuCores int `json:"cpu_cores" gorm:"type:int"`
|
||||
OS string `json:"os" gorm:"type:varchar(100)"`
|
||||
GpuName string `json:"gpu_name" gorm:"type:varchar(100)"`
|
||||
IPv4 string `json:"ipv4,omitempty" gorm:"type:varchar(100)"`
|
||||
IPv6 string `json:"ipv6,omitempty" gorm:"type:varchar(100)"`
|
||||
Region string `json:"region" gorm:"type:varchar(100)"`
|
||||
Remark string `json:"remark,omitempty" gorm:"type:longtext"`
|
||||
PublicRemark string `json:"public_remark,omitempty" gorm:"type:longtext"`
|
||||
MemTotal int64 `json:"mem_total" gorm:"type:bigint"`
|
||||
SwapTotal int64 `json:"swap_total" gorm:"type:bigint"`
|
||||
DiskTotal int64 `json:"disk_total" gorm:"type:bigint"`
|
||||
Version string `json:"version,omitempty" gorm:"type:varchar(100)"`
|
||||
Weight int `json:"weight" gorm:"type:int"`
|
||||
Price float64 `json:"price"`
|
||||
BillingCycle int `json:"billing_cycle"`
|
||||
ExpiredAt time.Time `json:"expired_at" gorm:"type:timestamp"`
|
||||
CreatedAt time.Time
|
||||
UpdatedAt time.Time
|
||||
}
|
||||
|
||||
type IPAddress struct {
|
||||
Ipv4 string `json:"ipv4"`
|
||||
Ipv6 string `json:"ipv6"`
|
||||
}
|
||||
type Report struct {
|
||||
UUID string `json:"uuid,omitempty"`
|
||||
CPU CPUReport `json:"cpu"`
|
||||
Ram RamReport `json:"ram"`
|
||||
Swap RamReport `json:"swap"`
|
||||
Load LoadReport `json:"load"`
|
||||
Disk DiskReport `json:"disk"`
|
||||
Network NetworkReport `json:"network"`
|
||||
Connections ConnectionsReport `json:"connections"`
|
||||
GPU *GPUDetailReport `json:"gpu,omitempty"` // 新增GPU详细信息
|
||||
Uptime int64 `json:"uptime"`
|
||||
Process int `json:"process"`
|
||||
Message string `json:"message"`
|
||||
Method string `json:"method,omitempty"`
|
||||
UpdatedAt time.Time `json:"updated_at"`
|
||||
}
|
||||
|
||||
type CPUReport struct {
|
||||
Name string `json:"name,omitempty"`
|
||||
Cores int `json:"cores,omitempty"`
|
||||
Arch string `json:"arch,omitempty"`
|
||||
Usage float64 `json:"usage,omitempty"`
|
||||
}
|
||||
|
||||
// GPUDetailReport 详细GPU信息报告
|
||||
type GPUDetailReport struct {
|
||||
Count int `json:"count"` // GPU数量
|
||||
AverageUsage float64 `json:"average_usage"` // 平均使用率
|
||||
DetailedInfo []GPUDeviceInfo `json:"detailed_info"` // 每个GPU的详细信息
|
||||
}
|
||||
|
||||
// GPUDeviceInfo 单个GPU设备信息
|
||||
type GPUDeviceInfo struct {
|
||||
Name string `json:"name"` // GPU型号名称
|
||||
MemoryTotal int64 `json:"memory_total"` // 显存总量(字节)
|
||||
MemoryUsed int64 `json:"memory_used"` // 显存已用(字节)
|
||||
Utilization float64 `json:"utilization"` // GPU使用率(%)
|
||||
Temperature int `json:"temperature"` // GPU温度(°C)
|
||||
}
|
||||
|
||||
// 保持向后兼容的旧GPUReport结构
|
||||
type GPUReport struct {
|
||||
Name string `json:"name,omitempty"`
|
||||
Usage float64 `json:"usage,omitempty"`
|
||||
}
|
||||
|
||||
type RamReport struct {
|
||||
Total int64 `json:"total"`
|
||||
Used int64 `json:"used"`
|
||||
}
|
||||
|
||||
type LoadReport struct {
|
||||
Load1 float64 `json:"load1"`
|
||||
Load5 float64 `json:"load5"`
|
||||
Load15 float64 `json:"load15"`
|
||||
}
|
||||
|
||||
type DiskReport struct {
|
||||
Total int64 `json:"total"`
|
||||
Used int64 `json:"used"`
|
||||
}
|
||||
|
||||
type NetworkReport struct {
|
||||
Up int64 `json:"up"`
|
||||
Down int64 `json:"down"`
|
||||
TotalUp int64 `json:"totalUp"`
|
||||
TotalDown int64 `json:"totalDown"`
|
||||
}
|
||||
|
||||
type ConnectionsReport struct {
|
||||
TCP int `json:"tcp"`
|
||||
UDP int `json:"udp"`
|
||||
}
|
||||
@@ -6,10 +6,10 @@ import (
|
||||
"net"
|
||||
"time"
|
||||
|
||||
"github.com/komari-monitor/komari/database/config"
|
||||
"github.com/komari-monitor/komari/database/dbcore"
|
||||
"github.com/komari-monitor/komari/database/models"
|
||||
messageevent "github.com/komari-monitor/komari/database/models/messageEvent"
|
||||
"github.com/komari-monitor/komari/pkg/config"
|
||||
"github.com/komari-monitor/komari/utils"
|
||||
"github.com/komari-monitor/komari/utils/geoip"
|
||||
"github.com/komari-monitor/komari/utils/messageSender"
|
||||
@@ -39,21 +39,23 @@ func CreateSession(uuid string, expires int, userAgent, ip, login_method string)
|
||||
LoginMethod: login_method,
|
||||
LatestOnline: models.FromTime(time.Now()),
|
||||
}
|
||||
cfg, _ := config.Get()
|
||||
if cfg.LoginNotification {
|
||||
ipAddr := net.ParseIP(ip)
|
||||
ipinfo, _ := geoip.GetGeoInfo(ipAddr)
|
||||
loc := "unknown"
|
||||
if ipinfo != nil {
|
||||
loc = ipinfo.Name
|
||||
go func() {
|
||||
LoginNotification, _ := config.GetAs[bool](config.LoginNotificationKey, false)
|
||||
if LoginNotification {
|
||||
ipAddr := net.ParseIP(ip)
|
||||
ipinfo, _ := geoip.GetGeoInfo(ipAddr)
|
||||
loc := "unknown"
|
||||
if ipinfo != nil && ipinfo.Name != "" {
|
||||
loc = ipinfo.Name
|
||||
}
|
||||
messageSender.SendEvent(models.EventMessage{
|
||||
Event: messageevent.Login,
|
||||
Time: time.Now(),
|
||||
Message: fmt.Sprintf("%s: %s (%s)\n%s", login_method, ip, loc, userAgent),
|
||||
Emoji: "🔑",
|
||||
})
|
||||
}
|
||||
messageSender.SendEvent(models.EventMessage{
|
||||
Event: messageevent.Login,
|
||||
Time: time.Now(),
|
||||
Message: fmt.Sprintf("%s: %s (%s)\n%s", login_method, ip, loc, userAgent),
|
||||
Emoji: "🔑",
|
||||
})
|
||||
}
|
||||
}()
|
||||
|
||||
err := db.Create(&sessionRecord).Error
|
||||
if err != nil {
|
||||
@@ -131,3 +133,12 @@ func UpdateLatest(session, useragent, ip string) error {
|
||||
"latest_ip": ip,
|
||||
}).Error
|
||||
}
|
||||
|
||||
func RemoveExpiredSessions() error {
|
||||
db := dbcore.GetDBInstance()
|
||||
result := db.Where("expires < ?", time.Now()).Delete(&models.Session{})
|
||||
if result.Error != nil {
|
||||
return result.Error
|
||||
}
|
||||
return nil
|
||||
}
|
||||
+75
-122
@@ -1,30 +1,21 @@
|
||||
package clients
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"log"
|
||||
"math"
|
||||
"time"
|
||||
|
||||
"github.com/komari-monitor/komari/common"
|
||||
"github.com/komari-monitor/komari/database/dbcore"
|
||||
"github.com/komari-monitor/komari/database/models"
|
||||
"github.com/komari-monitor/komari/database/tasks"
|
||||
"github.com/komari-monitor/komari/utils"
|
||||
"gorm.io/gorm"
|
||||
"gorm.io/gorm/clause"
|
||||
|
||||
"fmt"
|
||||
|
||||
"github.com/google/uuid"
|
||||
)
|
||||
|
||||
// Deprecated: DeleteClientConfig is deprecated and will be removed in a future release. Use DeleteClient instead.
|
||||
func DeleteClientConfig(clientUuid string) error {
|
||||
db := dbcore.GetDBInstance()
|
||||
err := db.Delete(&common.ClientConfig{ClientUUID: clientUuid}).Error
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
func DeleteClient(clientUuid string) error {
|
||||
db := dbcore.GetDBInstance()
|
||||
err := db.Delete(&models.Client{}, "uuid = ?", clientUuid).Error
|
||||
@@ -34,79 +25,6 @@ func DeleteClient(clientUuid string) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// Deprecated: UpdateOrInsertBasicInfo is deprecated and will be removed in a future release. Use SaveClientInfo instead.
|
||||
func UpdateOrInsertBasicInfo(cbi common.ClientInfo) error {
|
||||
db := dbcore.GetDBInstance()
|
||||
updates := make(map[string]interface{})
|
||||
|
||||
if cbi.Name != "" {
|
||||
updates["name"] = cbi.Name
|
||||
}
|
||||
if cbi.CpuName != "" {
|
||||
updates["cpu_name"] = cbi.CpuName
|
||||
}
|
||||
if cbi.Arch != "" {
|
||||
updates["arch"] = cbi.Arch
|
||||
}
|
||||
if cbi.CpuCores > 0 || cbi.CpuCores < math.MaxInt-1 {
|
||||
updates["cpu_cores"] = cbi.CpuCores
|
||||
}
|
||||
if cbi.OS != "" {
|
||||
updates["os"] = cbi.OS
|
||||
}
|
||||
if cbi.GpuName != "" {
|
||||
updates["gpu_name"] = cbi.GpuName
|
||||
}
|
||||
if cbi.IPv4 != "" {
|
||||
updates["ipv4"] = cbi.IPv4
|
||||
}
|
||||
if cbi.IPv6 != "" {
|
||||
updates["ipv6"] = cbi.IPv6
|
||||
}
|
||||
if cbi.Region != "" {
|
||||
updates["region"] = cbi.Region
|
||||
}
|
||||
if cbi.Remark != "" {
|
||||
updates["remark"] = cbi.Remark
|
||||
}
|
||||
updates["mem_total"] = cbi.MemTotal
|
||||
updates["swap_total"] = cbi.SwapTotal
|
||||
updates["disk_total"] = cbi.DiskTotal
|
||||
updates["version"] = cbi.Version
|
||||
updates["updated_at"] = time.Now()
|
||||
|
||||
// 转换为更新Client表
|
||||
client := models.Client{
|
||||
UUID: cbi.UUID,
|
||||
}
|
||||
|
||||
err := db.Model(&client).Clauses(clause.OnConflict{
|
||||
Columns: []clause.Column{{Name: "uuid"}},
|
||||
DoUpdates: clause.Assignments(updates),
|
||||
}).Create(map[string]interface{}{
|
||||
"uuid": cbi.UUID,
|
||||
"name": cbi.Name,
|
||||
"cpu_name": cbi.CpuName,
|
||||
"arch": cbi.Arch,
|
||||
"cpu_cores": cbi.CpuCores,
|
||||
"os": cbi.OS,
|
||||
"gpu_name": cbi.GpuName,
|
||||
"ipv4": cbi.IPv4,
|
||||
"ipv6": cbi.IPv6,
|
||||
"region": cbi.Region,
|
||||
"remark": cbi.Remark,
|
||||
"mem_total": cbi.MemTotal,
|
||||
"swap_total": cbi.SwapTotal,
|
||||
"disk_total": cbi.DiskTotal,
|
||||
"version": cbi.Version,
|
||||
"updated_at": time.Now(),
|
||||
}).Error
|
||||
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
func SaveClientInfo(update map[string]interface{}) error {
|
||||
db := dbcore.GetDBInstance()
|
||||
clientUUID, ok := update["uuid"].(string)
|
||||
@@ -121,27 +39,73 @@ func SaveClientInfo(update map[string]interface{}) error {
|
||||
|
||||
update["updated_at"] = time.Now()
|
||||
|
||||
checkInt64 := func(name string, val float64) error {
|
||||
if val < 0 {
|
||||
return fmt.Errorf("%s must be non-negative, got %d", name, int64(val))
|
||||
toFloat64 := func(value interface{}) (float64, bool) {
|
||||
switch typed := value.(type) {
|
||||
case float64:
|
||||
return typed, true
|
||||
case float32:
|
||||
return float64(typed), true
|
||||
case int:
|
||||
return float64(typed), true
|
||||
case int8:
|
||||
return float64(typed), true
|
||||
case int16:
|
||||
return float64(typed), true
|
||||
case int32:
|
||||
return float64(typed), true
|
||||
case int64:
|
||||
return float64(typed), true
|
||||
case uint:
|
||||
return float64(typed), true
|
||||
case uint8:
|
||||
return float64(typed), true
|
||||
case uint16:
|
||||
return float64(typed), true
|
||||
case uint32:
|
||||
return float64(typed), true
|
||||
case uint64:
|
||||
return float64(typed), true
|
||||
case json.Number:
|
||||
parsed, err := typed.Float64()
|
||||
if err != nil {
|
||||
return 0, false
|
||||
}
|
||||
return parsed, true
|
||||
default:
|
||||
return 0, false
|
||||
}
|
||||
if val > math.MaxInt64-1 {
|
||||
return fmt.Errorf("%s exceeds int64 max limit: %d", name, int64(val))
|
||||
}
|
||||
|
||||
checkOptionalInt := func(name, key string, maxValue float64) error {
|
||||
value, exists := update[key]
|
||||
if !exists || value == nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
numericValue, ok := toFloat64(value)
|
||||
if !ok {
|
||||
return fmt.Errorf("%s must be a valid number", name)
|
||||
}
|
||||
if numericValue < 0 || numericValue > maxValue {
|
||||
return fmt.Errorf("%s must be a valid non-negative number: %v", name, value)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
verify := func(update map[string]interface{}) error {
|
||||
if update["cpu_cores"].(float64) < 0 || update["cpu_cores"].(float64) > math.MaxInt-1 {
|
||||
return fmt.Errorf("Cpu.Cores be not a valid int64 number: %d", update["cpu_cores"])
|
||||
}
|
||||
if err := checkInt64("Ram.Total", update["mem_total"].(float64)); err != nil {
|
||||
if err := checkOptionalInt("Cpu.Cores", "cpu_cores", math.MaxInt-1); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := checkInt64("Swap.Total", update["swap_total"].(float64)); err != nil {
|
||||
if err := checkOptionalInt("Cpu.PhysicalCores", "cpu_physical_cores", math.MaxInt-1); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := checkInt64("Disk.Total", update["disk_total"].(float64)); err != nil {
|
||||
if err := checkOptionalInt("Ram.Total", "mem_total", math.MaxInt64-1); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := checkOptionalInt("Swap.Total", "swap_total", math.MaxInt64-1); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := checkOptionalInt("Disk.Total", "disk_total", math.MaxInt64-1); err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
@@ -158,16 +122,6 @@ func SaveClientInfo(update map[string]interface{}) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// 更新客户端设置
|
||||
func UpdateClientConfig(config common.ClientConfig) error {
|
||||
db := dbcore.GetDBInstance()
|
||||
err := db.Save(&config).Error
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func EditClientName(clientUUID, clientName string) error {
|
||||
db := dbcore.GetDBInstance()
|
||||
err := db.Model(&models.Client{}).Where("uuid = ?", clientUUID).Update("name", clientName).Error
|
||||
@@ -177,21 +131,6 @@ func EditClientName(clientUUID, clientName string) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
/*
|
||||
// UpdateClientByUUID 更新指定 UUID 的客户端配置
|
||||
|
||||
func UpdateClientByUUID(config common.ClientConfig) error {
|
||||
db := dbcore.GetDBInstance()
|
||||
result := db.Model(&common.ClientConfig{}).Where("client_uuid = ?", config.ClientUUID).Updates(config)
|
||||
if result.Error != nil {
|
||||
return result.Error
|
||||
}
|
||||
if result.RowsAffected == 0 {
|
||||
return gorm.ErrRecordNotFound
|
||||
}
|
||||
return nil
|
||||
}
|
||||
*/
|
||||
func EditClientToken(clientUUID, token string) error {
|
||||
db := dbcore.GetDBInstance()
|
||||
err := db.Model(&models.Client{}).Where("uuid = ?", clientUUID).Update("token", token).Error
|
||||
@@ -219,6 +158,9 @@ func CreateClient() (clientUUID, token string, err error) {
|
||||
if err != nil {
|
||||
return "", "", err
|
||||
}
|
||||
if err := tasks.AddDefaultOnClientUUID(clientUUID); err != nil {
|
||||
log.Println("Failed to apply default-on ping tasks to new client:", err)
|
||||
}
|
||||
return clientUUID, token, nil
|
||||
}
|
||||
|
||||
@@ -241,6 +183,9 @@ func CreateClientWithName(name string) (clientUUID, token string, err error) {
|
||||
if err != nil {
|
||||
return "", "", err
|
||||
}
|
||||
if err := tasks.AddDefaultOnClientUUID(clientUUID); err != nil {
|
||||
log.Println("Failed to apply default-on ping tasks to new client:", err)
|
||||
}
|
||||
return clientUUID, token, nil
|
||||
}
|
||||
|
||||
@@ -309,6 +254,14 @@ func SaveClient(updates map[string]interface{}) error {
|
||||
return fmt.Errorf("no fields to update")
|
||||
}
|
||||
|
||||
if v, exists := updates["traffic_limit"]; exists {
|
||||
if val, ok := v.(float64); ok {
|
||||
if val < 0 || val > math.MaxInt64-1 {
|
||||
return fmt.Errorf("traffic_limit must be a valid non-negative int64 value, got %v", val)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
updates["updated_at"] = time.Now()
|
||||
|
||||
err := db.Model(&models.Client{}).Where("uuid = ?", clientUUID).Updates(updates).Error
|
||||
|
||||
@@ -2,17 +2,48 @@ package clients
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"math"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/komari-monitor/komari/common"
|
||||
"github.com/komari-monitor/komari/database/dbcore"
|
||||
"github.com/komari-monitor/komari/database/models"
|
||||
v1 "github.com/komari-monitor/komari/protocol/v1"
|
||||
"github.com/komari-monitor/komari/utils"
|
||||
|
||||
"gorm.io/gorm"
|
||||
)
|
||||
|
||||
var reportSaveLocks sync.Map
|
||||
|
||||
func getReportSaveLock(clientUUID string) *sync.Mutex {
|
||||
lock, _ := reportSaveLocks.LoadOrStore(clientUUID, &sync.Mutex{})
|
||||
return lock.(*sync.Mutex)
|
||||
}
|
||||
|
||||
func getLatestTrafficRecord(tx *gorm.DB, clientUUID string) (*models.Record, error) {
|
||||
var record models.Record
|
||||
err := tx.Where("client = ?", clientUUID).Order("time DESC").First(&record).Error
|
||||
if err == nil {
|
||||
return &record, nil
|
||||
}
|
||||
if !errors.Is(err, gorm.ErrRecordNotFound) {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
err = tx.Table("records_long_term").Where("client = ?", clientUUID).Order("time DESC").First(&record).Error
|
||||
if err == nil {
|
||||
return &record, nil
|
||||
}
|
||||
if !errors.Is(err, gorm.ErrRecordNotFound) {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
// Report 表示客户端报告数据
|
||||
// SaveReport 保存报告数据
|
||||
func SaveReport(uuid string, data map[string]interface{}) (err error) {
|
||||
@@ -40,20 +71,20 @@ func GetClientUUIDByToken(token string) (clientUUID string, err error) {
|
||||
return client.UUID, nil
|
||||
}
|
||||
|
||||
func ParseReport(data map[string]interface{}) (report common.Report, err error) {
|
||||
func ParseReport(data map[string]interface{}) (report v1.Report, err error) {
|
||||
jsonData, err := json.Marshal(data)
|
||||
if err != nil {
|
||||
return common.Report{}, err
|
||||
return v1.Report{}, err
|
||||
}
|
||||
err = json.Unmarshal(jsonData, &report)
|
||||
if err != nil {
|
||||
return common.Report{}, err
|
||||
return v1.Report{}, err
|
||||
}
|
||||
return report, nil
|
||||
}
|
||||
|
||||
// 检查数据防止异常数据导致数据库损坏
|
||||
func ReportVerify(report common.Report) error {
|
||||
func ReportVerify(report v1.Report) error {
|
||||
// 防止输入不合理范围
|
||||
if report.CPU.Usage < 0 || report.CPU.Usage > 100 {
|
||||
return fmt.Errorf("CPU.Usage must be between 0 and 100")
|
||||
@@ -137,11 +168,14 @@ func ReportVerify(report common.Report) error {
|
||||
}
|
||||
|
||||
// SaveClientReport 保存客户端报告到 Record 表
|
||||
func SaveClientReport(clientUUID string, report common.Report) (err error) {
|
||||
func SaveClientReport(clientUUID string, report v1.Report) (err error) {
|
||||
db := dbcore.GetDBInstance()
|
||||
lock := getReportSaveLock(clientUUID)
|
||||
lock.Lock()
|
||||
defer lock.Unlock()
|
||||
|
||||
if err := ReportVerify(report); err != nil {
|
||||
return fmt.Errorf("failed to save Record: %v", err)
|
||||
return fmt.Errorf("failed to save Record: %w", err)
|
||||
}
|
||||
|
||||
// 保存GPU详细记录到独立表
|
||||
@@ -159,7 +193,7 @@ func SaveClientReport(clientUUID string, report common.Report) (err error) {
|
||||
Temperature: gpu.Temperature,
|
||||
}
|
||||
if err := db.Create(&gpuRecord).Error; err != nil {
|
||||
return fmt.Errorf("failed to save GPU record: %v", err)
|
||||
return fmt.Errorf("failed to save GPU record: %w", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -187,6 +221,8 @@ func SaveClientReport(clientUUID string, report common.Report) (err error) {
|
||||
NetOut: report.Network.Up,
|
||||
NetTotalUp: report.Network.TotalUp,
|
||||
NetTotalDown: report.Network.TotalDown,
|
||||
TrafficUp: 0,
|
||||
TrafficDown: 0,
|
||||
Process: report.Process,
|
||||
Connections: report.Connections.TCP,
|
||||
ConnectionsUdp: report.Connections.UDP,
|
||||
@@ -195,9 +231,18 @@ func SaveClientReport(clientUUID string, report common.Report) (err error) {
|
||||
|
||||
// 使用事务确保 Record 和 ClientsInfo 一致性
|
||||
err = db.Transaction(func(tx *gorm.DB) error {
|
||||
previous, err := getLatestTrafficRecord(tx, clientUUID)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to load previous Record: %w", err)
|
||||
}
|
||||
if previous != nil {
|
||||
Record.TrafficUp = utils.ComputeTrafficDelta(report.Network.TotalUp, previous.NetTotalUp)
|
||||
Record.TrafficDown = utils.ComputeTrafficDelta(report.Network.TotalDown, previous.NetTotalDown)
|
||||
}
|
||||
|
||||
// 保存 Record
|
||||
if err := tx.Create(&Record).Error; err != nil {
|
||||
return fmt.Errorf("failed to save Record: %v", err)
|
||||
return fmt.Errorf("failed to save Record: %w", err)
|
||||
}
|
||||
return nil
|
||||
})
|
||||
|
||||
@@ -0,0 +1,83 @@
|
||||
package clients
|
||||
|
||||
import (
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/komari-monitor/komari/utils"
|
||||
"github.com/stretchr/testify/assert"
|
||||
)
|
||||
|
||||
func TestComputeTrafficDeltaHandlesZeroAndReset(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
current int64
|
||||
previous int64
|
||||
want int64
|
||||
}{
|
||||
{name: "previous zero counts current delta", current: 120, previous: 0, want: 120},
|
||||
{name: "monotonic counter uses difference", current: 250, previous: 200, want: 50},
|
||||
{name: "counter reset uses current", current: 15, previous: 250, want: 15},
|
||||
{name: "negative previous remains guarded", current: 15, previous: -1, want: 0},
|
||||
}
|
||||
|
||||
for _, test := range tests {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
assert.Equal(t, test.want, utils.ComputeTrafficDelta(test.current, test.previous))
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestReportSaveLockSerializesSameClient(t *testing.T) {
|
||||
clientUUID := "client-lock-same"
|
||||
lock := getReportSaveLock(clientUUID)
|
||||
lock.Lock()
|
||||
defer lock.Unlock()
|
||||
|
||||
acquired := make(chan struct{})
|
||||
go func() {
|
||||
secondLock := getReportSaveLock(clientUUID)
|
||||
secondLock.Lock()
|
||||
defer secondLock.Unlock()
|
||||
close(acquired)
|
||||
}()
|
||||
|
||||
select {
|
||||
case <-acquired:
|
||||
t.Fatal("same-client report save lock was acquired while already held")
|
||||
case <-time.After(20 * time.Millisecond):
|
||||
}
|
||||
|
||||
lock.Unlock()
|
||||
select {
|
||||
case <-acquired:
|
||||
case <-time.After(time.Second):
|
||||
t.Fatal("same-client report save lock was not released")
|
||||
}
|
||||
lock.Lock()
|
||||
}
|
||||
|
||||
func TestReportSaveLockAllowsDifferentClients(t *testing.T) {
|
||||
firstLock := getReportSaveLock("client-lock-a")
|
||||
firstLock.Lock()
|
||||
defer firstLock.Unlock()
|
||||
|
||||
var waitGroup sync.WaitGroup
|
||||
waitGroup.Add(1)
|
||||
acquired := make(chan struct{})
|
||||
go func() {
|
||||
defer waitGroup.Done()
|
||||
secondLock := getReportSaveLock("client-lock-b")
|
||||
secondLock.Lock()
|
||||
defer secondLock.Unlock()
|
||||
close(acquired)
|
||||
}()
|
||||
|
||||
select {
|
||||
case <-acquired:
|
||||
case <-time.After(time.Second):
|
||||
t.Fatal("different-client report save lock should not block")
|
||||
}
|
||||
waitGroup.Wait()
|
||||
}
|
||||
@@ -1,113 +0,0 @@
|
||||
package config
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"log"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/komari-monitor/komari/database/dbcore"
|
||||
"github.com/komari-monitor/komari/database/models"
|
||||
"gorm.io/gorm"
|
||||
)
|
||||
|
||||
var mu sync.Mutex
|
||||
|
||||
func init() {
|
||||
mu = sync.Mutex{}
|
||||
}
|
||||
|
||||
func Get() (models.Config, error) {
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
db := dbcore.GetDBInstance()
|
||||
var config models.Config
|
||||
if err := db.First(&config).Error; err != nil {
|
||||
if err == gorm.ErrRecordNotFound {
|
||||
config = models.Config{
|
||||
ID: 1,
|
||||
Sitename: "Komari",
|
||||
Description: "Komari Monitor, a simple server monitoring tool.",
|
||||
AllowCors: false,
|
||||
OAuthEnabled: false,
|
||||
GeoIpEnabled: true,
|
||||
GeoIpProvider: "ipinfo",
|
||||
NezhaCompatEnabled: false,
|
||||
NezhaCompatListen: "",
|
||||
NotificationTemplate: "{{emoji}}{{emoji}}{{emoji}}\nEvent: {{event}}\nClients: {{client}}\nMessage: {{message}}\nTime: {{time}}",
|
||||
UpdatedAt: models.FromTime(time.Now()),
|
||||
CreatedAt: models.FromTime(time.Now()),
|
||||
}
|
||||
if err := db.Create(&config).Error; err != nil {
|
||||
log.Fatal("Failed to create default config:", err)
|
||||
}
|
||||
return config, nil
|
||||
}
|
||||
return config, err
|
||||
}
|
||||
return config, nil
|
||||
}
|
||||
|
||||
func Save(cst models.Config) error {
|
||||
db := dbcore.GetDBInstance()
|
||||
oldConfig, _ := Get()
|
||||
// Only one records
|
||||
cst.ID = 1
|
||||
cst.UpdatedAt = models.FromTime(time.Now())
|
||||
if err := db.Model(&models.Config{}).Where("id = ?", cst.ID).
|
||||
Select("*").
|
||||
Updates(cst).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
newConfig, _ := Get()
|
||||
publishEvent(oldConfig, newConfig)
|
||||
return nil
|
||||
}
|
||||
|
||||
func Update(cst map[string]interface{}) error {
|
||||
db := dbcore.GetDBInstance()
|
||||
oldConfig, _ := Get()
|
||||
// Proceed with update
|
||||
cst["id"] = 1
|
||||
cst["updated_at"] = time.Now()
|
||||
delete(cst, "created_at")
|
||||
delete(cst, "CreatedAt")
|
||||
|
||||
// 至少有一种登录方式启用
|
||||
newDisablePasswordLogin := oldConfig.DisablePasswordLogin
|
||||
newOAuthEnabled := oldConfig.OAuthEnabled
|
||||
if val, exists := cst["disable_password_login"]; exists {
|
||||
newDisablePasswordLogin = val.(bool)
|
||||
}
|
||||
if val, exists := cst["o_auth_enabled"]; exists {
|
||||
newOAuthEnabled = val.(bool)
|
||||
}
|
||||
if newDisablePasswordLogin && !newOAuthEnabled {
|
||||
return errors.New("at least one login method must be enabled (password/oauth)")
|
||||
}
|
||||
// 没绑定账号也不能禁用
|
||||
if newDisablePasswordLogin {
|
||||
usr := &models.User{}
|
||||
if err := db.Model(&models.User{}).First(usr).Error; err != nil {
|
||||
return errors.Join(err, errors.New("failed to retrieve user"))
|
||||
}
|
||||
if usr.SSOID == "" {
|
||||
return errors.New("cannot disable password login when no SSO-bound account exists")
|
||||
}
|
||||
}
|
||||
err := db.Transaction(func(tx *gorm.DB) error {
|
||||
if err := tx.Model(&models.Config{}).Where("id = ?", oldConfig.ID).Updates(cst).Error; err != nil {
|
||||
return errors.Join(err, errors.New("failed to update configuration"))
|
||||
}
|
||||
newConfig := &models.Config{}
|
||||
if err := tx.Where("id = ?", oldConfig.ID).First(newConfig).Error; err != nil {
|
||||
return errors.Join(err, errors.New("failed to retrieve updated configuration"))
|
||||
}
|
||||
publishEvent(oldConfig, *newConfig)
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -1,38 +0,0 @@
|
||||
package config
|
||||
|
||||
import (
|
||||
"sync"
|
||||
|
||||
"github.com/komari-monitor/komari/database/models"
|
||||
)
|
||||
|
||||
// ConfigEvent represents a configuration change event
|
||||
type ConfigEvent struct {
|
||||
Old models.Config
|
||||
New models.Config
|
||||
}
|
||||
|
||||
// ConfigSubscriber handles config events
|
||||
type ConfigSubscriber func(event ConfigEvent)
|
||||
|
||||
var (
|
||||
subscribersMu sync.RWMutex
|
||||
subscribers []ConfigSubscriber
|
||||
)
|
||||
|
||||
// Subscribe registers a subscriber for all config events.
|
||||
func Subscribe(subscriber ConfigSubscriber) {
|
||||
subscribersMu.Lock()
|
||||
defer subscribersMu.Unlock()
|
||||
subscribers = append(subscribers, subscriber)
|
||||
}
|
||||
|
||||
// publishEvent notifies all subscribers of a config change.
|
||||
func publishEvent(oldVal, newVal models.Config) {
|
||||
subscribersMu.RLock()
|
||||
defer subscribersMu.RUnlock()
|
||||
event := ConfigEvent{Old: oldVal, New: newVal}
|
||||
for _, sub := range subscribers {
|
||||
go sub(event)
|
||||
}
|
||||
}
|
||||
+206
-198
@@ -1,206 +1,168 @@
|
||||
package dbcore
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"archive/zip"
|
||||
"fmt"
|
||||
"io"
|
||||
"log"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/komari-monitor/komari/cmd/flags"
|
||||
"github.com/komari-monitor/komari/common"
|
||||
"github.com/komari-monitor/komari/database/models"
|
||||
"github.com/komari-monitor/komari/utils"
|
||||
"gorm.io/driver/mysql"
|
||||
"github.com/komari-monitor/komari/pkg/config"
|
||||
"github.com/komari-monitor/komari/pkg/migrations"
|
||||
logutil "github.com/komari-monitor/komari/utils/log"
|
||||
"gorm.io/driver/sqlite"
|
||||
"gorm.io/gorm"
|
||||
"gorm.io/gorm/logger"
|
||||
)
|
||||
|
||||
// mergeClientInfo 将旧版ClientInfo数据迁移到新版Client表
|
||||
func mergeClientInfo(db *gorm.DB) {
|
||||
var clientInfos []common.ClientInfo
|
||||
if err := db.Find(&clientInfos).Error; err != nil {
|
||||
log.Printf("Failed to read ClientInfo table: %v", err)
|
||||
return
|
||||
// zipDirectoryExcluding 将 srcDir 打包为 dstZip,exclude 是绝对路径集合需要排除
|
||||
func zipDirectoryExcluding(srcDir, dstZip string, exclude map[string]struct{}) error {
|
||||
// 规范化排除路径为绝对路径
|
||||
normExclude := make(map[string]struct{}, len(exclude))
|
||||
for p := range exclude {
|
||||
abs, _ := filepath.Abs(p)
|
||||
normExclude[abs] = struct{}{}
|
||||
}
|
||||
|
||||
for _, info := range clientInfos {
|
||||
var client models.Client
|
||||
if err := db.Where("uuid = ?", info.UUID).First(&client).Error; err != nil {
|
||||
log.Printf("Could not find Client record with UUID %s: %v", info.UUID, err)
|
||||
continue
|
||||
out, err := os.Create(dstZip)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer out.Close()
|
||||
|
||||
zw := zip.NewWriter(out)
|
||||
defer zw.Close()
|
||||
|
||||
absSrc, _ := filepath.Abs(srcDir)
|
||||
walkErr := filepath.Walk(absSrc, func(path string, info os.FileInfo, err error) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// 更新Client记录
|
||||
client.Name = info.Name
|
||||
client.CpuName = info.CpuName
|
||||
client.Virtualization = info.Virtualization
|
||||
client.Arch = info.Arch
|
||||
client.CpuCores = info.CpuCores
|
||||
client.OS = info.OS
|
||||
client.GpuName = info.GpuName
|
||||
client.IPv4 = info.IPv4
|
||||
client.IPv6 = info.IPv6
|
||||
client.Region = info.Region
|
||||
client.Remark = info.Remark
|
||||
client.PublicRemark = info.PublicRemark
|
||||
client.MemTotal = info.MemTotal
|
||||
client.SwapTotal = info.SwapTotal
|
||||
client.DiskTotal = info.DiskTotal
|
||||
client.Version = info.Version
|
||||
client.Weight = info.Weight
|
||||
client.Price = info.Price
|
||||
client.BillingCycle = info.BillingCycle
|
||||
client.ExpiredAt = models.FromTime(info.ExpiredAt)
|
||||
// Save updated Client record
|
||||
if err := db.Save(&client).Error; err != nil {
|
||||
log.Printf("Failed to update Client record: %v", err)
|
||||
continue
|
||||
// 排除 backup.zip 本身
|
||||
if _, ok := normExclude[path]; ok {
|
||||
if info.IsDir() {
|
||||
return filepath.SkipDir
|
||||
}
|
||||
return nil
|
||||
}
|
||||
}
|
||||
// 计算 zip 内相对路径
|
||||
rel, err := filepath.Rel(absSrc, path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
// 根目录跳过
|
||||
if rel == "." {
|
||||
return nil
|
||||
}
|
||||
// 替换为正斜杠
|
||||
zipName := filepath.ToSlash(rel)
|
||||
|
||||
// Backup and rename old table after migration
|
||||
if err := db.Migrator().RenameTable("client_infos", "client_infos_backup"); err != nil {
|
||||
log.Printf("Failed to backup ClientInfo table: %v", err)
|
||||
return
|
||||
if info.IsDir() {
|
||||
_, err := zw.Create(zipName + "/")
|
||||
return err
|
||||
}
|
||||
// 普通文件
|
||||
fh, err := os.Open(path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
w, err := zw.Create(zipName)
|
||||
if err != nil {
|
||||
fh.Close()
|
||||
return err
|
||||
}
|
||||
if _, err := io.Copy(w, fh); err != nil {
|
||||
fh.Close()
|
||||
return err
|
||||
}
|
||||
fh.Close()
|
||||
return nil
|
||||
})
|
||||
if walkErr != nil {
|
||||
return walkErr
|
||||
}
|
||||
log.Println("Data migration completed, old table has been backed up as client_infos_backup")
|
||||
return zw.Close()
|
||||
}
|
||||
|
||||
func MergeDatabase(db *gorm.DB) {
|
||||
if db.Migrator().HasTable("client_infos") {
|
||||
log.Println("[>0.0.5] Legacy ClientInfo table detected, starting data migration...")
|
||||
mergeClientInfo(db)
|
||||
// removeAllInDirExcept 删除 dir 下除 exclude 指定绝对路径外的所有文件和文件夹
|
||||
func removeAllInDirExcept(dir string, exclude map[string]struct{}) error {
|
||||
absDir, err := filepath.Abs(dir)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if db.Migrator().HasColumn(&models.Config{}, "allow_cros") {
|
||||
log.Println("[>0.0.5a] Renaming column 'allow_cros' to 'allow_cors' in config table...")
|
||||
db.Migrator().RenameColumn(&models.Config{}, "allow_cros", "allow_cors")
|
||||
normExclude := make(map[string]struct{}, len(exclude))
|
||||
for p := range exclude {
|
||||
abs, _ := filepath.Abs(p)
|
||||
normExclude[abs] = struct{}{}
|
||||
}
|
||||
if db.Migrator().HasColumn(&models.LoadNotification{}, "client") {
|
||||
log.Println("[>0.1.4] Rebuilding LoadNotification table....")
|
||||
db.Migrator().DropTable(&models.LoadNotification{})
|
||||
entries, err := os.ReadDir(absDir)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !db.Migrator().HasTable(&models.OidcProvider{}) && db.Migrator().HasTable(&models.Config{}) {
|
||||
log.Println("[>1.0.2] Merge OidcProvider table....")
|
||||
var config struct {
|
||||
OAuthClientID string `json:"o_auth_client_id" gorm:"type:varchar(255)"`
|
||||
OAuthClientSecret string `json:"o_auth_client_secret" gorm:"type:varchar(255)"`
|
||||
for _, e := range entries {
|
||||
full := filepath.Join(absDir, e.Name())
|
||||
if _, ok := normExclude[full]; ok {
|
||||
continue
|
||||
}
|
||||
if err := db.Raw("SELECT * FROM configs LIMIT 1").Scan(&config).Error; err != nil {
|
||||
log.Println("Failed to get config for OIDC provider migration:", err)
|
||||
if err := os.RemoveAll(full); err != nil {
|
||||
return err
|
||||
}
|
||||
db.AutoMigrate(&models.OidcProvider{})
|
||||
j, err := json.Marshal(&map[string]string{
|
||||
"client_id": config.OAuthClientID,
|
||||
"client_secret": config.OAuthClientSecret,
|
||||
})
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// unzipToDir 将 zipPath 解压到 dstDir,包含路径遍历保护
|
||||
func unzipToDir(zipPath, dstDir string) error {
|
||||
zr, err := zip.OpenReader(zipPath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer zr.Close()
|
||||
|
||||
if err := os.MkdirAll(dstDir, 0755); err != nil {
|
||||
return err
|
||||
}
|
||||
absDst, _ := filepath.Abs(dstDir)
|
||||
|
||||
for _, f := range zr.File {
|
||||
// 构造目标路径并做路径遍历保护
|
||||
cleanName := filepath.Clean(f.Name)
|
||||
targetPath := filepath.Join(absDst, cleanName)
|
||||
if !strings.HasPrefix(targetPath, absDst+string(os.PathSeparator)) && targetPath != absDst {
|
||||
return fmt.Errorf("illegal file path in zip: %s", f.Name)
|
||||
}
|
||||
if f.FileInfo().IsDir() {
|
||||
if err := os.MkdirAll(targetPath, 0755); err != nil {
|
||||
return err
|
||||
}
|
||||
continue
|
||||
}
|
||||
if err := os.MkdirAll(filepath.Dir(targetPath), 0755); err != nil {
|
||||
return err
|
||||
}
|
||||
rc, err := f.Open()
|
||||
if err != nil {
|
||||
log.Println("Failed to marshal OIDC provider config:", err)
|
||||
return
|
||||
}
|
||||
db.Save(&models.OidcProvider{
|
||||
Name: "github",
|
||||
Addition: string(j),
|
||||
})
|
||||
db.AutoMigrate(&models.Config{})
|
||||
db.Model(&models.Config{}).Where("id = 1").Update("o_auth_provider", "github")
|
||||
}
|
||||
if !db.Migrator().HasTable(&models.MessageSenderProvider{}) && db.Migrator().HasTable(&models.Config{}) {
|
||||
log.Println("[>1.0.2] Migrate MessageSender configuration....")
|
||||
var config struct {
|
||||
TelegramBotToken string `json:"telegram_bot_token" gorm:"type:varchar(255)"`
|
||||
TelegramChatID string `json:"telegram_chat_id" gorm:"type:varchar(255)"`
|
||||
TelegramEndpoint string `json:"telegram_endpoint" gorm:"type:varchar(255)"`
|
||||
EmailHost string `json:"email_host" gorm:"type:varchar(255)"`
|
||||
EmailPort int `json:"email_port" gorm:"type:int"`
|
||||
EmailUsername string `json:"email_username" gorm:"type:varchar(255)"`
|
||||
EmailPassword string `json:"email_password" gorm:"type:varchar(255)"`
|
||||
EmailSender string `json:"email_sender" gorm:"type:varchar(255)"`
|
||||
EmailReceiver string `json:"email_receiver" gorm:"type:varchar(255)"`
|
||||
EmailUseSSL bool `json:"email_use_ssl" gorm:"type:boolean"`
|
||||
NotificationMethod string `json:"notification_method" gorm:"type:varchar(50)"`
|
||||
}
|
||||
if err := db.Raw("SELECT * FROM configs LIMIT 1").Scan(&config).Error; err != nil {
|
||||
log.Println("Failed to get config for MessageSender migration:", err)
|
||||
}
|
||||
|
||||
db.AutoMigrate(&models.MessageSenderProvider{})
|
||||
|
||||
// 迁移Telegram配置
|
||||
if config.NotificationMethod == "telegram" && config.TelegramBotToken != "" {
|
||||
telegramConfig := map[string]interface{}{
|
||||
"bot_token": config.TelegramBotToken,
|
||||
"chat_id": config.TelegramChatID,
|
||||
"endpoint": config.TelegramEndpoint,
|
||||
}
|
||||
if telegramConfig["endpoint"] == "" {
|
||||
telegramConfig["endpoint"] = "https://api.telegram.org/bot"
|
||||
}
|
||||
telegramConfigJSON, err := json.Marshal(telegramConfig)
|
||||
if err != nil {
|
||||
log.Println("Failed to marshal Telegram config:", err)
|
||||
} else {
|
||||
db.Save(&models.MessageSenderProvider{
|
||||
Name: "telegram",
|
||||
Addition: string(telegramConfigJSON),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// 迁移Email配置
|
||||
if config.NotificationMethod == "email" && config.EmailHost != "" {
|
||||
emailConfig := map[string]interface{}{
|
||||
"host": config.EmailHost,
|
||||
"port": config.EmailPort,
|
||||
"username": config.EmailUsername,
|
||||
"password": config.EmailPassword,
|
||||
"sender": config.EmailSender,
|
||||
"receiver": config.EmailReceiver,
|
||||
"use_ssl": config.EmailUseSSL,
|
||||
}
|
||||
emailConfigJSON, err := json.Marshal(emailConfig)
|
||||
if err != nil {
|
||||
log.Println("Failed to marshal Email config:", err)
|
||||
} else {
|
||||
db.Save(&models.MessageSenderProvider{
|
||||
Name: "email",
|
||||
Addition: string(emailConfigJSON),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// 删除旧的配置字段
|
||||
if db.Migrator().HasColumn(&models.Config{}, "telegram_bot_token") {
|
||||
db.Migrator().DropColumn(&models.Config{}, "telegram_bot_token")
|
||||
}
|
||||
if db.Migrator().HasColumn(&models.Config{}, "telegram_chat_id") {
|
||||
db.Migrator().DropColumn(&models.Config{}, "telegram_chat_id")
|
||||
}
|
||||
if db.Migrator().HasColumn(&models.Config{}, "telegram_endpoint") {
|
||||
db.Migrator().DropColumn(&models.Config{}, "telegram_endpoint")
|
||||
}
|
||||
if db.Migrator().HasColumn(&models.Config{}, "email_host") {
|
||||
db.Migrator().DropColumn(&models.Config{}, "email_host")
|
||||
}
|
||||
if db.Migrator().HasColumn(&models.Config{}, "email_port") {
|
||||
db.Migrator().DropColumn(&models.Config{}, "email_port")
|
||||
}
|
||||
if db.Migrator().HasColumn(&models.Config{}, "email_username") {
|
||||
db.Migrator().DropColumn(&models.Config{}, "email_username")
|
||||
}
|
||||
if db.Migrator().HasColumn(&models.Config{}, "email_password") {
|
||||
db.Migrator().DropColumn(&models.Config{}, "email_password")
|
||||
}
|
||||
if db.Migrator().HasColumn(&models.Config{}, "email_sender") {
|
||||
db.Migrator().DropColumn(&models.Config{}, "email_sender")
|
||||
}
|
||||
if db.Migrator().HasColumn(&models.Config{}, "email_receiver") {
|
||||
db.Migrator().DropColumn(&models.Config{}, "email_receiver")
|
||||
}
|
||||
if db.Migrator().HasColumn(&models.Config{}, "email_use_ssl") {
|
||||
db.Migrator().DropColumn(&models.Config{}, "email_use_ssl")
|
||||
return err
|
||||
}
|
||||
out, err := os.Create(targetPath)
|
||||
if err != nil {
|
||||
rc.Close()
|
||||
return err
|
||||
}
|
||||
if _, err := io.Copy(out, rc); err != nil {
|
||||
out.Close()
|
||||
rc.Close()
|
||||
return err
|
||||
}
|
||||
out.Close()
|
||||
rc.Close()
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
var (
|
||||
@@ -210,56 +172,92 @@ var (
|
||||
|
||||
func GetDBInstance() *gorm.DB {
|
||||
once.Do(func() {
|
||||
|
||||
var err error
|
||||
|
||||
logConfig := &gorm.Config{
|
||||
Logger: logger.Default.LogMode(logger.Silent),
|
||||
}
|
||||
// 在数据库初始化前执行:如果存在 ./data/backup.zip,则进行恢复逻辑
|
||||
func() {
|
||||
backupZipPath := filepath.Join(".", "data", "backup.zip")
|
||||
if _, statErr := os.Stat(backupZipPath); statErr == nil {
|
||||
// 4. 把除了 ./data/backup.zip 之外的所有文件压缩到 ./backup/{time}.zip
|
||||
if err := os.MkdirAll("./backup", 0755); err != nil {
|
||||
log.Printf("[restore] failed to create backup dir: %v", err)
|
||||
} else {
|
||||
tsName := time.Now().Format("20060102-150405")
|
||||
bakPath := filepath.Join("./backup", fmt.Sprintf("%s.zip", tsName))
|
||||
if zipErr := zipDirectoryExcluding("./data", bakPath, map[string]struct{}{backupZipPath: {}}); zipErr != nil {
|
||||
log.Printf("[restore] failed to zip current data: %v", zipErr)
|
||||
} else {
|
||||
log.Printf("[restore] current data zipped to %s", bakPath)
|
||||
}
|
||||
}
|
||||
|
||||
if utils.VersionHash == "unknown" {
|
||||
logConfig = &gorm.Config{
|
||||
Logger: logger.Default.LogMode(logger.Info),
|
||||
// 5. 删除除了 ./data/backup.zip 之外的所有文件
|
||||
if delErr := removeAllInDirExcept("./data", map[string]struct{}{backupZipPath: {}}); delErr != nil {
|
||||
log.Printf("[restore] failed to cleanup data dir: %v", delErr)
|
||||
}
|
||||
|
||||
// 6. 解压 ./data/backup.zip 到 ./data
|
||||
if unzipErr := unzipToDir(backupZipPath, "./data"); unzipErr != nil {
|
||||
log.Printf("[restore] failed to unzip backup into data: %v", unzipErr)
|
||||
} else {
|
||||
log.Printf("[restore] backup.zip extracted to ./data")
|
||||
}
|
||||
|
||||
// 7. 删除 ./data/backup.zip
|
||||
if rmErr := os.Remove(backupZipPath); rmErr != nil {
|
||||
log.Printf("[restore] failed to remove backup.zip: %v", rmErr)
|
||||
} else {
|
||||
log.Printf("[restore] backup.zip removed")
|
||||
}
|
||||
// 8. 删除标记
|
||||
if rmErr := os.Remove("./data/komari-backup-markup"); rmErr != nil {
|
||||
log.Printf("[restore] failed to remove komari-backup-markup: %v", rmErr)
|
||||
} else {
|
||||
log.Printf("[restore] komari-backup-markup removed")
|
||||
}
|
||||
}
|
||||
}()
|
||||
|
||||
logConfig := &gorm.Config{
|
||||
Logger: logutil.NewGormLogger(),
|
||||
}
|
||||
|
||||
// 根据数据库类型选择不同的连接方式
|
||||
switch flags.DatabaseType {
|
||||
case "sqlite", "":
|
||||
switch flags.ApplyDatabaseTypeNormalization() {
|
||||
case flags.DatabaseTypeSQLite:
|
||||
// SQLite 连接
|
||||
instance, err = gorm.Open(sqlite.Open(flags.DatabaseFile), logConfig)
|
||||
if err != nil {
|
||||
log.Fatalf("Failed to connect to SQLite3 database: %v", err)
|
||||
}
|
||||
log.Printf("Using SQLite database file: %s", flags.DatabaseFile)
|
||||
|
||||
case "mysql":
|
||||
// MySQL 连接
|
||||
dsn := fmt.Sprintf("%s:%s@tcp(%s:%s)/%s?charset=utf8mb4&collation=utf8mb4_unicode_ci&parseTime=True&loc=Local",
|
||||
flags.DatabaseUser,
|
||||
flags.DatabasePass,
|
||||
flags.DatabaseHost,
|
||||
flags.DatabasePort,
|
||||
flags.DatabaseName)
|
||||
instance, err = gorm.Open(mysql.Open(dsn), logConfig)
|
||||
if err != nil {
|
||||
log.Fatalf("Failed to connect to MySQL database: %v", err)
|
||||
instance.Exec("PRAGMA wal = ON;")
|
||||
if err := instance.Exec("PRAGMA journal_mode = WAL;").Error; err != nil {
|
||||
log.Printf("Failed to enable WAL mode for SQLite: %v", err)
|
||||
}
|
||||
log.Printf("Using MySQL database: %s@%s:%s/%s", flags.DatabaseUser, flags.DatabaseHost, flags.DatabasePort, flags.DatabaseName)
|
||||
instance.Exec("PRAGMA synchronous = NORMAL;")
|
||||
instance.Exec("PRAGMA cache_size = -65536;")
|
||||
instance.Exec("PRAGMA temp_store = MEMORY;")
|
||||
instance.Exec("PRAGMA wal_checkpoint(TRUNCATE);")
|
||||
default:
|
||||
log.Fatalf("Unsupported database type: %s", flags.DatabaseType)
|
||||
log.Fatalf("Unsupported database type: %s (supported: %s)", flags.DatabaseType, flags.SupportedDatabaseTypes())
|
||||
}
|
||||
MergeDatabase(instance)
|
||||
if err := migrations.Run(migrations.Context{DB: instance}); err != nil {
|
||||
log.Fatalf("Failed to run startup migrations: %v", err)
|
||||
}
|
||||
config.SetDb(instance)
|
||||
// 自动迁移模型
|
||||
err = instance.AutoMigrate(
|
||||
&models.User{},
|
||||
&models.Client{},
|
||||
&models.Record{},
|
||||
&models.GPURecord{},
|
||||
&models.Config{},
|
||||
&models.Log{},
|
||||
&models.Clipboard{},
|
||||
&models.LoadNotification{},
|
||||
&models.OfflineNotification{},
|
||||
&models.TrafficReportNotification{},
|
||||
&models.PingRecord{},
|
||||
&models.PingTask{},
|
||||
&models.OidcProvider{},
|
||||
@@ -295,6 +293,16 @@ func GetDBInstance() *gorm.DB {
|
||||
log.Printf("Failed to create Task and TaskResult table, it may already exist: %v", err)
|
||||
}
|
||||
|
||||
// Manually create composite indexes
|
||||
if flags.IsSQLite() {
|
||||
instance.Exec("CREATE INDEX IF NOT EXISTS idx_record_client_time ON records(client, time)")
|
||||
instance.Exec("CREATE INDEX IF NOT EXISTS idx_record_lt_client_time ON records_long_term(client, time)")
|
||||
instance.Exec("CREATE INDEX IF NOT EXISTS idx_gpu_record_client_time ON gpu_records(client, time)")
|
||||
instance.Exec("CREATE INDEX IF NOT EXISTS idx_gpu_record_lt_client_time ON gpu_records_long_term(client, time)")
|
||||
instance.Exec("CREATE INDEX IF NOT EXISTS idx_ping_record_client_time ON ping_records(client, time)")
|
||||
}
|
||||
|
||||
})
|
||||
|
||||
return instance
|
||||
}
|
||||
@@ -1,39 +0,0 @@
|
||||
package models
|
||||
|
||||
type Config struct {
|
||||
ID uint `json:"id,omitempty" gorm:"primaryKey;autoIncrement"` // 1
|
||||
Sitename string `json:"sitename" gorm:"type:varchar(100);not null"`
|
||||
Description string `json:"description" gorm:"type:text"`
|
||||
AllowCors bool `json:"allow_cors" gorm:"column:allow_cors;default:false"`
|
||||
Theme string `json:"theme" gorm:"type:varchar(100);default:'default'"` // 主题名称,默认 'default'
|
||||
PrivateSite bool `json:"private_site" gorm:"default:false"` // 是否为私有站点,默认 false
|
||||
ApiKey string `json:"api_key" gorm:"type:varchar(255);default:''"`
|
||||
AutoDiscoveryKey string `json:"auto_discovery_key" gorm:"type:varchar(255);default:''"` // 自动发现密钥
|
||||
// GeoIP 配置
|
||||
GeoIpEnabled bool `json:"geo_ip_enabled" gorm:"default:true"`
|
||||
GeoIpProvider string `json:"geo_ip_provider" gorm:"type:varchar(20);default:'ip-api'"` // empty, mmdb, ip-api, geojs
|
||||
// Nezha 兼容(Agent gRPC)
|
||||
NezhaCompatEnabled bool `json:"nezha_compat_enabled" gorm:"default:false"`
|
||||
NezhaCompatListen string `json:"nezha_compat_listen" gorm:"type:varchar(100);default:''"` // 例如 0.0.0.0:5555
|
||||
// OAuth 配置
|
||||
OAuthEnabled bool `json:"o_auth_enabled" gorm:"default:false"`
|
||||
OAuthProvider string `json:"o_auth_provider" gorm:"type:varchar(50);default:'github'"`
|
||||
DisablePasswordLogin bool `json:"disable_password_login" gorm:"default:false"`
|
||||
// 自定义美化
|
||||
CustomHead string `json:"custom_head" gorm:"type:longtext"`
|
||||
CustomBody string `json:"custom_body" gorm:"type:longtext"`
|
||||
// 通知
|
||||
NotificationEnabled bool `json:"notification_enabled" gorm:"default:false"` // 通知总开关
|
||||
NotificationMethod string `json:"notification_method" gorm:"type:varchar(64);default:'none'"`
|
||||
NotificationTemplate string `json:"notification_template" gorm:"type:longtext;default:'{{event}}: {{client}}\n{{time}}'"`
|
||||
ExpireNotificationEnabled bool `json:"expire_notification_enabled" gorm:"default:false"` // 是否启用过期通知
|
||||
ExpireNotificationLeadDays int `json:"expire_notification_lead_days" gorm:"default:7"` // 过期前多少天通知,默认7天
|
||||
LoginNotification bool `json:"login_notification" gorm:"default:false"` // 登录通知
|
||||
TrafficLimitPercentage float64 `json:"traffic_limit_percentage" gorm:"default:80.00"` // 流量限制百分比,默认80.00%
|
||||
// Record
|
||||
RecordEnabled bool `json:"record_enabled" gorm:"default:true"` // 是否启用记录功能
|
||||
RecordPreserveTime int `json:"record_preserve_time" gorm:"default:720"` // 记录保留时间,单位小时,默认30天
|
||||
PingRecordPreserveTime int `json:"ping_record_preserve_time" gorm:"default:24"` // Ping 记录保留时间,单位小时,默认1天
|
||||
CreatedAt LocalTime
|
||||
UpdatedAt LocalTime
|
||||
}
|
||||
@@ -8,4 +8,7 @@ const (
|
||||
Login = "Login"
|
||||
Alert = "Alert"
|
||||
Traffic = "Traffic"
|
||||
DReport = "DReport" // 日报
|
||||
WReport = "WReport" // 周报
|
||||
MReport = "MReport" // 月报
|
||||
)
|
||||
+26
-11
@@ -15,6 +15,7 @@ type Client struct {
|
||||
Virtualization string `json:"virtualization" gorm:"type:varchar(50)"`
|
||||
Arch string `json:"arch" gorm:"type:varchar(50)"`
|
||||
CpuCores int `json:"cpu_cores" gorm:"type:int"`
|
||||
CpuPhysicalCores int `json:"cpu_physical_cores" gorm:"type:int"`
|
||||
OS string `json:"os" gorm:"type:varchar(100)"`
|
||||
KernelVersion string `json:"kernel_version" gorm:"type:varchar(100)"`
|
||||
GpuName string `json:"gpu_name" gorm:"type:varchar(100)"`
|
||||
@@ -87,6 +88,8 @@ type Record struct {
|
||||
NetOut int64 `json:"net_out" gorm:"type:bigint"`
|
||||
NetTotalUp int64 `json:"net_total_up" gorm:"type:bigint"`
|
||||
NetTotalDown int64 `json:"net_total_down" gorm:"type:bigint"`
|
||||
TrafficUp int64 `json:"traffic_up" gorm:"type:bigint"`
|
||||
TrafficDown int64 `json:"traffic_down" gorm:"type:bigint"`
|
||||
Process int `json:"process"`
|
||||
Connections int `json:"connections"`
|
||||
ConnectionsUdp int `json:"connections_udp"`
|
||||
@@ -95,14 +98,14 @@ type Record struct {
|
||||
|
||||
// GPURecord logs individual GPU metrics over time
|
||||
type GPURecord struct {
|
||||
Client string `json:"client" gorm:"type:varchar(36);index"` // 客户端UUID
|
||||
Time LocalTime `json:"time" gorm:"index"` // 记录时间
|
||||
DeviceIndex int `json:"device_index" gorm:"index"` // GPU设备索引 (0,1,2...)
|
||||
DeviceName string `json:"device_name" gorm:"type:varchar(100)"` // GPU型号
|
||||
MemTotal int64 `json:"mem_total" gorm:"type:bigint"` // 显存总量(字节)
|
||||
MemUsed int64 `json:"mem_used" gorm:"type:bigint"` // 显存使用(字节)
|
||||
Utilization float32 `json:"utilization" gorm:"type:decimal(5,2)"` // GPU使用率(%)
|
||||
Temperature int `json:"temperature"` // GPU温度(°C)
|
||||
Client string `json:"client" gorm:"type:varchar(36);index"` // 客户端UUID
|
||||
Time LocalTime `json:"time" gorm:"index"` // 记录时间
|
||||
DeviceIndex int `json:"device_index" gorm:"index"` // GPU设备索引 (0,1,2...)
|
||||
DeviceName string `json:"device_name" gorm:"type:varchar(100)"` // GPU型号
|
||||
MemTotal int64 `json:"mem_total" gorm:"type:bigint"` // 显存总量(字节)
|
||||
MemUsed int64 `json:"mem_used" gorm:"type:bigint"` // 显存使用(字节)
|
||||
Utilization float32 `json:"utilization" gorm:"type:decimal(5,2)"` // GPU使用率(%)
|
||||
Temperature int `json:"temperature"` // GPU温度(°C)
|
||||
}
|
||||
|
||||
// StringArray represents a slice of strings stored as JSON in the database
|
||||
@@ -110,9 +113,21 @@ type GPURecord struct {
|
||||
type StringArray []string
|
||||
|
||||
func (sa *StringArray) Scan(value interface{}) error {
|
||||
bytes, ok := value.([]byte)
|
||||
if !ok {
|
||||
return fmt.Errorf("failed to scan StringArray: value is not []byte")
|
||||
var bytes []byte
|
||||
switch v := value.(type) {
|
||||
case nil:
|
||||
*sa = StringArray{}
|
||||
return nil
|
||||
case []byte:
|
||||
bytes = v
|
||||
case string:
|
||||
bytes = []byte(v)
|
||||
default:
|
||||
return fmt.Errorf("failed to scan StringArray: unsupported value type %T", value)
|
||||
}
|
||||
if len(bytes) == 0 {
|
||||
*sa = StringArray{}
|
||||
return nil
|
||||
}
|
||||
return json.Unmarshal(bytes, sa)
|
||||
}
|
||||
|
||||
@@ -0,0 +1,30 @@
|
||||
package models
|
||||
|
||||
import "testing"
|
||||
|
||||
func TestStringArrayScanAcceptsSQLiteTextShapes(t *testing.T) {
|
||||
tests := map[string]interface{}{
|
||||
"bytes": []byte(`["a","b"]`),
|
||||
"text": `["a","b"]`,
|
||||
"empty": "",
|
||||
"nil": nil,
|
||||
}
|
||||
|
||||
for name, input := range tests {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
var got StringArray
|
||||
if err := got.Scan(input); err != nil {
|
||||
t.Fatalf("scan StringArray: %v", err)
|
||||
}
|
||||
if input == "" || input == nil {
|
||||
if len(got) != 0 {
|
||||
t.Fatalf("expected empty StringArray, got %v", got)
|
||||
}
|
||||
return
|
||||
}
|
||||
if len(got) != 2 || got[0] != "a" || got[1] != "b" {
|
||||
t.Fatalf("unexpected StringArray: %v", got)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -21,3 +21,13 @@ type LoadNotification struct {
|
||||
Interval int `json:"interval" gorm:"type:int;not null;default:15"` // 监测间隔(分钟)
|
||||
LastNotified LocalTime `json:"last_notified"` // 上次通知时间
|
||||
}
|
||||
|
||||
// TrafficReportNotification 定义了流量定时报告的数据库模型
|
||||
type TrafficReportNotification struct {
|
||||
Client string `json:"client" gorm:"type:varchar(36);not null;index;unique;constraint:OnDelete:CASCADE,OnUpdate:CASCADE;foreignKey:client;references:UUID"`
|
||||
ClientInfo Client `json:"client_info,omitempty" gorm:"foreignKey:Client;references:UUID"`
|
||||
Enable bool `json:"enable" gorm:"type:boolean;default:false"`
|
||||
Daily bool `json:"daily" gorm:"type:boolean;default:false"` // 日报
|
||||
Weekly bool `json:"weekly" gorm:"type:boolean;default:false"` // 周报
|
||||
Monthly bool `json:"monthly" gorm:"type:boolean;default:false"` // 月报
|
||||
}
|
||||
@@ -9,11 +9,27 @@ type PingRecord struct {
|
||||
Value int `json:"value" gorm:"type:int;not null"` // Ping 值,单位毫秒
|
||||
}
|
||||
|
||||
// PingTask 表示一次延迟监测任务配置。
|
||||
type PingTask struct {
|
||||
Id uint `json:"id,omitempty" gorm:"primaryKey;autoIncrement"`
|
||||
Name string `json:"name" gorm:"type:varchar(255);not null;index"`
|
||||
Clients StringArray `json:"clients" gorm:"type:longtext"`
|
||||
Type string `json:"type" gorm:"type:varchar(12);not null;default:'icmp'"` // icmp tcp http
|
||||
Target string `json:"target" gorm:"type:varchar(255);not null"`
|
||||
Interval int `json:"interval" gorm:"type:int;not null;default:60"` // 间隔时间
|
||||
Id uint `json:"id,omitempty" gorm:"primaryKey;autoIncrement"`
|
||||
Weight int `json:"weight" gorm:"type:int;not null;default:0;index"`
|
||||
Name string `json:"name" gorm:"type:varchar(255);not null;index"`
|
||||
Clients StringArray `json:"clients" gorm:"type:longtext"`
|
||||
DefaultOn bool `json:"default_on" gorm:"column:all_clients;not null;default:false"` // 新加入的服务器是否自动开启此监测;现有服务器不受此字段影响
|
||||
Type string `json:"type" gorm:"type:varchar(12);not null;default:'icmp'"` // icmp tcp http
|
||||
Target string `json:"target" gorm:"type:varchar(255);not null"` // Ping 目标地址
|
||||
Interval int `json:"interval" gorm:"type:int;not null;default:60"` // 间隔时间
|
||||
}
|
||||
|
||||
// AppliesToClient 判断当前 PingTask 是否适用于指定服务器。
|
||||
func (task PingTask) AppliesToClient(uuid string) bool {
|
||||
if uuid == "" {
|
||||
return false
|
||||
}
|
||||
for _, client := range task.Clients {
|
||||
if client == uuid {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
+115
-4
@@ -1,5 +1,18 @@
|
||||
package models
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"net/url"
|
||||
"path"
|
||||
"strings"
|
||||
)
|
||||
|
||||
const (
|
||||
ThemeConfigurationManaged = "managed"
|
||||
ThemeConfigurationRaw = "raw"
|
||||
ThemeConfigurationRedirect = "redirect"
|
||||
)
|
||||
|
||||
// Theme represents a komari theme information
|
||||
type Theme struct {
|
||||
Name string `json:"name"` // 主题名称
|
||||
@@ -13,23 +26,121 @@ type Theme struct {
|
||||
}
|
||||
|
||||
type Configuration struct {
|
||||
Type string `json:"type"` // managed
|
||||
Type string `json:"type"` // managed raw redirect
|
||||
Icon string `json:"icon"` // 图标
|
||||
Name string `json:"name"`
|
||||
Name any `json:"name"`
|
||||
Data any `json:"data"` // 配置数据
|
||||
}
|
||||
|
||||
type ManagedThemeConfigurationItem struct {
|
||||
Key string `json:"key"`
|
||||
Name string `json:"name"`
|
||||
Name any `json:"name"`
|
||||
Required bool `json:"required"`
|
||||
Type string `json:"type"` // string number select switch title
|
||||
Options string `json:"options"`
|
||||
Default any `json:"default"`
|
||||
Help string `json:"help"`
|
||||
Help any `json:"help"`
|
||||
}
|
||||
|
||||
type ThemeConfiguration struct {
|
||||
Short string `json:"short" gorm:"primaryKey;unique;not null"`
|
||||
Data string `json:"data" gorm:"type:longtext" default:"{}"`
|
||||
}
|
||||
|
||||
func (t Theme) ConfigurationType() string {
|
||||
typ := strings.ToLower(strings.TrimSpace(t.Configuration.Type))
|
||||
if typ == "" {
|
||||
return ThemeConfigurationManaged
|
||||
}
|
||||
return typ
|
||||
}
|
||||
|
||||
func (t Theme) RawHTML() (string, bool) {
|
||||
if t.ConfigurationType() != ThemeConfigurationRaw {
|
||||
return "", false
|
||||
}
|
||||
return configurationDataString(t.Configuration.Data)
|
||||
}
|
||||
|
||||
func (t Theme) RedirectTarget() (string, bool) {
|
||||
if t.ConfigurationType() != ThemeConfigurationRedirect {
|
||||
return "", false
|
||||
}
|
||||
return NormalizeThemeRedirectTarget(t.Configuration.Data)
|
||||
}
|
||||
|
||||
func (t Theme) ValidateConfiguration() error {
|
||||
switch t.ConfigurationType() {
|
||||
case ThemeConfigurationManaged:
|
||||
return nil
|
||||
case ThemeConfigurationRaw:
|
||||
html, ok := t.RawHTML()
|
||||
if !ok || strings.TrimSpace(html) == "" {
|
||||
return fmt.Errorf("raw 类型主题需要在 configuration.data 中提供 HTML 字符串")
|
||||
}
|
||||
return nil
|
||||
case ThemeConfigurationRedirect:
|
||||
if _, ok := t.RedirectTarget(); !ok {
|
||||
return fmt.Errorf("redirect 类型主题需要在 configuration.data 中提供站内相对路径")
|
||||
}
|
||||
return nil
|
||||
default:
|
||||
return fmt.Errorf("不支持的主题类型: %s", t.Configuration.Type)
|
||||
}
|
||||
}
|
||||
|
||||
func NormalizeThemeRedirectTarget(data any) (string, bool) {
|
||||
target, ok := configurationDataString(data)
|
||||
if !ok {
|
||||
return "", false
|
||||
}
|
||||
|
||||
target = strings.TrimSpace(target)
|
||||
if target == "" || strings.Contains(target, "\\") || strings.HasPrefix(target, "//") {
|
||||
return "", false
|
||||
}
|
||||
|
||||
parsed, err := url.Parse(target)
|
||||
if err != nil || parsed.IsAbs() || parsed.Host != "" {
|
||||
return "", false
|
||||
}
|
||||
|
||||
cleanInputPath := parsed.Path
|
||||
if strings.HasPrefix(cleanInputPath, "/") {
|
||||
cleanInputPath = strings.TrimLeft(cleanInputPath, "/")
|
||||
} else {
|
||||
for strings.HasPrefix(cleanInputPath, "../") {
|
||||
cleanInputPath = strings.TrimPrefix(cleanInputPath, "../")
|
||||
}
|
||||
}
|
||||
|
||||
for _, segment := range strings.Split(cleanInputPath, "/") {
|
||||
if segment == ".." {
|
||||
return "", false
|
||||
}
|
||||
}
|
||||
|
||||
cleanPath := cleanInputPath
|
||||
if cleanPath == "" {
|
||||
cleanPath = "/"
|
||||
} else {
|
||||
cleanPath = path.Clean(cleanPath)
|
||||
if cleanPath == "." {
|
||||
cleanPath = "/"
|
||||
} else {
|
||||
cleanPath = "/" + strings.TrimPrefix(cleanPath, "/")
|
||||
}
|
||||
}
|
||||
|
||||
normalized := url.URL{
|
||||
Path: cleanPath,
|
||||
RawQuery: parsed.RawQuery,
|
||||
Fragment: parsed.Fragment,
|
||||
}
|
||||
return normalized.String(), true
|
||||
}
|
||||
|
||||
func configurationDataString(data any) (string, bool) {
|
||||
value, ok := data.(string)
|
||||
return value, ok
|
||||
}
|
||||
@@ -0,0 +1,141 @@
|
||||
package models
|
||||
|
||||
import "testing"
|
||||
|
||||
func TestThemeValidateConfiguration(t *testing.T) {
|
||||
tests := map[string]struct {
|
||||
theme Theme
|
||||
wantErr bool
|
||||
}{
|
||||
"managed empty data": {
|
||||
theme: Theme{
|
||||
Configuration: Configuration{Type: ThemeConfigurationManaged},
|
||||
},
|
||||
},
|
||||
"missing type defaults to managed": {
|
||||
theme: Theme{},
|
||||
},
|
||||
"raw html": {
|
||||
theme: Theme{
|
||||
Configuration: Configuration{
|
||||
Type: ThemeConfigurationRaw,
|
||||
Data: "<!doctype html><title>raw</title>",
|
||||
},
|
||||
},
|
||||
},
|
||||
"raw rejects empty html": {
|
||||
theme: Theme{
|
||||
Configuration: Configuration{
|
||||
Type: ThemeConfigurationRaw,
|
||||
Data: " ",
|
||||
},
|
||||
},
|
||||
wantErr: true,
|
||||
},
|
||||
"redirect relative path": {
|
||||
theme: Theme{
|
||||
Configuration: Configuration{
|
||||
Type: ThemeConfigurationRedirect,
|
||||
Data: "/dashboard?tab=nodes",
|
||||
},
|
||||
},
|
||||
},
|
||||
"redirect rejects absolute url": {
|
||||
theme: Theme{
|
||||
Configuration: Configuration{
|
||||
Type: ThemeConfigurationRedirect,
|
||||
Data: "https://example.com",
|
||||
},
|
||||
},
|
||||
wantErr: true,
|
||||
},
|
||||
"redirect rejects parent traversal": {
|
||||
theme: Theme{
|
||||
Configuration: Configuration{
|
||||
Type: ThemeConfigurationRedirect,
|
||||
Data: "/../admin",
|
||||
},
|
||||
},
|
||||
wantErr: true,
|
||||
},
|
||||
"unknown type": {
|
||||
theme: Theme{
|
||||
Configuration: Configuration{Type: "hosted"},
|
||||
},
|
||||
wantErr: true,
|
||||
},
|
||||
}
|
||||
|
||||
for name, tt := range tests {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
err := tt.theme.ValidateConfiguration()
|
||||
if tt.wantErr && err == nil {
|
||||
t.Fatal("expected error, got nil")
|
||||
}
|
||||
if !tt.wantErr && err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestNormalizeThemeRedirectTarget(t *testing.T) {
|
||||
tests := map[string]struct {
|
||||
input string
|
||||
want string
|
||||
ok bool
|
||||
}{
|
||||
"leading slash": {
|
||||
input: "/nodes",
|
||||
want: "/nodes",
|
||||
ok: true,
|
||||
},
|
||||
"missing leading slash": {
|
||||
input: "nodes",
|
||||
want: "/nodes",
|
||||
ok: true,
|
||||
},
|
||||
"parent prefix points to root path": {
|
||||
input: "../settings",
|
||||
want: "/settings",
|
||||
ok: true,
|
||||
},
|
||||
"multiple parent prefixes point to root path": {
|
||||
input: "../../settings/site",
|
||||
want: "/settings/site",
|
||||
ok: true,
|
||||
},
|
||||
"query and fragment": {
|
||||
input: "nodes?id=1#cpu",
|
||||
want: "/nodes?id=1#cpu",
|
||||
ok: true,
|
||||
},
|
||||
"absolute parent traversal": {
|
||||
input: "/../admin",
|
||||
},
|
||||
"nested parent traversal": {
|
||||
input: "nodes/../admin",
|
||||
},
|
||||
"protocol relative": {
|
||||
input: "//example.com",
|
||||
},
|
||||
"backslash": {
|
||||
input: `\admin`,
|
||||
},
|
||||
"absolute url": {
|
||||
input: "https://example.com",
|
||||
},
|
||||
}
|
||||
|
||||
for name, tt := range tests {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
got, ok := NormalizeThemeRedirectTarget(tt.input)
|
||||
if ok != tt.ok {
|
||||
t.Fatalf("ok = %v, want %v", ok, tt.ok)
|
||||
}
|
||||
if got != tt.want {
|
||||
t.Fatalf("target = %q, want %q", got, tt.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,144 @@
|
||||
package notification
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
|
||||
"github.com/komari-monitor/komari/database/dbcore"
|
||||
"github.com/komari-monitor/komari/database/models"
|
||||
"gorm.io/gorm/clause"
|
||||
)
|
||||
|
||||
func validateTrafficReportNotification(notification models.TrafficReportNotification) error {
|
||||
if notification.Client == "" {
|
||||
return fmt.Errorf("client UUID cannot be empty")
|
||||
}
|
||||
if notification.Enable && !notification.Daily && !notification.Weekly && !notification.Monthly {
|
||||
return fmt.Errorf("at least one cadence must be selected when enabling traffic reports")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func ValidateTrafficReportNotifications(notifications []models.TrafficReportNotification) error {
|
||||
for _, notification := range notifications {
|
||||
if err := validateTrafficReportNotification(notification); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func buildEnabledTrafficReportNotifications(uuids []string, existing []models.TrafficReportNotification) ([]models.TrafficReportNotification, error) {
|
||||
if len(uuids) == 0 {
|
||||
return nil, fmt.Errorf("at least one client UUID is required")
|
||||
}
|
||||
|
||||
existingByClient := make(map[string]models.TrafficReportNotification, len(existing))
|
||||
for _, notification := range existing {
|
||||
existingByClient[notification.Client] = notification
|
||||
}
|
||||
|
||||
notifications := make([]models.TrafficReportNotification, 0, len(uuids))
|
||||
for _, uuid := range uuids {
|
||||
if uuid == "" {
|
||||
return nil, fmt.Errorf("client UUID cannot be empty")
|
||||
}
|
||||
existingNotification, ok := existingByClient[uuid]
|
||||
if !ok || (!existingNotification.Daily && !existingNotification.Weekly && !existingNotification.Monthly) {
|
||||
return nil, fmt.Errorf("at least one cadence must be selected when enabling traffic reports")
|
||||
}
|
||||
notifications = append(notifications, models.TrafficReportNotification{
|
||||
Client: uuid,
|
||||
Enable: true,
|
||||
})
|
||||
}
|
||||
|
||||
return notifications, nil
|
||||
}
|
||||
|
||||
// ListTrafficReportNotifications 获取所有流量定时报告配置(关联客户端信息)
|
||||
func ListTrafficReportNotifications() ([]models.TrafficReportNotification, error) {
|
||||
db := dbcore.GetDBInstance()
|
||||
var notifications []models.TrafficReportNotification
|
||||
err := db.Model(&models.TrafficReportNotification{}).Preload("ClientInfo").Find(¬ifications).Error
|
||||
return notifications, err
|
||||
}
|
||||
|
||||
// EditTrafficReportNotifications 批量更新流量定时报告配置
|
||||
func EditTrafficReportNotifications(notifications []models.TrafficReportNotification) error {
|
||||
if err := ValidateTrafficReportNotifications(notifications); err != nil {
|
||||
return err
|
||||
}
|
||||
db := dbcore.GetDBInstance()
|
||||
return db.Model(&models.TrafficReportNotification{}).
|
||||
Clauses(clause.OnConflict{
|
||||
Columns: []clause.Column{{Name: "client"}},
|
||||
DoUpdates: clause.AssignmentColumns([]string{"enable", "daily", "weekly", "monthly"}),
|
||||
}).
|
||||
Select("client", "enable", "daily", "weekly", "monthly").
|
||||
Create(notifications).Error
|
||||
}
|
||||
|
||||
// EnableTrafficReportNotifications 批量启用(仅更新 enable 字段)
|
||||
func EnableTrafficReportNotifications(uuids []string) error {
|
||||
if len(uuids) == 0 {
|
||||
return fmt.Errorf("at least one client UUID is required")
|
||||
}
|
||||
|
||||
db := dbcore.GetDBInstance()
|
||||
var existing []models.TrafficReportNotification
|
||||
if err := db.Where("client IN ?", uuids).Find(&existing).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
notifications, err := buildEnabledTrafficReportNotifications(uuids, existing)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return db.Model(&models.TrafficReportNotification{}).
|
||||
Clauses(clause.OnConflict{
|
||||
Columns: []clause.Column{{Name: "client"}},
|
||||
DoUpdates: clause.AssignmentColumns([]string{"enable"}),
|
||||
}).
|
||||
Select("client", "enable").
|
||||
Create(notifications).Error
|
||||
}
|
||||
|
||||
// DisableTrafficReportNotifications 批量禁用
|
||||
func DisableTrafficReportNotifications(uuids []string) error {
|
||||
if len(uuids) == 0 {
|
||||
return fmt.Errorf("at least one client UUID is required")
|
||||
}
|
||||
db := dbcore.GetDBInstance()
|
||||
var notifications []models.TrafficReportNotification
|
||||
for _, uuid := range uuids {
|
||||
if uuid == "" {
|
||||
return fmt.Errorf("client UUID cannot be empty")
|
||||
}
|
||||
notifications = append(notifications, models.TrafficReportNotification{
|
||||
Client: uuid,
|
||||
Enable: false,
|
||||
})
|
||||
}
|
||||
return db.Model(&models.TrafficReportNotification{}).
|
||||
Clauses(clause.OnConflict{
|
||||
Columns: []clause.Column{{Name: "client"}},
|
||||
DoUpdates: clause.AssignmentColumns([]string{"enable"}),
|
||||
}).
|
||||
Select("client", "enable").
|
||||
Create(notifications).Error
|
||||
}
|
||||
|
||||
// GetEnabledTrafficReportByType 获取启用了指定类型报告的客户端配置
|
||||
func GetEnabledTrafficReportByType(daily, weekly, monthly bool) ([]models.TrafficReportNotification, error) {
|
||||
db := dbcore.GetDBInstance()
|
||||
var notifications []models.TrafficReportNotification
|
||||
query := db.Model(&models.TrafficReportNotification{}).Where("enable = ?", true)
|
||||
if daily {
|
||||
query = query.Where("daily = ?", true)
|
||||
} else if weekly {
|
||||
query = query.Where("weekly = ?", true)
|
||||
} else if monthly {
|
||||
query = query.Where("monthly = ?", true)
|
||||
}
|
||||
err := query.Find(¬ifications).Error
|
||||
return notifications, err
|
||||
}
|
||||
@@ -0,0 +1,36 @@
|
||||
package notification
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"github.com/komari-monitor/komari/database/models"
|
||||
)
|
||||
|
||||
func TestValidateTrafficReportNotificationsRejectsEnabledWithoutCadence(t *testing.T) {
|
||||
err := ValidateTrafficReportNotifications([]models.TrafficReportNotification{{
|
||||
Client: "client-a",
|
||||
Enable: true,
|
||||
}})
|
||||
|
||||
assert.Error(t, err)
|
||||
}
|
||||
|
||||
func TestBuildEnabledTrafficReportNotificationsRequiresExistingCadence(t *testing.T) {
|
||||
_, err := buildEnabledTrafficReportNotifications([]string{"client-a"}, nil)
|
||||
assert.Error(t, err)
|
||||
|
||||
_, err = buildEnabledTrafficReportNotifications([]string{"client-a"}, []models.TrafficReportNotification{{Client: "client-a"}})
|
||||
assert.Error(t, err)
|
||||
|
||||
notifications, err := buildEnabledTrafficReportNotifications([]string{"client-a"}, []models.TrafficReportNotification{{
|
||||
Client: "client-a",
|
||||
Daily: true,
|
||||
}})
|
||||
require.NoError(t, err)
|
||||
require.Len(t, notifications, 1)
|
||||
assert.Equal(t, "client-a", notifications[0].Client)
|
||||
assert.True(t, notifications[0].Enable)
|
||||
}
|
||||
+152
-48
@@ -1,6 +1,7 @@
|
||||
package records
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"log"
|
||||
"sort"
|
||||
"strings"
|
||||
@@ -11,6 +12,7 @@ import (
|
||||
"github.com/komari-monitor/komari/cmd/flags"
|
||||
"github.com/komari-monitor/komari/database/dbcore"
|
||||
"github.com/komari-monitor/komari/database/models"
|
||||
"github.com/komari-monitor/komari/utils"
|
||||
)
|
||||
|
||||
func RecordOne(rec models.Record) error {
|
||||
@@ -41,9 +43,9 @@ func DeleteAll() error {
|
||||
func GetGPURecordsByClientAndTime(uuid string, start, end time.Time) ([]models.GPURecord, error) {
|
||||
db := dbcore.GetDBInstance()
|
||||
var records []models.GPURecord
|
||||
|
||||
|
||||
fourHoursAgo := time.Now().Add(-4*time.Hour - time.Minute)
|
||||
|
||||
|
||||
var recentRecords []models.GPURecord
|
||||
recentStart := start
|
||||
if end.After(fourHoursAgo) {
|
||||
@@ -57,7 +59,7 @@ func GetGPURecordsByClientAndTime(uuid string, start, end time.Time) ([]models.G
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
var longTermRecords []models.GPURecord
|
||||
err := db.Table("gpu_records_long_term").Where("client = ? AND time >= ? AND time <= ?", uuid, start, end).
|
||||
Order("time ASC, device_index ASC").Find(&longTermRecords).Error
|
||||
@@ -172,49 +174,64 @@ func CompactRecord() error {
|
||||
return err
|
||||
}
|
||||
|
||||
if flags.DatabaseType == "sqlite" {
|
||||
if err := db.Exec("VACUUM").Error; err != nil {
|
||||
log.Printf("Error vacuuming database: %v", err)
|
||||
}
|
||||
if flags.IsSQLite() {
|
||||
db.Exec("PRAGMA wal_checkpoint(PASSIVE);")
|
||||
}
|
||||
//log.Printf("Record compaction completed")
|
||||
return nil
|
||||
}
|
||||
|
||||
func migrateOldRecords(db *gorm.DB) error {
|
||||
// 计算 4 小时前的时间
|
||||
fourHoursAgo := time.Now().Add(-4 * time.Hour)
|
||||
return migrateOldRecordsAt(db, time.Now())
|
||||
}
|
||||
|
||||
func compactRecordCutoff(now time.Time) time.Time {
|
||||
return now.Add(-4 * time.Hour).Truncate(15 * time.Minute)
|
||||
}
|
||||
|
||||
func migrateOldRecordsAt(db *gorm.DB, now time.Time) error {
|
||||
cutoff := compactRecordCutoff(now)
|
||||
|
||||
// 查询 records 表中超过 4 小时的记录
|
||||
var records []models.Record
|
||||
if err := db.Table("records").Where("time < ?", fourHoursAgo).Find(&records).Error; err != nil {
|
||||
if err := db.Table("records").Where("time < ?", cutoff).Find(&records).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if len(records) == 0 {
|
||||
return nil
|
||||
}
|
||||
previousByClient, err := getPreviousTrafficRecordsBefore(db, records)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
repairZeroTrafficDeltas(records, previousByClient)
|
||||
|
||||
// 按 Client 和 15 分钟时间段分组,并存储所有记录以计算分位数
|
||||
type groupData struct {
|
||||
Cpu []float32
|
||||
Gpu []float32
|
||||
Load []float32
|
||||
Temp []float32
|
||||
Ram []int64
|
||||
RamTotal []int64
|
||||
Swap []int64
|
||||
SwapTotal []int64
|
||||
Disk []int64
|
||||
DiskTotal []int64
|
||||
NetIn []int64
|
||||
NetOut []int64
|
||||
NetTotalUp []int64
|
||||
NetTotalDown []int64
|
||||
Process []int
|
||||
Connections []int
|
||||
ConnectionsUdp []int
|
||||
Uptime []int64
|
||||
Cpu []float32
|
||||
Gpu []float32
|
||||
Load []float32
|
||||
Temp []float32
|
||||
Ram []int64
|
||||
RamTotal []int64
|
||||
Swap []int64
|
||||
SwapTotal []int64
|
||||
Disk []int64
|
||||
DiskTotal []int64
|
||||
NetIn []int64
|
||||
NetOut []int64
|
||||
NetTotalUp []int64
|
||||
NetTotalDown []int64
|
||||
TrafficUp int64
|
||||
TrafficDown int64
|
||||
LatestTime time.Time
|
||||
LatestTotalUp int64
|
||||
LatestTotalDown int64
|
||||
Process []int
|
||||
Connections []int
|
||||
ConnectionsUdp []int
|
||||
Uptime []int64
|
||||
}
|
||||
|
||||
groupedRecords := make(map[string]*groupData)
|
||||
@@ -238,6 +255,13 @@ func migrateOldRecords(db *gorm.DB) error {
|
||||
data.NetOut = append(data.NetOut, record.NetOut)
|
||||
data.NetTotalUp = append(data.NetTotalUp, record.NetTotalUp)
|
||||
data.NetTotalDown = append(data.NetTotalDown, record.NetTotalDown)
|
||||
data.TrafficUp += record.TrafficUp
|
||||
data.TrafficDown += record.TrafficDown
|
||||
if data.LatestTime.IsZero() || record.Time.ToTime().After(data.LatestTime) {
|
||||
data.LatestTime = record.Time.ToTime()
|
||||
data.LatestTotalUp = record.NetTotalUp
|
||||
data.LatestTotalDown = record.NetTotalDown
|
||||
}
|
||||
data.Process = append(data.Process, record.Process)
|
||||
data.Connections = append(data.Connections, record.Connections)
|
||||
data.ConnectionsUdp = append(data.ConnectionsUdp, record.ConnectionsUdp)
|
||||
@@ -331,8 +355,10 @@ func migrateOldRecords(db *gorm.DB) error {
|
||||
DiskTotal: getIntPercentile(data.DiskTotal, high_percentile),
|
||||
NetIn: getIntPercentile(data.NetIn, 0.2),
|
||||
NetOut: getIntPercentile(data.NetOut, 0.2),
|
||||
NetTotalUp: getIntPercentile(data.NetTotalUp, high_percentile),
|
||||
NetTotalDown: getIntPercentile(data.NetTotalDown, high_percentile),
|
||||
NetTotalUp: data.LatestTotalUp,
|
||||
NetTotalDown: data.LatestTotalDown,
|
||||
TrafficUp: data.TrafficUp,
|
||||
TrafficDown: data.TrafficDown,
|
||||
Process: getInt32Percentile(data.Process, high_percentile),
|
||||
Connections: getInt32Percentile(data.Connections, high_percentile),
|
||||
ConnectionsUdp: getInt32Percentile(data.ConnectionsUdp, high_percentile),
|
||||
@@ -352,7 +378,7 @@ func migrateOldRecords(db *gorm.DB) error {
|
||||
}
|
||||
|
||||
// 删除 records 表中的旧数据
|
||||
if err := tx.Table("records").Where("time < ?", fourHoursAgo.Add(-1*time.Hour)).Delete(&models.Record{}).Error; err != nil {
|
||||
if err := tx.Table("records").Where("time < ?", cutoff.Add(-1*time.Hour)).Delete(&models.Record{}).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -360,20 +386,98 @@ func migrateOldRecords(db *gorm.DB) error {
|
||||
})
|
||||
}
|
||||
|
||||
func getPreviousTrafficRecordsBefore(db *gorm.DB, records []models.Record) (map[string]*models.Record, error) {
|
||||
firstTimeByClient := make(map[string]time.Time)
|
||||
for _, record := range records {
|
||||
recordTime := record.Time.ToTime()
|
||||
firstTime, ok := firstTimeByClient[record.Client]
|
||||
if !ok || recordTime.Before(firstTime) {
|
||||
firstTimeByClient[record.Client] = recordTime
|
||||
}
|
||||
}
|
||||
|
||||
previousByClient := make(map[string]*models.Record, len(firstTimeByClient))
|
||||
for clientUUID, firstTime := range firstTimeByClient {
|
||||
previous, err := getPreviousTrafficRecordBefore(db, clientUUID, firstTime)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if previous != nil {
|
||||
previousByClient[clientUUID] = previous
|
||||
}
|
||||
}
|
||||
return previousByClient, nil
|
||||
}
|
||||
|
||||
func getPreviousTrafficRecordBefore(db *gorm.DB, clientUUID string, before time.Time) (*models.Record, error) {
|
||||
var latest *models.Record
|
||||
for _, table := range []string{"records", "records_long_term"} {
|
||||
var record models.Record
|
||||
queryBefore := before
|
||||
if table == "records_long_term" {
|
||||
queryBefore = before.Truncate(15 * time.Minute)
|
||||
}
|
||||
err := db.Table(table).
|
||||
Where("client = ? AND time < ?", clientUUID, models.FromTime(queryBefore)).
|
||||
Order("time DESC").
|
||||
First(&record).Error
|
||||
if err == nil {
|
||||
if latest == nil || record.Time.ToTime().After(latest.Time.ToTime()) {
|
||||
latest = &record
|
||||
}
|
||||
continue
|
||||
}
|
||||
if !errors.Is(err, gorm.ErrRecordNotFound) {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
return latest, nil
|
||||
}
|
||||
|
||||
func repairZeroTrafficDeltas(records []models.Record, previousByClient map[string]*models.Record) {
|
||||
recordsByClient := make(map[string][]*models.Record)
|
||||
for i := range records {
|
||||
recordsByClient[records[i].Client] = append(recordsByClient[records[i].Client], &records[i])
|
||||
}
|
||||
|
||||
for _, clientRecords := range recordsByClient {
|
||||
sort.Slice(clientRecords, func(i, j int) bool {
|
||||
return clientRecords[i].Time.ToTime().Before(clientRecords[j].Time.ToTime())
|
||||
})
|
||||
var previous *models.Record
|
||||
if previousByClient != nil {
|
||||
previous = previousByClient[clientRecords[0].Client]
|
||||
}
|
||||
for _, current := range clientRecords {
|
||||
if previous == nil {
|
||||
previous = current
|
||||
continue
|
||||
}
|
||||
if current.TrafficUp == 0 {
|
||||
current.TrafficUp = utils.ComputeTrafficDelta(current.NetTotalUp, previous.NetTotalUp)
|
||||
}
|
||||
if current.TrafficDown == 0 {
|
||||
current.TrafficDown = utils.ComputeTrafficDelta(current.NetTotalDown, previous.NetTotalDown)
|
||||
}
|
||||
previous = current
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// migrateGPURecords 压缩GPU记录数据
|
||||
func migrateGPURecords(db *gorm.DB) error {
|
||||
fourHoursAgo := time.Now().Add(-4 * time.Hour)
|
||||
|
||||
cutoff := compactRecordCutoff(time.Now())
|
||||
|
||||
// 查询超过4小时的GPU记录
|
||||
var gpuRecords []models.GPURecord
|
||||
if err := db.Where("time < ?", fourHoursAgo).Find(&gpuRecords).Error; err != nil {
|
||||
if err := db.Where("time < ?", cutoff).Find(&gpuRecords).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
|
||||
if len(gpuRecords) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
|
||||
// 按Client + DeviceIndex + 15分钟时间窗口分组
|
||||
type gpuGroupKey struct {
|
||||
Client string
|
||||
@@ -381,16 +485,16 @@ func migrateGPURecords(db *gorm.DB) error {
|
||||
TimeSlot time.Time
|
||||
DeviceName string
|
||||
}
|
||||
|
||||
|
||||
type gpuGroupData struct {
|
||||
MemTotal []int64
|
||||
MemUsed []int64
|
||||
Utilization []float32
|
||||
Temperature []int
|
||||
}
|
||||
|
||||
|
||||
groupedGPUs := make(map[gpuGroupKey]*gpuGroupData)
|
||||
|
||||
|
||||
for _, record := range gpuRecords {
|
||||
key := gpuGroupKey{
|
||||
Client: record.Client,
|
||||
@@ -398,18 +502,18 @@ func migrateGPURecords(db *gorm.DB) error {
|
||||
TimeSlot: record.Time.ToTime().Truncate(15 * time.Minute),
|
||||
DeviceName: record.DeviceName,
|
||||
}
|
||||
|
||||
|
||||
if _, ok := groupedGPUs[key]; !ok {
|
||||
groupedGPUs[key] = &gpuGroupData{}
|
||||
}
|
||||
|
||||
|
||||
data := groupedGPUs[key]
|
||||
data.MemTotal = append(data.MemTotal, record.MemTotal)
|
||||
data.MemUsed = append(data.MemUsed, record.MemUsed)
|
||||
data.Utilization = append(data.Utilization, record.Utilization)
|
||||
data.Temperature = append(data.Temperature, record.Temperature)
|
||||
}
|
||||
|
||||
|
||||
// 百分位数计算函数 (复用传统Record压缩逻辑)
|
||||
getPercentile := func(values []float64, percentile float64) float64 {
|
||||
if len(values) == 0 {
|
||||
@@ -457,7 +561,7 @@ func migrateGPURecords(db *gorm.DB) error {
|
||||
}
|
||||
return float32(getPercentile(floats, percentile))
|
||||
}
|
||||
|
||||
|
||||
// 保持与传统Record压缩的一致性
|
||||
high_percentile := 0.7
|
||||
|
||||
@@ -480,10 +584,10 @@ func migrateGPURecords(db *gorm.DB) error {
|
||||
Utilization: getFloat32Percentile(data.Utilization, high_percentile),
|
||||
Temperature: int(getIntPercentile(convertIntToInt64(data.Temperature), high_percentile)),
|
||||
}
|
||||
|
||||
|
||||
if existingCount > 0 {
|
||||
// 更新已存在记录
|
||||
if err := tx.Table("gpu_records_long_term").Where("client = ? AND device_index = ? AND time = ?",
|
||||
if err := tx.Table("gpu_records_long_term").Where("client = ? AND device_index = ? AND time = ?",
|
||||
key.Client, key.DeviceIndex, key.TimeSlot).Updates(&compressedGPU).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -494,12 +598,12 @@ func migrateGPURecords(db *gorm.DB) error {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
// 删除已压缩的原始GPU数据
|
||||
if err := tx.Where("time < ?", fourHoursAgo.Add(-1*time.Hour)).Delete(&models.GPURecord{}).Error; err != nil {
|
||||
if err := tx.Where("time < ?", cutoff.Add(-1*time.Hour)).Delete(&models.GPURecord{}).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
|
||||
return nil
|
||||
})
|
||||
}
|
||||
@@ -8,6 +8,7 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
"gorm.io/driver/sqlite"
|
||||
"gorm.io/gorm"
|
||||
|
||||
@@ -25,8 +26,10 @@ var _ = func() bool {
|
||||
// then running migrateOldRecords and verifying the aggregation and cleanup.
|
||||
func TestCompactRecord(t *testing.T) {
|
||||
const totalMinutes = 12*60 + 30
|
||||
now := time.Now()
|
||||
threshold := now.Add(-4 * time.Hour)
|
||||
loc := models.GetAppLocation()
|
||||
now := time.Date(2026, 6, 15, 14, 30, 0, 0, loc)
|
||||
threshold := compactRecordCutoff(now)
|
||||
overlapCutoff := threshold.Add(-1 * time.Hour)
|
||||
|
||||
// 使用 sqlite 内存数据库并迁移表结构
|
||||
db, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{})
|
||||
@@ -35,7 +38,7 @@ func TestCompactRecord(t *testing.T) {
|
||||
assert.NoError(t, db.Table("records_long_term").AutoMigrate(&models.Record{}))
|
||||
|
||||
expectedGroups := make(map[time.Time]struct{})
|
||||
expectedRemain := 0
|
||||
expectedRawRemain := 0
|
||||
|
||||
// 插入数据
|
||||
for i := 0; i < totalMinutes; i++ {
|
||||
@@ -45,10 +48,11 @@ func TestCompactRecord(t *testing.T) {
|
||||
assert.NoError(t, err)
|
||||
|
||||
if recTime.Before(threshold) {
|
||||
slot := recTime.Truncate(time.Hour)
|
||||
slot := recTime.Truncate(15 * time.Minute)
|
||||
expectedGroups[slot] = struct{}{}
|
||||
} else {
|
||||
expectedRemain++
|
||||
}
|
||||
if !recTime.Before(overlapCutoff) {
|
||||
expectedRawRemain++
|
||||
}
|
||||
}
|
||||
|
||||
@@ -75,7 +79,7 @@ func TestCompactRecord(t *testing.T) {
|
||||
}
|
||||
|
||||
// 运行压缩(迁移)逻辑
|
||||
err = migrateOldRecords(db)
|
||||
err = migrateOldRecordsAt(db, now)
|
||||
assert.NoError(t, err)
|
||||
|
||||
// 验证 long-term 表中的聚合记录数
|
||||
@@ -86,7 +90,7 @@ func TestCompactRecord(t *testing.T) {
|
||||
// 验证原始表中剩余记录数
|
||||
var remainCount int64
|
||||
assert.NoError(t, db.Table("records").Count(&remainCount).Error)
|
||||
assert.Equal(t, int64(expectedRemain), remainCount+1)
|
||||
assert.Equal(t, int64(expectedRawRemain), remainCount)
|
||||
|
||||
// 导出压缩后的数据到 CSV
|
||||
var compRecs []models.Record
|
||||
@@ -128,3 +132,190 @@ func TestCompactRecord(t *testing.T) {
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestCompactRecordPreservesExactTrafficDelta(t *testing.T) {
|
||||
loc := models.GetAppLocation()
|
||||
currentTime := time.Date(2026, 6, 15, 14, 30, 0, 0, loc)
|
||||
now := currentTime.Truncate(15 * time.Minute).Add(-5*time.Hour + time.Minute)
|
||||
db, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{})
|
||||
assert.NoError(t, err)
|
||||
assert.NoError(t, db.AutoMigrate(&models.Record{}))
|
||||
assert.NoError(t, db.Table("records_long_term").AutoMigrate(&models.Record{}))
|
||||
|
||||
records := []models.Record{
|
||||
{
|
||||
Client: uuid,
|
||||
Time: models.FromTime(now),
|
||||
NetTotalUp: 100,
|
||||
NetTotalDown: 200,
|
||||
TrafficUp: 0,
|
||||
TrafficDown: 0,
|
||||
},
|
||||
{
|
||||
Client: uuid,
|
||||
Time: models.FromTime(now.Add(5 * time.Minute)),
|
||||
NetTotalUp: 150,
|
||||
NetTotalDown: 260,
|
||||
TrafficUp: 50,
|
||||
TrafficDown: 60,
|
||||
},
|
||||
{
|
||||
Client: uuid,
|
||||
Time: models.FromTime(now.Add(10 * time.Minute)),
|
||||
NetTotalUp: 10,
|
||||
NetTotalDown: 30,
|
||||
TrafficUp: 10,
|
||||
TrafficDown: 30,
|
||||
},
|
||||
}
|
||||
|
||||
for _, rec := range records {
|
||||
assert.NoError(t, db.Create(&rec).Error)
|
||||
}
|
||||
|
||||
assert.NoError(t, migrateOldRecordsAt(db, currentTime))
|
||||
|
||||
var compacted []models.Record
|
||||
assert.NoError(t, db.Table("records_long_term").Find(&compacted).Error)
|
||||
require.Len(t, compacted, 1)
|
||||
assert.Equal(t, int64(60), compacted[0].TrafficUp)
|
||||
assert.Equal(t, int64(90), compacted[0].TrafficDown)
|
||||
assert.Equal(t, int64(10), compacted[0].NetTotalUp)
|
||||
assert.Equal(t, int64(30), compacted[0].NetTotalDown)
|
||||
assert.True(t, compacted[0].Time.ToTime().Equal(records[2].Time.ToTime().Truncate(15*time.Minute)))
|
||||
}
|
||||
|
||||
func TestRepairZeroTrafficDeltasPreservesRawResetDetailBeforeCompaction(t *testing.T) {
|
||||
loc := models.GetAppLocation()
|
||||
start := time.Date(2026, 6, 6, 0, 0, 0, 0, loc)
|
||||
records := []models.Record{
|
||||
{Client: uuid, Time: models.FromTime(start), NetTotalUp: 100, NetTotalDown: 200},
|
||||
{Client: uuid, Time: models.FromTime(start.Add(5 * time.Minute)), NetTotalUp: 140, NetTotalDown: 260},
|
||||
{Client: uuid, Time: models.FromTime(start.Add(10 * time.Minute)), NetTotalUp: 10, NetTotalDown: 20},
|
||||
{Client: uuid, Time: models.FromTime(start.Add(15 * time.Minute)), NetTotalUp: 25, NetTotalDown: 35},
|
||||
}
|
||||
|
||||
repairZeroTrafficDeltas(records, nil)
|
||||
|
||||
assert.Equal(t, int64(0), records[0].TrafficUp)
|
||||
assert.Equal(t, int64(0), records[0].TrafficDown)
|
||||
assert.Equal(t, int64(40), records[1].TrafficUp)
|
||||
assert.Equal(t, int64(60), records[1].TrafficDown)
|
||||
assert.Equal(t, int64(10), records[2].TrafficUp)
|
||||
assert.Equal(t, int64(20), records[2].TrafficDown)
|
||||
assert.Equal(t, int64(15), records[3].TrafficUp)
|
||||
assert.Equal(t, int64(15), records[3].TrafficDown)
|
||||
}
|
||||
|
||||
func TestRepairZeroTrafficDeltasUsesPreviousPersistedBaseline(t *testing.T) {
|
||||
loc := models.GetAppLocation()
|
||||
now := time.Date(2026, 6, 7, 12, 0, 0, 0, loc)
|
||||
db, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{})
|
||||
assert.NoError(t, err)
|
||||
assert.NoError(t, db.AutoMigrate(&models.Record{}))
|
||||
assert.NoError(t, db.Table("records_long_term").AutoMigrate(&models.Record{}))
|
||||
|
||||
assert.NoError(t, db.Table("records_long_term").Create(&models.Record{
|
||||
Client: uuid,
|
||||
Time: models.FromTime(now.Add(-10 * time.Minute)),
|
||||
NetTotalUp: 100,
|
||||
NetTotalDown: 200,
|
||||
}).Error)
|
||||
|
||||
rawRecords := []models.Record{
|
||||
{Client: uuid, Time: models.FromTime(now), NetTotalUp: 150, NetTotalDown: 260},
|
||||
{Client: uuid, Time: models.FromTime(now.Add(5 * time.Minute)), NetTotalUp: 175, NetTotalDown: 300},
|
||||
}
|
||||
|
||||
previousByClient, err := getPreviousTrafficRecordsBefore(db, rawRecords)
|
||||
assert.NoError(t, err)
|
||||
repairZeroTrafficDeltas(rawRecords, previousByClient)
|
||||
|
||||
assert.Equal(t, int64(50), rawRecords[0].TrafficUp)
|
||||
assert.Equal(t, int64(60), rawRecords[0].TrafficDown)
|
||||
assert.Equal(t, int64(25), rawRecords[1].TrafficUp)
|
||||
assert.Equal(t, int64(40), rawRecords[1].TrafficDown)
|
||||
}
|
||||
|
||||
func TestRepairZeroTrafficDeltasIgnoresSameSlotLongTermBaseline(t *testing.T) {
|
||||
loc := models.GetAppLocation()
|
||||
now := time.Date(2026, 6, 7, 12, 0, 30, 0, loc)
|
||||
db, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{})
|
||||
assert.NoError(t, err)
|
||||
assert.NoError(t, db.AutoMigrate(&models.Record{}))
|
||||
assert.NoError(t, db.Table("records_long_term").AutoMigrate(&models.Record{}))
|
||||
|
||||
assert.NoError(t, db.Table("records_long_term").Create(&models.Record{
|
||||
Client: uuid,
|
||||
Time: models.FromTime(now.Truncate(15 * time.Minute)),
|
||||
NetTotalUp: 900,
|
||||
NetTotalDown: 900,
|
||||
}).Error)
|
||||
|
||||
rawRecords := []models.Record{
|
||||
{Client: uuid, Time: models.FromTime(now), NetTotalUp: 150, NetTotalDown: 260},
|
||||
{Client: uuid, Time: models.FromTime(now.Add(5 * time.Minute)), NetTotalUp: 175, NetTotalDown: 300},
|
||||
}
|
||||
|
||||
previousByClient, err := getPreviousTrafficRecordsBefore(db, rawRecords)
|
||||
assert.NoError(t, err)
|
||||
repairZeroTrafficDeltas(rawRecords, previousByClient)
|
||||
|
||||
assert.Equal(t, int64(0), rawRecords[0].TrafficUp)
|
||||
assert.Equal(t, int64(0), rawRecords[0].TrafficDown)
|
||||
assert.Equal(t, int64(25), rawRecords[1].TrafficUp)
|
||||
assert.Equal(t, int64(40), rawRecords[1].TrafficDown)
|
||||
}
|
||||
|
||||
func TestCompactRecordRetainsOneHourOverlapWindow(t *testing.T) {
|
||||
loc := models.GetAppLocation()
|
||||
now := time.Date(2026, 6, 7, 12, 7, 0, 0, loc)
|
||||
cutoff := compactRecordCutoff(now)
|
||||
db, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{})
|
||||
assert.NoError(t, err)
|
||||
assert.NoError(t, db.AutoMigrate(&models.Record{}))
|
||||
assert.NoError(t, db.Table("records_long_term").AutoMigrate(&models.Record{}))
|
||||
|
||||
records := []models.Record{
|
||||
{Client: uuid, Time: models.FromTime(cutoff.Add(-time.Hour - time.Minute)), TrafficUp: 5},
|
||||
{Client: uuid, Time: models.FromTime(cutoff.Add(-30 * time.Minute)), TrafficUp: 7},
|
||||
{Client: uuid, Time: models.FromTime(now.Add(-3 * time.Hour)), TrafficUp: 11},
|
||||
}
|
||||
for _, rec := range records {
|
||||
assert.NoError(t, db.Create(&rec).Error)
|
||||
}
|
||||
|
||||
assert.NoError(t, migrateOldRecordsAt(db, now))
|
||||
|
||||
var remainTimes []models.Record
|
||||
assert.NoError(t, db.Table("records").Order("time ASC").Find(&remainTimes).Error)
|
||||
require.Len(t, remainTimes, 2)
|
||||
assert.True(t, remainTimes[0].Time.ToTime().Equal(records[1].Time.ToTime()))
|
||||
assert.True(t, remainTimes[1].Time.ToTime().Equal(records[2].Time.ToTime()))
|
||||
}
|
||||
|
||||
func TestCompactRecordOnlyMigratesCompleteFifteenMinuteBuckets(t *testing.T) {
|
||||
loc := models.GetAppLocation()
|
||||
now := time.Date(2026, 6, 7, 12, 7, 0, 0, loc)
|
||||
cutoff := compactRecordCutoff(now)
|
||||
db, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{})
|
||||
assert.NoError(t, err)
|
||||
assert.NoError(t, db.AutoMigrate(&models.Record{}))
|
||||
assert.NoError(t, db.Table("records_long_term").AutoMigrate(&models.Record{}))
|
||||
|
||||
compactable := models.Record{Client: uuid, Time: models.FromTime(cutoff.Add(-time.Minute)), TrafficUp: 5}
|
||||
partialSlot := models.Record{Client: uuid, Time: models.FromTime(cutoff.Add(time.Minute)), TrafficUp: 7}
|
||||
assert.NoError(t, db.Create(&compactable).Error)
|
||||
assert.NoError(t, db.Create(&partialSlot).Error)
|
||||
|
||||
assert.NoError(t, migrateOldRecordsAt(db, now))
|
||||
|
||||
var compacted []models.Record
|
||||
assert.NoError(t, db.Table("records_long_term").Order("time ASC").Find(&compacted).Error)
|
||||
require.Len(t, compacted, 1)
|
||||
assert.True(t, compacted[0].Time.ToTime().Equal(compactable.Time.ToTime().Truncate(15*time.Minute)))
|
||||
|
||||
var rawCount int64
|
||||
assert.NoError(t, db.Table("records").Where("time = ?", partialSlot.Time).Count(&rawCount).Error)
|
||||
assert.Equal(t, int64(1), rawCount)
|
||||
}
|
||||
+116
-9
@@ -9,16 +9,35 @@ import (
|
||||
"gorm.io/gorm"
|
||||
)
|
||||
|
||||
func AddPingTask(clients []string, name string, target, task_type string, interval int) (uint, error) {
|
||||
// AddPingTask 创建延迟监测任务。defaultOn 表示新加入的服务器是否自动开启此监测。
|
||||
func AddPingTask(clients []string, defaultOn bool, name string, target, task_type string, interval int) (uint, error) {
|
||||
db := dbcore.GetDBInstance()
|
||||
normalizedClients := normalizePingClients(models.StringArray(clients))
|
||||
task := models.PingTask{
|
||||
Clients: clients,
|
||||
Name: name,
|
||||
Type: task_type,
|
||||
Target: target,
|
||||
Interval: interval,
|
||||
Clients: normalizedClients,
|
||||
DefaultOn: defaultOn,
|
||||
Name: name,
|
||||
Type: task_type,
|
||||
Target: target,
|
||||
Interval: interval,
|
||||
}
|
||||
if err := db.Create(&task).Error; err != nil {
|
||||
err := db.Transaction(func(tx *gorm.DB) error {
|
||||
if err := tx.Create(&task).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Append by id to avoid races between concurrent create requests.
|
||||
result := tx.Model(&models.PingTask{}).Where("id = ?", task.Id).Update("weight", int(task.Id))
|
||||
if result.Error != nil {
|
||||
return result.Error
|
||||
}
|
||||
if result.RowsAffected == 0 {
|
||||
return gorm.ErrRecordNotFound
|
||||
}
|
||||
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
ReloadPingSchedule()
|
||||
@@ -35,10 +54,21 @@ func DeletePingTask(id []uint) error {
|
||||
return result.Error
|
||||
}
|
||||
|
||||
// EditPingTask 批量更新延迟监测任务配置。
|
||||
func EditPingTask(tasks []*models.PingTask) error {
|
||||
db := dbcore.GetDBInstance()
|
||||
for _, task := range tasks {
|
||||
result := db.Model(&models.PingTask{}).Where("id = ?", task.Id).Updates(task)
|
||||
task.Clients = normalizePingClients(task.Clients)
|
||||
// 使用 map 显式更新,避免 GORM struct Updates 跳过 false/0/空切片等零值。
|
||||
updates := map[string]interface{}{
|
||||
"name": task.Name,
|
||||
"clients": task.Clients,
|
||||
"all_clients": task.DefaultOn,
|
||||
"type": task.Type,
|
||||
"target": task.Target,
|
||||
"interval": task.Interval,
|
||||
}
|
||||
result := db.Model(&models.PingTask{}).Where("id = ?", task.Id).Updates(updates)
|
||||
if result.RowsAffected == 0 {
|
||||
return gorm.ErrRecordNotFound
|
||||
}
|
||||
@@ -47,15 +77,54 @@ func EditPingTask(tasks []*models.PingTask) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// normalizePingClients 保持 clients 字段序列化为 JSON 数组,避免空值变成 null。
|
||||
func normalizePingClients(clients models.StringArray) models.StringArray {
|
||||
if clients == nil {
|
||||
return models.StringArray{}
|
||||
}
|
||||
return clients
|
||||
}
|
||||
|
||||
func GetAllPingTasks() ([]models.PingTask, error) {
|
||||
db := dbcore.GetDBInstance()
|
||||
var tasks []models.PingTask
|
||||
if err := db.Find(&tasks).Error; err != nil {
|
||||
if err := db.Order("weight ASC").Order("id ASC").Find(&tasks).Error; err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return tasks, nil
|
||||
}
|
||||
|
||||
// GetPingTasksByClient 获取指定服务器需要执行的延迟监测任务。
|
||||
func GetPingTasksByClient(uuid string) []models.PingTask {
|
||||
db := dbcore.GetDBInstance()
|
||||
var tasks []models.PingTask
|
||||
if err := db.Where("clients LIKE ?", `%"`+uuid+`"%`).Find(&tasks).Error; err != nil {
|
||||
return nil
|
||||
}
|
||||
return tasks
|
||||
}
|
||||
|
||||
func UpdatePingTaskOrder(order map[uint]int) error {
|
||||
db := dbcore.GetDBInstance()
|
||||
err := db.Transaction(func(tx *gorm.DB) error {
|
||||
for id, weight := range order {
|
||||
result := tx.Model(&models.PingTask{}).Where("id = ?", id).Update("weight", weight)
|
||||
if result.Error != nil {
|
||||
return result.Error
|
||||
}
|
||||
if result.RowsAffected == 0 {
|
||||
return gorm.ErrRecordNotFound
|
||||
}
|
||||
}
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
ReloadPingSchedule()
|
||||
return nil
|
||||
}
|
||||
|
||||
func SavePingRecord(record models.PingRecord) error {
|
||||
db := dbcore.GetDBInstance()
|
||||
return db.Create(&record).Error
|
||||
@@ -93,6 +162,44 @@ func ReloadPingSchedule() error {
|
||||
return utils.ReloadPingSchedule(pingTasks)
|
||||
}
|
||||
|
||||
// AddDefaultOnClientUUID 在新客户端注册后,把该 UUID 追加到所有 default_on=true 的任务的 clients 中(去重)。
|
||||
func AddDefaultOnClientUUID(uuid string) error {
|
||||
if uuid == "" {
|
||||
return nil
|
||||
}
|
||||
db := dbcore.GetDBInstance()
|
||||
var tasks []models.PingTask
|
||||
if err := db.Where("all_clients = ?", true).Find(&tasks).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
if len(tasks) == 0 {
|
||||
return nil
|
||||
}
|
||||
changed := false
|
||||
for _, task := range tasks {
|
||||
exists := false
|
||||
for _, c := range task.Clients {
|
||||
if c == uuid {
|
||||
exists = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if exists {
|
||||
continue
|
||||
}
|
||||
next := append(models.StringArray{}, task.Clients...)
|
||||
next = append(next, uuid)
|
||||
if err := db.Model(&models.PingTask{}).Where("id = ?", task.Id).Update("clients", next).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
changed = true
|
||||
}
|
||||
if changed {
|
||||
return ReloadPingSchedule()
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func GetPingRecords(uuid string, taskId int, start, end time.Time) ([]models.PingRecord, error) {
|
||||
db := dbcore.GetDBInstance()
|
||||
var records []models.PingRecord
|
||||
|
||||
+48
-4
@@ -8,16 +8,60 @@ import (
|
||||
"strings"
|
||||
|
||||
"github.com/gin-gonic/gin"
|
||||
"github.com/komari-monitor/komari/database/config"
|
||||
"github.com/komari-monitor/komari/database/dbcore"
|
||||
"github.com/komari-monitor/komari/database/models"
|
||||
"github.com/komari-monitor/komari/pkg/config"
|
||||
)
|
||||
|
||||
func GetPublicInfo() (any, error) {
|
||||
cst, err := config.Get()
|
||||
func GetPublicInfo() (map[string]interface{}, error) {
|
||||
cstPtr, err := config.GetManyAs[config.Settings]()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
cst := *cstPtr
|
||||
|
||||
all, allErr := config.GetAll()
|
||||
hasKey := func(k string) bool {
|
||||
if allErr != nil {
|
||||
return false
|
||||
}
|
||||
_, ok := all[k]
|
||||
return ok
|
||||
}
|
||||
|
||||
// Apply defaults only when a key is missing.
|
||||
if !hasKey("sitename") {
|
||||
cst.Sitename = "Komari"
|
||||
}
|
||||
if !hasKey("description") {
|
||||
cst.Description = "Komari Monitor, a simple server monitoring tool."
|
||||
}
|
||||
if !hasKey("theme") {
|
||||
cst.Theme = "default"
|
||||
}
|
||||
if !hasKey("o_auth_provider") {
|
||||
cst.OAuthProvider = "github"
|
||||
}
|
||||
if !hasKey("record_enabled") {
|
||||
cst.RecordEnabled = true
|
||||
}
|
||||
if !hasKey("record_preserve_time") {
|
||||
cst.RecordPreserveTime = 720
|
||||
}
|
||||
if !hasKey("ping_record_preserve_time") {
|
||||
cst.PingRecordPreserveTime = 24
|
||||
}
|
||||
|
||||
// Fallback defaults if we couldn't enumerate keys.
|
||||
if allErr != nil {
|
||||
if cst.Sitename == "" {
|
||||
cst.Sitename = "Komari"
|
||||
}
|
||||
if cst.Description == "" {
|
||||
cst.Description = "Komari Monitor, a simple server monitoring tool."
|
||||
}
|
||||
}
|
||||
|
||||
db := dbcore.GetDBInstance()
|
||||
tc := models.ThemeConfiguration{}
|
||||
err = db.Model(&models.ThemeConfiguration{}).Where("short = ?", cst.Theme).First(&tc).Error
|
||||
@@ -90,7 +134,7 @@ func GetPublicInfo() (any, error) {
|
||||
"oauth_enable": cst.OAuthEnabled,
|
||||
"oauth_provider": cst.OAuthProvider,
|
||||
"disable_password_login": cst.DisablePasswordLogin,
|
||||
"allow_cors": cst.AllowCors,
|
||||
"cors_origin_check_enabled": cst.CorsOriginCheckEnabled,
|
||||
"record_enabled": cst.RecordEnabled,
|
||||
"record_preserve_time": cst.RecordPreserveTime,
|
||||
"ping_record_preserve_time": cst.PingRecordPreserveTime,
|
||||
|
||||
+27
-10
@@ -1,14 +1,20 @@
|
||||
# Komari
|
||||
|
||||
# Komari
|
||||

|
||||
|
||||

|
||||
|
||||
Komariは、サーバーのパフォーマンスを監視するためのシンプルで効率的なソリューションを提供することを目的とした、軽量の自己ホスト型サーバー監視ツールです。Webインターフェースを介してサーバーのステータスを表示し、軽量エージェントを介してデータを収集します。
|
||||
|
||||
> [!WARNING]
|
||||
> Komariは、自己ホスト型の監視/制御プログラムであり、所有しているシステム、または管理権限を得ているシステムでのみ使用してください。Komariを武器化したり、許可なく展開、アクセス、永続化、コマンド実行、その他の不正利用に使用したりしないでください。実際の悪用リスクについては、Huntressの分析をご参照ください:[Komari C2 agent abuse](https://www.huntress.com/blog/komari-c2-agent-abuse)。
|
||||
> Komariの展開および運用方法については、利用者自身が責任を負います。開発者は、無許可または不正な利用、およびその結果について責任を負いません。
|
||||
> Windows端末でリモートコントロールを有効にした場合、クライアントはユーザーがログインするたびに、KomariがリモートコントロールソフトウェアであることをWindows通知で知らせます。
|
||||
|
||||
[ドキュメント](https://komari-document.pages.dev/) | [Telegramグループ](https://t.me/komari_monitor)
|
||||
|
||||
## 特徴
|
||||
|
||||
- **軽量で効率的**: リソース消費が少なく、あらゆる規模のサーバーに適しています。
|
||||
- **自己ホスト型**: データプライバシーを完全に制御でき、展開も簡単です。
|
||||
- **Webインターフェース**: 直感的な監視ダッシュボードで、使いやすいです。
|
||||
@@ -16,16 +22,19 @@ Komariは、サーバーのパフォーマンスを監視するためのシン
|
||||
## クイックスタート
|
||||
|
||||
### 0. クラウドホスティングによるワンクリック展開
|
||||
- Rainyun - CNY 4.5/月
|
||||
|
||||
[](https://app.rainyun.com/apps/rca/store/6780/NzYxNzAz_)
|
||||
- Rainyun - CNY 4.5/月
|
||||
|
||||
- Claw Run - USD 1.5/月
|
||||
[](https://app.rainyun.com/apps/rca/store/6780/NzYxNzAz_)
|
||||
|
||||
[](https://template.run.claw.cloud/?openapp=system-fastdeploy%3FtemplateName%3Dkomari)
|
||||
- 1Panel アプリストア
|
||||
|
||||
1Panel アプリストアで利用可能です。「アプリストア」>「ユーティリティ」>「Komari」からインストールしてください。
|
||||
|
||||
### 1. ワンクリックインストールスクリプトを使用する
|
||||
|
||||
systemdを使用するディストリビューション(Ubuntu、Debianなど)に適しています。
|
||||
|
||||
```bash
|
||||
curl -fsSL https://raw.githubusercontent.com/komari-monitor/komari/main/install-komari.sh -o install-komari.sh
|
||||
chmod +x install-komari.sh
|
||||
@@ -33,6 +42,7 @@ sudo ./install-komari.sh
|
||||
```
|
||||
|
||||
### 2. Docker展開
|
||||
|
||||
1. データディレクトリを作成します:
|
||||
```bash
|
||||
mkdir -p ./data
|
||||
@@ -55,6 +65,7 @@ sudo ./install-komari.sh
|
||||
> 環境変数 `ADMIN_USERNAME` と `ADMIN_PASSWORD` を使用して、初期のユーザー名とパスワードをカスタマイズすることもできます。
|
||||
|
||||
### 3. バイナリファイル展開
|
||||
|
||||
1. Komariの[GitHubリリース](https://github.com/komari-monitor/komari/releases)ページにアクセスして、お使いのオペレーティングシステム用の最新のバイナリをダウンロードします。
|
||||
2. Komariを実行します:
|
||||
```bash
|
||||
@@ -66,7 +77,6 @@ sudo ./install-komari.sh
|
||||
> [!NOTE]
|
||||
> バイナリに実行権限があることを確認してください(`chmod +x komari`)。データは実行ディレクトリの `data` フォルダに保存されます。
|
||||
|
||||
|
||||
### 手動ビルド
|
||||
|
||||
#### 依存関係
|
||||
@@ -85,36 +95,43 @@ sudo ./install-komari.sh
|
||||
git clone https://github.com/komari-monitor/komari
|
||||
cd komari
|
||||
```
|
||||
ステップ1で生成された静的ファイルを `komari` プロジェクトのルートにある `/public/dist` フォルダにコピーします。
|
||||
```bash
|
||||
ステップ1で生成された静的ファイルを `komari` プロジェクトのルートにある `/web/public/defaultTheme/dist` フォルダにコピーし、`komari-theme.json` と `preview.png`/`perview.png` を `/web/public/defaultTheme` にコピーします。
|
||||
```bash
|
||||
go build -o komari
|
||||
```
|
||||
4. 実行:
|
||||
3. 実行:
|
||||
```bash
|
||||
./komari server -l 0.0.0.0:25774
|
||||
```
|
||||
デフォルトのリスニングポートは `25774` です。`http://localhost:25774` にアクセスします。
|
||||
|
||||
## フロントエンド開発ガイド
|
||||
|
||||
[Komariテーマ開発ガイド | Komari](https://komari-document.pages.dev/dev/theme.html)
|
||||
|
||||
[CrowdinでKomariを翻訳する](https://crowdin.com/project/komari/invite?h=cd051bf172c9a9f7f1360e87ffb521692507706)
|
||||
|
||||
## クライアントエージェント開発ガイド
|
||||
|
||||
[Komariエージェント情報レポートおよびイベント処理ドキュメント](https://komari-document.pages.dev/dev/agent.html)
|
||||
|
||||
## 貢献
|
||||
|
||||
IssueやPull Requestを歓迎します!
|
||||
|
||||
## 謝辞
|
||||
|
||||
### 破碎工坊云 (Crash Work)
|
||||
|
||||
[破碎工坊云 - 効率的で安定した、安全な高防御サーバーとCDNソリューションを提供する専門的なクラウドコンピューティングプラットフォーム](https://www.crash.work/)
|
||||
|
||||
### DreamCloud
|
||||
|
||||
[DreamCloud - コストパフォーマンスに優れたアジア太平洋向け高防御直通](https://as211392.com/)
|
||||
|
||||
### 🚀 SharonNetworks スポンサー
|
||||
|
||||
[](https://sharon.io)
|
||||
[](https://sharon.io)
|
||||
|
||||
SharonNetworks は、あなたのビジネスの離陸を力強くサポートします!
|
||||
|
||||
|
||||
+26
-11
@@ -1,14 +1,20 @@
|
||||
# Komari
|
||||
|
||||
# Komari
|
||||

|
||||
|
||||

|
||||
|
||||
Komari 是一款輕量級的自託管伺服器監控工具,旨在提供簡單、高效的伺服器性能監控解決方案。它支援透過 Web 介面查看伺服器狀態,並透過輕量級 Agent 收集數據。
|
||||
|
||||
> [!WARNING]
|
||||
> Komari 是一款自託管的監控/控制程式,僅應部署在你擁有或已獲授權管理的系統上。請勿將 Komari 武器化,或在未獲授權的情況下部署、存取、持久化、執行命令及從事其他濫用行為。關於現實中的濫用風險,可參考 Huntress 的分析:[Komari C2 agent abuse](https://www.huntress.com/blog/komari-c2-agent-abuse)。
|
||||
> 使用者需要自行承擔部署和使用 Komari 的責任。開發者不對未經授權或濫用行為及其後果承擔責任。
|
||||
> 在 Windows 端啟用遠端控制後,客戶端會在每次使用者登入時透過 Windows 通知提醒使用者 Komari 是一款遠端控制軟體。
|
||||
|
||||
[文檔](https://komari-document.pages.dev/) | [Telegram 群組](https://t.me/komari_monitor)
|
||||
|
||||
## 特性
|
||||
|
||||
- **輕量高效**:低資源佔用,適合各種規模的伺服器。
|
||||
- **自託管**:完全掌控數據隱私,部署簡單。
|
||||
- **Web 介面**:直觀的監控儀表盤,易於使用。
|
||||
@@ -17,16 +23,18 @@ Komari 是一款輕量級的自託管伺服器監控工具,旨在提供簡單
|
||||
|
||||
### 0. 容器雲一鍵部署
|
||||
|
||||
- 雨雲雲應用 - CNY 4.5/月
|
||||
- 雨雲雲應用 - CNY 4.5/月
|
||||
|
||||
[](https://app.rainyun.com/apps/rca/store/6780/NzYxNzAz_)
|
||||
[](https://app.rainyun.com/apps/rca/store/6780/NzYxNzAz_)
|
||||
|
||||
- Claw Run - USD 1.5/月
|
||||
- 1Panel 應用商店
|
||||
|
||||
[](https://template.run.claw.cloud/?openapp=system-fastdeploy%3FtemplateName%3Dkomari)
|
||||
已上架 1Panel 應用商店,應用商店-實用工具-Komari 即可安裝
|
||||
|
||||
### 1. 使用一鍵安裝腳本
|
||||
|
||||
適用於使用了 systemd 的發行版(Ubuntu、Debian...)。
|
||||
|
||||
```bash
|
||||
curl -fsSL https://raw.githubusercontent.com/komari-monitor/komari/main/install-komari.sh -o install-komari.sh
|
||||
chmod +x install-komari.sh
|
||||
@@ -34,6 +42,7 @@ sudo ./install-komari.sh
|
||||
```
|
||||
|
||||
### 2. Docker 部署
|
||||
|
||||
1. 建立資料目錄:
|
||||
```bash
|
||||
mkdir -p ./data
|
||||
@@ -56,6 +65,7 @@ sudo ./install-komari.sh
|
||||
> 你也可以透過環境變數 `ADMIN_USERNAME` 和 `ADMIN_PASSWORD` 自訂初始使用者名稱和密碼。
|
||||
|
||||
### 3. 二進位檔案部署
|
||||
|
||||
1. 存取 Komari 的 [GitHub Release 頁面](https://github.com/komari-monitor/komari/releases) 下載適用於你作業系統的最新二進位檔案。
|
||||
2. 執行 Komari:
|
||||
```bash
|
||||
@@ -67,7 +77,6 @@ sudo ./install-komari.sh
|
||||
> [!NOTE]
|
||||
> 確保二進位檔案具有可執行權限(`chmod +x komari`)。資料將保存在執行目錄下的 `data` 資料夾中。
|
||||
|
||||
|
||||
### 手工建置
|
||||
|
||||
#### 依賴
|
||||
@@ -86,36 +95,43 @@ sudo ./install-komari.sh
|
||||
git clone https://github.com/komari-monitor/komari
|
||||
cd komari
|
||||
```
|
||||
將步驟1中產生的靜態檔案複製到 `komari` 專案根目錄下的 `/public/dist` 資料夾。
|
||||
```bash
|
||||
將步驟1中產生的靜態檔案複製到 `komari` 專案根目錄下的 `/web/public/defaultTheme/dist` 資料夾,並將 `komari-theme.json` 與 `preview.png`/`perview.png` 複製到 `/web/public/defaultTheme`。
|
||||
```bash
|
||||
go build -o komari
|
||||
```
|
||||
4. 執行:
|
||||
3. 執行:
|
||||
```bash
|
||||
./komari server -l 0.0.0.0:25774
|
||||
```
|
||||
預設監聽 `25774` 連接埠,存取 `http://localhost:25774`。
|
||||
|
||||
## 前端開發指南
|
||||
|
||||
[Komari 主題開發指南 | Komari](https://komari-document.pages.dev/dev/theme.html)
|
||||
|
||||
[在 Crowdin 上翻譯 Komari](https://crowdin.com/project/komari/invite?h=cd051bf172c9a9f7f1360e87ffb521692507706)
|
||||
|
||||
## 客戶端 Agent 開發指南
|
||||
|
||||
[Komari Agent 資訊上報與事件處理文檔](https://komari-document.pages.dev/dev/agent.html)
|
||||
|
||||
## 貢獻
|
||||
|
||||
歡迎提交 Issue 或 Pull Request!
|
||||
|
||||
## 鳴謝
|
||||
|
||||
### 破碎工坊雲
|
||||
|
||||
[破碎工坊雲 - 專業雲計算服務平台,提供高效、穩定、安全的高防伺服器與CDN解決方案](https://www.crash.work/)
|
||||
|
||||
### DreamCloud
|
||||
|
||||
[DreamCloud - 極高性價比解鎖直連亞太高防](https://as211392.com/)
|
||||
|
||||
### 🚀 由 SharonNetworks 贊助
|
||||
|
||||
[](https://sharon.io)
|
||||
[](https://sharon.io)
|
||||
|
||||
SharonNetworks 為您的業務起飛保駕護航!
|
||||
|
||||
@@ -130,4 +146,3 @@ SharonNetworks 為您的業務起飛保駕護航!
|
||||
## Star History
|
||||
|
||||
[](https://www.star-history.com/#komari-monitor/komari&Date)
|
||||
|
||||
+27
-11
@@ -1,14 +1,20 @@
|
||||
# Komari
|
||||
|
||||
# Komari
|
||||

|
||||
|
||||

|
||||
|
||||
Komari 是一款轻量级的自托管服务器监控工具,旨在提供简单、高效的服务器性能监控解决方案。它支持通过 Web 界面查看服务器状态,并通过轻量级 Agent 收集数据。
|
||||
|
||||
[文档](https://komari-document.pages.dev/) | [Telegram 群组](https://t.me/komari_monitor)
|
||||
> [!WARNING]
|
||||
> Komari 是一款自托管的监控/控制程序,仅应部署在你拥有或已获得授权管理的系统上。请勿将 Komari 武器化,或在未获授权的情况下部署、访问、持久化、执行命令及从事其他滥用行为。关于现实中的滥用风险,可参考 Huntress 的分析:[Komari C2 agent abuse](https://www.huntress.com/blog/komari-c2-agent-abuse)。
|
||||
> 用户需要自行承担部署和使用 Komari 的责任。开发者不对未经授权或滥用行为及其后果承担责任。
|
||||
> 在 Windows 端开启远程控制后,客户端会在每次用户登录时通过 Windows 通知提醒用户 Komari 是一款远程控制软件。
|
||||
|
||||
[文档](https://komari-document.pages.dev/) | [文档(镜像站 By Geekertao)](https://www.komari.wiki) | [Telegram 群组](https://t.me/komari_monitor)
|
||||
|
||||
## 特性
|
||||
|
||||
- **轻量高效**:低资源占用,适合各种规模的服务器。
|
||||
- **自托管**:完全掌控数据隐私,部署简单。
|
||||
- **Web 界面**:直观的监控仪表盘,易于使用。
|
||||
@@ -17,16 +23,18 @@ Komari 是一款轻量级的自托管服务器监控工具,旨在提供简单
|
||||
|
||||
### 0. 容器云一键部署
|
||||
|
||||
- 雨云云应用 - CNY 4.5/月
|
||||
- 雨云云应用 - CNY 4.5/月
|
||||
|
||||
[](https://app.rainyun.com/apps/rca/store/6780/NzYxNzAz_)
|
||||
[](https://app.rainyun.com/apps/rca/store/6780/NzYxNzAz_)
|
||||
|
||||
- Claw Run - USD 1.5/月
|
||||
- 1Panel 应用商店
|
||||
|
||||
[](https://template.run.claw.cloud/?openapp=system-fastdeploy%3FtemplateName%3Dkomari)
|
||||
已上架1Panel应用商店,应用商店-实用工具-Komari 即可安装
|
||||
|
||||
### 1. 使用一键安装脚本
|
||||
|
||||
适用于使用了 systemd 的发行版(Ubuntu、Debian...)。
|
||||
|
||||
```bash
|
||||
curl -fsSL https://raw.githubusercontent.com/komari-monitor/komari/main/install-komari.sh -o install-komari.sh
|
||||
chmod +x install-komari.sh
|
||||
@@ -34,6 +42,7 @@ sudo ./install-komari.sh
|
||||
```
|
||||
|
||||
### 2. Docker 部署
|
||||
|
||||
1. 创建数据目录:
|
||||
```bash
|
||||
mkdir -p ./data
|
||||
@@ -56,6 +65,7 @@ sudo ./install-komari.sh
|
||||
> 你也可以通过环境变量 `ADMIN_USERNAME` 和 `ADMIN_PASSWORD` 自定义初始用户名和密码。
|
||||
|
||||
### 3. 二进制文件部署
|
||||
|
||||
1. 访问 Komari 的 [GitHub Release 页面](https://github.com/komari-monitor/komari/releases) 下载适用于你操作系统的最新二进制文件。
|
||||
2. 运行 Komari:
|
||||
```bash
|
||||
@@ -67,7 +77,6 @@ sudo ./install-komari.sh
|
||||
> [!NOTE]
|
||||
> 确保二进制文件具有可执行权限(`chmod +x komari`)。数据将保存在运行目录下的 `data` 文件夹中。
|
||||
|
||||
|
||||
### 手工构建
|
||||
|
||||
#### 依赖
|
||||
@@ -86,36 +95,43 @@ sudo ./install-komari.sh
|
||||
git clone https://github.com/komari-monitor/komari
|
||||
cd komari
|
||||
```
|
||||
将步骤1中生成的静态文件复制到 `komari` 项目根目录下的 `/public/dist` 文件夹。
|
||||
```bash
|
||||
将步骤1中生成的静态文件复制到 `komari` 项目根目录下的 `/web/public/defaultTheme/dist` 文件夹,并将 `komari-theme.json` 与 `preview.png`/`perview.png` 复制到 `/web/public/defaultTheme`。
|
||||
```bash
|
||||
go build -o komari
|
||||
```
|
||||
4. 运行:
|
||||
3. 运行:
|
||||
```bash
|
||||
./komari server -l 0.0.0.0:25774
|
||||
```
|
||||
默认监听 `25774` 端口,访问 `http://localhost:25774`。
|
||||
|
||||
## 前端开发指南
|
||||
|
||||
[Komari 主题开发指南 | Komari](https://komari-document.pages.dev/dev/theme.html)
|
||||
|
||||
[在 Crowdin 上翻译 Komari](https://crowdin.com/project/komari/invite?h=cd051bf172c9a9f7f1360e87ffb521692507706)
|
||||
|
||||
## 客户端 Agent 开发指南
|
||||
|
||||
[Komari Agent 信息上报与事件处理文档](https://komari-document.pages.dev/dev/agent.html)
|
||||
|
||||
## 贡献
|
||||
|
||||
欢迎提交 Issue 或 Pull Request!
|
||||
|
||||
## 鸣谢
|
||||
|
||||
### 破碎工坊云
|
||||
|
||||
[破碎工坊云 - 专业云计算服务平台,提供高效、稳定、安全的高防服务器与CDN解决方案](https://www.crash.work/)
|
||||
|
||||
### DreamCloud
|
||||
|
||||
[DreamCloud - 极高性价比解锁直连亚太高防](https://as211392.com/)
|
||||
|
||||
### 🚀 由 SharonNetworks 赞助
|
||||
|
||||
[](https://sharon.io)
|
||||
[](https://sharon.io)
|
||||
|
||||
SharonNetworks 为您的业务起飞保驾护航!
|
||||
|
||||
|
||||
@@ -1,23 +1,27 @@
|
||||
module github.com/komari-monitor/komari
|
||||
|
||||
go 1.23.2
|
||||
go 1.25.0
|
||||
|
||||
require (
|
||||
github.com/coreos/go-oidc/v3 v3.15.0
|
||||
github.com/dop251/goja v0.0.0-20251008123653-cf18d89f3cf6
|
||||
github.com/dop251/goja_nodejs v0.0.0-20251015164255-5e94316bedaf
|
||||
github.com/gin-gonic/gin v1.10.0
|
||||
github.com/go-sql-driver/mysql v1.10.0
|
||||
github.com/google/uuid v1.6.0
|
||||
github.com/gorilla/websocket v1.5.3
|
||||
github.com/jackc/pgx/v5 v5.10.0
|
||||
github.com/mattn/go-sqlite3 v1.14.27
|
||||
github.com/oschwald/maxminddb-golang v1.13.1
|
||||
github.com/patrickmn/go-cache v2.1.0+incompatible
|
||||
github.com/pquerna/otp v1.5.0
|
||||
github.com/spf13/cobra v1.9.1
|
||||
github.com/stretchr/testify v1.10.0
|
||||
github.com/stretchr/testify v1.11.1
|
||||
golang.org/x/image v0.35.0
|
||||
google.golang.org/grpc v1.75.0
|
||||
google.golang.org/protobuf v1.36.6
|
||||
gorm.io/driver/mysql v1.5.7
|
||||
gorm.io/driver/sqlite v1.5.7
|
||||
gorm.io/gorm v1.26.1
|
||||
|
||||
)
|
||||
|
||||
// Misuse of ServerConfig.PublicKeyCallback may cause authorization bypass in golang.org/x/crypto #1
|
||||
@@ -29,29 +33,33 @@ require golang.org/x/crypto v0.39.0 // indirect
|
||||
require golang.org/x/net v0.41.0 // indirect
|
||||
|
||||
require (
|
||||
filippo.io/edwards25519 v1.1.0 // indirect
|
||||
filippo.io/edwards25519 v1.2.0 // indirect
|
||||
github.com/boombuler/barcode v1.0.1-0.20190219062509-6c824513bacc // indirect
|
||||
github.com/bytedance/sonic v1.13.2 // indirect
|
||||
github.com/bytedance/sonic/loader v0.2.4 // indirect
|
||||
github.com/cloudwego/base64x v0.1.5 // indirect
|
||||
github.com/davecgh/go-spew v1.1.1 // indirect
|
||||
github.com/dlclark/regexp2 v1.11.4 // indirect
|
||||
github.com/gabriel-vasile/mimetype v1.4.8 // indirect
|
||||
github.com/gin-contrib/sse v1.0.0 // indirect
|
||||
github.com/go-jose/go-jose/v4 v4.1.1 // indirect
|
||||
github.com/go-playground/locales v0.14.1 // indirect
|
||||
github.com/go-playground/universal-translator v0.18.1 // indirect
|
||||
github.com/go-playground/validator/v10 v10.26.0 // indirect
|
||||
github.com/go-sql-driver/mysql v1.9.2 // indirect
|
||||
github.com/go-sourcemap/sourcemap v2.1.4+incompatible // indirect
|
||||
github.com/goccy/go-json v0.10.5 // indirect
|
||||
github.com/google/pprof v0.0.0-20240727154555-813a5fbdbec8 // indirect
|
||||
github.com/inconshreveable/mousetrap v1.1.0 // indirect
|
||||
github.com/jackc/pgpassfile v1.0.0 // indirect
|
||||
github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect
|
||||
github.com/jackc/puddle/v2 v2.2.2 // indirect
|
||||
github.com/jinzhu/inflection v1.0.0 // indirect
|
||||
github.com/jinzhu/now v1.1.5 // indirect
|
||||
github.com/json-iterator/go v1.1.12 // indirect
|
||||
github.com/klauspost/cpuid/v2 v2.2.10 // indirect
|
||||
github.com/kr/pretty v0.3.0 // indirect
|
||||
github.com/kr/text v0.2.0 // indirect
|
||||
github.com/leodido/go-urn v1.4.0 // indirect
|
||||
github.com/mattn/go-isatty v0.0.20 // indirect
|
||||
github.com/mattn/go-sqlite3 v1.14.27 // indirect
|
||||
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
|
||||
github.com/modern-go/reflect2 v1.0.2 // indirect
|
||||
github.com/pelletier/go-toml/v2 v2.2.3 // indirect
|
||||
@@ -62,9 +70,9 @@ require (
|
||||
github.com/ugorji/go/codec v1.2.12 // indirect
|
||||
golang.org/x/arch v0.15.0 // indirect
|
||||
golang.org/x/oauth2 v0.30.0 // indirect
|
||||
golang.org/x/sync v0.19.0 // indirect
|
||||
golang.org/x/sys v0.33.0 // indirect
|
||||
golang.org/x/text v0.26.0 // indirect
|
||||
golang.org/x/text v0.33.0 // indirect
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20250707201910-8d1bb00bc6a7 // indirect
|
||||
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect
|
||||
gopkg.in/yaml.v3 v3.0.1 // indirect
|
||||
)
|
||||
@@ -1,5 +1,7 @@
|
||||
filippo.io/edwards25519 v1.1.0 h1:FNf4tywRC1HmFuKW5xopWpigGjJKiJSV0Cqo0cJWDaA=
|
||||
filippo.io/edwards25519 v1.1.0/go.mod h1:BxyFTGdWcka3PhytdK4V28tE5sGfRvvvRV7EaN4VDT4=
|
||||
filippo.io/edwards25519 v1.2.0 h1:crnVqOiS4jqYleHd9vaKZ+HKtHfllngJIiOpNpoJsjo=
|
||||
filippo.io/edwards25519 v1.2.0/go.mod h1:xzAOLCNug/yB62zG1bQ8uziwrIqIuxhctzJT18Q77mc=
|
||||
github.com/Masterminds/semver/v3 v3.2.1 h1:RN9w6+7QoMeJVGyfmbcgs28Br8cvmnucEXnY0rYXWg0=
|
||||
github.com/Masterminds/semver/v3 v3.2.1/go.mod h1:qvl/7zhW3nngYb5+80sSMF+FG2BjYrf8m9wsX0PNOMQ=
|
||||
github.com/boombuler/barcode v1.0.1-0.20190219062509-6c824513bacc h1:biVzkmvwrH8WK8raXaxBx6fRVTlJILwEwQGL1I/ByEI=
|
||||
github.com/boombuler/barcode v1.0.1-0.20190219062509-6c824513bacc/go.mod h1:paBWMcWSl3LHKBqUq+rly7CNSldXjb2rDl3JlRe0mD8=
|
||||
github.com/bytedance/sonic v1.13.2 h1:8/H1FempDZqC4VqjptGo14QQlJx8VdZJegxs6wwfqpQ=
|
||||
@@ -17,6 +19,12 @@ github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ3
|
||||
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
|
||||
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/dlclark/regexp2 v1.11.4 h1:rPYF9/LECdNymJufQKmri9gV604RvvABwgOA8un7yAo=
|
||||
github.com/dlclark/regexp2 v1.11.4/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8=
|
||||
github.com/dop251/goja v0.0.0-20251008123653-cf18d89f3cf6 h1:6dE1TmjqkY6tehR4A67gDNhvDtuZ54ocu7ab4K9o540=
|
||||
github.com/dop251/goja v0.0.0-20251008123653-cf18d89f3cf6/go.mod h1:MxLav0peU43GgvwVgNbLAj1s/bSGboKkhuULvq/7hx4=
|
||||
github.com/dop251/goja_nodejs v0.0.0-20251015164255-5e94316bedaf h1:gbmvliZnCut4NjaPSNOQlfqBoZ9C5Dpf72mHMMYhgVE=
|
||||
github.com/dop251/goja_nodejs v0.0.0-20251015164255-5e94316bedaf/go.mod h1:Tb7Xxye4LX7cT3i8YLvmPMGCV92IOi4CDZvm/V8ylc0=
|
||||
github.com/gabriel-vasile/mimetype v1.4.8 h1:FfZ3gj38NjllZIeJAmMhr+qKL8Wu+nOoI3GqacKw1NM=
|
||||
github.com/gabriel-vasile/mimetype v1.4.8/go.mod h1:ByKUIKGjh1ODkGM1asKUbQZOLGrPjydw3hYPU2YU9t8=
|
||||
github.com/gin-contrib/sse v1.0.0 h1:y3bT1mUWUxDpW4JLQg/HnTqV4rozuW4tC9eFKTxYI9E=
|
||||
@@ -37,9 +45,10 @@ github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJn
|
||||
github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91TpwSH2VMlDf28Uj24BCp08ZFTUY=
|
||||
github.com/go-playground/validator/v10 v10.26.0 h1:SP05Nqhjcvz81uJaRfEV0YBSSSGMc/iMaVtFbr3Sw2k=
|
||||
github.com/go-playground/validator/v10 v10.26.0/go.mod h1:I5QpIEbmr8On7W0TktmJAumgzX4CA1XNl4ZmDuVHKKo=
|
||||
github.com/go-sql-driver/mysql v1.7.0/go.mod h1:OXbVy3sEdcQ2Doequ6Z5BW6fXNQTmx+9S1MCJN5yJMI=
|
||||
github.com/go-sql-driver/mysql v1.9.2 h1:4cNKDYQ1I84SXslGddlsrMhc8k4LeDVj6Ad6WRjiHuU=
|
||||
github.com/go-sql-driver/mysql v1.9.2/go.mod h1:qn46aNg1333BRMNU69Lq93t8du/dwxI64Gl8i5p1WMU=
|
||||
github.com/go-sourcemap/sourcemap v2.1.4+incompatible h1:a+iTbH5auLKxaNwQFg0B+TCYl6lbukKPc7b5x0n1s6Q=
|
||||
github.com/go-sourcemap/sourcemap v2.1.4+incompatible/go.mod h1:F8jJfvm2KbVjc5NqelyYJmf/v5J0dwNLS2mL4sNA1Jg=
|
||||
github.com/go-sql-driver/mysql v1.10.0 h1:Q+1LV8DkHJvSYAdR83XzuhDaTykuDx0l6fkXxoWCWfw=
|
||||
github.com/go-sql-driver/mysql v1.10.0/go.mod h1:M+cqaI7+xxXGG9swrdeUIoPG3Y3KCkF0pZej+SK+nWk=
|
||||
github.com/goccy/go-json v0.10.5 h1:Fq85nIqj+gXn/S5ahsiTlK3TmC85qgirsdTP/+DeaC4=
|
||||
github.com/goccy/go-json v0.10.5/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M=
|
||||
github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek=
|
||||
@@ -47,12 +56,22 @@ github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6
|
||||
github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=
|
||||
github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU=
|
||||
github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
|
||||
github.com/google/pprof v0.0.0-20240727154555-813a5fbdbec8 h1:FKHo8hFI3A+7w0aUQuYXQ+6EN5stWmeY/AZqtM8xk9k=
|
||||
github.com/google/pprof v0.0.0-20240727154555-813a5fbdbec8/go.mod h1:K1liHPHnj73Fdn/EKuT8nrFqBihUSKXoLYU0BuatOYo=
|
||||
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
|
||||
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
|
||||
github.com/gorilla/websocket v1.5.3 h1:saDtZ6Pbx/0u+bgYQ3q96pZgCzfhKXGPqt7kZ72aNNg=
|
||||
github.com/gorilla/websocket v1.5.3/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
|
||||
github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8=
|
||||
github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
|
||||
github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM=
|
||||
github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg=
|
||||
github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 h1:iCEnooe7UlwOQYpKFhBabPMi4aNAfoODPEFNiAnClxo=
|
||||
github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM=
|
||||
github.com/jackc/pgx/v5 v5.10.0 h1:VhSvgU2jSli8o3AqIEOTJr7rZwAEUVo4E4XhR94Zfr0=
|
||||
github.com/jackc/pgx/v5 v5.10.0/go.mod h1:mal1tBGAFfLHvZzaYh77YS/eC6IX9OWbRV1QIIM0Jn4=
|
||||
github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo=
|
||||
github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4=
|
||||
github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E=
|
||||
github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc=
|
||||
github.com/jinzhu/now v1.1.5 h1:/o9tlHleP7gOFmsnYNz3RGnqzefHA47wQpKrrdTIwXQ=
|
||||
@@ -64,7 +83,6 @@ github.com/klauspost/cpuid/v2 v2.2.10 h1:tBs3QSyvjDyFTq3uoc/9xFpCuOsJQFNPiAhYdw2
|
||||
github.com/klauspost/cpuid/v2 v2.2.10/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0=
|
||||
github.com/knz/go-libedit v1.10.1/go.mod h1:MZTVkCWyz0oBc7JOWP3wNAzd002ZbM/5hgShxwh4x8M=
|
||||
github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
|
||||
github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
|
||||
github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0=
|
||||
github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk=
|
||||
github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
|
||||
@@ -93,7 +111,6 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb
|
||||
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
||||
github.com/pquerna/otp v1.5.0 h1:NMMR+WrmaqXU4EzdGJEE1aUUI0AMRzsp96fFFWNPwxs=
|
||||
github.com/pquerna/otp v1.5.0/go.mod h1:dkJfzwRKNiegxyNb54X/3fLwhCynbMspSyWKnvi1AEg=
|
||||
github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc=
|
||||
github.com/rogpeppe/go-internal v1.8.0 h1:FCbCCtXNOY3UtUuHUYaghJg4y7Fd14rXifAYUAtL9R8=
|
||||
github.com/rogpeppe/go-internal v1.8.0/go.mod h1:WmiCO8CzOY8rg0OYDC4/i/2WRWAB6poM+XZ2dLUbcbE=
|
||||
github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
|
||||
@@ -111,8 +128,9 @@ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/
|
||||
github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
|
||||
github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
|
||||
github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
|
||||
github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA=
|
||||
github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
|
||||
github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U=
|
||||
github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U=
|
||||
github.com/twitchyliquid64/golang-asm v0.15.1 h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS4MhqMhdFk5YI=
|
||||
github.com/twitchyliquid64/golang-asm v0.15.1/go.mod h1:a1lVb/DtPvCB8fslRZhAngC2+aY1QWCk3Cedj/Gdt08=
|
||||
github.com/ugorji/go/codec v1.2.12 h1:9LC83zGrHhuUA9l16C9AHXAqEV/2wBQ4nkvumAE65EE=
|
||||
@@ -133,15 +151,19 @@ golang.org/x/arch v0.15.0 h1:QtOrQd0bTUnhNVNndMpLHNWrDmYzZ2KDqSrEymqInZw=
|
||||
golang.org/x/arch v0.15.0/go.mod h1:JmwW7aLIoRUKgaTzhkiEFxvcEiQGyOg9BMonBJUS7EE=
|
||||
golang.org/x/crypto v0.39.0 h1:SHs+kF4LP+f+p14esP5jAoDpHU8Gu/v9lFRK6IT5imM=
|
||||
golang.org/x/crypto v0.39.0/go.mod h1:L+Xg3Wf6HoL4Bn4238Z6ft6KfEpN0tJGo53AAPC632U=
|
||||
golang.org/x/image v0.35.0 h1:LKjiHdgMtO8z7Fh18nGY6KDcoEtVfsgLDPeLyguqb7I=
|
||||
golang.org/x/image v0.35.0/go.mod h1:MwPLTVgvxSASsxdLzKrl8BRFuyqMyGhLwmC+TO1Sybk=
|
||||
golang.org/x/net v0.41.0 h1:vBTly1HeNPEn3wtREYfy4GZ/NECgw2Cnl+nK6Nz3uvw=
|
||||
golang.org/x/net v0.41.0/go.mod h1:B/K4NNqkfmg07DQYrbwvSluqCJOOXwUjeb/5lOisjbA=
|
||||
golang.org/x/oauth2 v0.30.0 h1:dnDm7JmhM45NNpd8FDDeLhK6FwqbOf4MLCM9zb1BOHI=
|
||||
golang.org/x/oauth2 v0.30.0/go.mod h1:B++QgG3ZKulg6sRPGD/mqlHQs5rB3Ml9erfeDY7xKlU=
|
||||
golang.org/x/sync v0.19.0 h1:vV+1eWNmZ5geRlYjzm2adRgW2/mcpevXNg50YZtPCE4=
|
||||
golang.org/x/sync v0.19.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI=
|
||||
golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.33.0 h1:q3i8TbbEz+JRD9ywIRlyRAQbM0qF7hu24q3teo2hbuw=
|
||||
golang.org/x/sys v0.33.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k=
|
||||
golang.org/x/text v0.26.0 h1:P42AVeLghgTYr4+xUnTRKDMqpar+PtX7KWuNQL21L8M=
|
||||
golang.org/x/text v0.26.0/go.mod h1:QK15LZJUUQVJxhz7wXgxSy/CJaTFjd0G+YLonydOVQA=
|
||||
golang.org/x/text v0.33.0 h1:B3njUFyqtHDUI5jMn1YIr5B0IE2U0qck04r6d4KPAxE=
|
||||
golang.org/x/text v0.33.0/go.mod h1:LuMebE6+rBincTi9+xWTY8TztLzKHc/9C1uBCG27+q8=
|
||||
gonum.org/v1/gonum v0.16.0 h1:5+ul4Swaf3ESvrOnidPp4GZbzf0mxVQpDCYUQE7OJfk=
|
||||
gonum.org/v1/gonum v0.16.0/go.mod h1:fef3am4MQ93R2HHpKnLk4/Tbh/s0+wqD5nfa6Pnwy4E=
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20250707201910-8d1bb00bc6a7 h1:pFyd6EwwL2TqFf8emdthzeX+gZE1ElRq3iM8pui4KBY=
|
||||
@@ -155,14 +177,13 @@ gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8
|
||||
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
|
||||
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
|
||||
gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
|
||||
gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
|
||||
gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
|
||||
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
|
||||
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
gorm.io/driver/mysql v1.5.7 h1:MndhOPYOfEp2rHKgkZIhJ16eVUIRf2HmzgoPmh7FCWo=
|
||||
gorm.io/driver/mysql v1.5.7/go.mod h1:sEtPWMiqiN1N1cMXoXmBbd8C6/l+TESwriotuRRpkDM=
|
||||
gorm.io/driver/sqlite v1.5.7 h1:8NvsrhP0ifM7LX9G4zPB97NwovUakUxc+2V2uuf3Z1I=
|
||||
gorm.io/driver/sqlite v1.5.7/go.mod h1:U+J8craQU6Fzkcvu8oLeAQmi50TkwPEhHDEjQZXDah4=
|
||||
gorm.io/gorm v1.25.7/go.mod h1:hbnx/Oo0ChWMn1BIhpy1oYozzpM15i4YPuHDmfYtwg8=
|
||||
gorm.io/gorm v1.26.1 h1:ghB2gUI9FkS46luZtn6DLZ0f6ooBJ5IbVej2ENFDjRw=
|
||||
gorm.io/gorm v1.26.1/go.mod h1:8Z33v652h4//uMA76KjeDH8mJXPm1QNCYrMeatR0DOE=
|
||||
nullprogram.com/x/optparse v1.0.0/go.mod h1:KdyPE+Igbe0jQUrVfMqDMeJQIJZEuyV7pjYmp6pbG50=
|
||||
@@ -2,12 +2,21 @@ package main
|
||||
|
||||
import (
|
||||
"log"
|
||||
"log/slog"
|
||||
|
||||
"github.com/komari-monitor/komari/cmd"
|
||||
"github.com/komari-monitor/komari/utils"
|
||||
logutil "github.com/komari-monitor/komari/utils/log"
|
||||
)
|
||||
|
||||
func main() {
|
||||
if utils.VersionHash == "unknown" {
|
||||
logutil.SetupGlobalLogger(slog.LevelDebug)
|
||||
} else {
|
||||
logutil.SetupGlobalLogger(slog.LevelInfo)
|
||||
}
|
||||
|
||||
log.Printf("Komari Monitor %s (hash: %s)", utils.CurrentVersion, utils.VersionHash)
|
||||
|
||||
cmd.Execute()
|
||||
}
|
||||
@@ -0,0 +1,635 @@
|
||||
package config
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"reflect"
|
||||
"strings"
|
||||
"sync"
|
||||
|
||||
"gorm.io/gorm"
|
||||
"gorm.io/gorm/clause"
|
||||
)
|
||||
|
||||
type ConfigItem struct {
|
||||
Key string `gorm:"primaryKey;column:key;type:text"`
|
||||
Value string `gorm:"column:value;type:text"` // 存 JSON 字符串
|
||||
}
|
||||
|
||||
func (ConfigItem) TableName() string {
|
||||
return "configs"
|
||||
}
|
||||
|
||||
var (
|
||||
db *gorm.DB
|
||||
SetDb = func(gdb *gorm.DB) {
|
||||
db = gdb
|
||||
if err := db.AutoMigrate(&ConfigItem{}); err != nil {
|
||||
panic("failed to migrate config item table: " + err.Error())
|
||||
}
|
||||
}
|
||||
)
|
||||
|
||||
// Get 获取原始值 (反序列化为 interface{})
|
||||
func Get(key string, defaul ...any) (any, error) {
|
||||
var item ConfigItem
|
||||
err := db.First(&item, "key = ?", key).Error
|
||||
if err != nil {
|
||||
if len(defaul) > 0 {
|
||||
v := defaul[0]
|
||||
err = Set(key, v)
|
||||
return v, err
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
|
||||
var val any
|
||||
if err := json.Unmarshal([]byte(item.Value), &val); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return val, nil
|
||||
}
|
||||
|
||||
// GetAs 获取并转换为指定类型 (泛型),支持数值类型自动转换
|
||||
func GetAs[T any](key string, defaul ...any) (T, error) {
|
||||
var t T
|
||||
var item ConfigItem
|
||||
|
||||
err := db.First(&item, "key = ?", key).Error
|
||||
if err != nil {
|
||||
if len(defaul) > 0 {
|
||||
// 尝试直接类型断言
|
||||
if v, ok := defaul[0].(T); ok {
|
||||
err = Set(key, v)
|
||||
return v, err
|
||||
}
|
||||
// 尝试类型转换
|
||||
val := reflect.ValueOf(&t).Elem()
|
||||
if err := convertAndSet(defaul[0], val); err != nil {
|
||||
return t, fmt.Errorf("default value type mismatch: expected %T, got %T", t, defaul[0])
|
||||
}
|
||||
err = Set(key, t)
|
||||
return t, err
|
||||
}
|
||||
return t, err
|
||||
}
|
||||
|
||||
// 先尝试直接反序列化
|
||||
if err = json.Unmarshal([]byte(item.Value), &t); err != nil {
|
||||
// 尝试通用解析后转换
|
||||
var generic any
|
||||
if err := json.Unmarshal([]byte(item.Value), &generic); err != nil {
|
||||
return t, err
|
||||
}
|
||||
val := reflect.ValueOf(&t).Elem()
|
||||
if err := convertAndSet(generic, val); err != nil {
|
||||
return t, err
|
||||
}
|
||||
}
|
||||
return t, nil
|
||||
}
|
||||
|
||||
// GetMany 获取多个配置项,keys 为 map[key]defaultValue
|
||||
// 如果 defaultValue 为 nil,则数据库不存在时不写入
|
||||
// 如果 defaultValue 不为 nil,则数据库不存在时写入默认值
|
||||
func GetMany(keys map[string]any) (map[string]any, error) {
|
||||
var items []ConfigItem
|
||||
result := make(map[string]any)
|
||||
keyList := make([]string, 0, len(keys))
|
||||
for k := range keys {
|
||||
keyList = append(keyList, k)
|
||||
}
|
||||
if len(keyList) == 0 {
|
||||
return result, nil
|
||||
}
|
||||
if err := db.Where("key IN ?", keyList).Find(&items).Error; err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
foundKeys := make(map[string]bool)
|
||||
for _, item := range items {
|
||||
var parsed any
|
||||
if err := json.Unmarshal([]byte(item.Value), &parsed); err == nil {
|
||||
result[item.Key] = parsed
|
||||
foundKeys[item.Key] = true
|
||||
}
|
||||
}
|
||||
|
||||
// 收集需要写入数据库的默认值
|
||||
var toInsert []ConfigItem
|
||||
for k, def := range keys {
|
||||
if _, found := foundKeys[k]; !found {
|
||||
if def != nil {
|
||||
result[k] = def
|
||||
// 序列化后加入待写入列表
|
||||
jsonBytes, err := json.Marshal(def)
|
||||
if err != nil {
|
||||
slog.Warn("marshal default value failed", "key", k, "error", err)
|
||||
continue
|
||||
}
|
||||
toInsert = append(toInsert, ConfigItem{
|
||||
Key: k,
|
||||
Value: string(jsonBytes),
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 批量写入默认值到数据库
|
||||
if len(toInsert) > 0 {
|
||||
if err := db.Clauses(clause.OnConflict{
|
||||
Columns: []clause.Column{{Name: "key"}},
|
||||
DoUpdates: clause.AssignmentColumns([]string{"value"}),
|
||||
}).Create(&toInsert).Error; err != nil {
|
||||
slog.Warn("batch insert default config failed", "error", err)
|
||||
}
|
||||
}
|
||||
|
||||
return result, nil
|
||||
}
|
||||
|
||||
// GetManyAs 将多个配置项映射到一个结构体中,json tag 作为 Key
|
||||
// 支持 default tag 作为默认值,如果数据库中不存在且有 default tag 则写入数据库
|
||||
// 没有 default tag 的字段使用零值,不写入数据库
|
||||
func GetManyAs[T any]() (*T, error) {
|
||||
var t T
|
||||
val := reflect.ValueOf(&t).Elem()
|
||||
typ := val.Type()
|
||||
|
||||
type fieldInfo struct {
|
||||
index int
|
||||
key string
|
||||
hasDefault bool
|
||||
defaultVal string
|
||||
}
|
||||
|
||||
fields := make([]fieldInfo, 0)
|
||||
keys := make([]string, 0)
|
||||
|
||||
for i := 0; i < val.NumField(); i++ {
|
||||
field := typ.Field(i)
|
||||
jsonTag := field.Tag.Get("json")
|
||||
if jsonTag == "" || jsonTag == "-" {
|
||||
continue
|
||||
}
|
||||
// 解析 json tag,处理 "key,omitempty" 格式
|
||||
key := strings.Split(jsonTag, ",")[0]
|
||||
if key == "" || key == "-" {
|
||||
continue
|
||||
}
|
||||
|
||||
defaultTag := field.Tag.Get("default")
|
||||
hasDefault := defaultTag != "" || field.Tag.Get("default") == ""
|
||||
// 检查是否显式定义了 default tag (即使值为空)
|
||||
_, hasDefault = field.Tag.Lookup("default")
|
||||
|
||||
fields = append(fields, fieldInfo{
|
||||
index: i,
|
||||
key: key,
|
||||
hasDefault: hasDefault,
|
||||
defaultVal: defaultTag,
|
||||
})
|
||||
keys = append(keys, key)
|
||||
}
|
||||
|
||||
if len(keys) == 0 {
|
||||
return &t, nil
|
||||
}
|
||||
|
||||
var items []ConfigItem
|
||||
if err := db.Where("key IN ?", keys).Find(&items).Error; err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// 建立数据库中存在的 key 映射
|
||||
foundItems := make(map[string]string) // key -> value
|
||||
for _, item := range items {
|
||||
foundItems[item.Key] = item.Value
|
||||
}
|
||||
|
||||
// 需要写入数据库的新配置项
|
||||
var toInsert []ConfigItem
|
||||
|
||||
for _, fi := range fields {
|
||||
fieldVal := val.Field(fi.index)
|
||||
if !fieldVal.CanSet() {
|
||||
continue
|
||||
}
|
||||
|
||||
if dbValue, found := foundItems[fi.key]; found {
|
||||
// 数据库中存在,使用数据库值
|
||||
if err := unmarshalToField(dbValue, fieldVal); err != nil {
|
||||
slog.Warn("unmarshal config failed", "key", fi.key, "error", err)
|
||||
}
|
||||
} else if fi.hasDefault {
|
||||
// 数据库中不存在,但有 default tag,解析默认值并写入数据库
|
||||
if err := parseDefaultToField(fi.defaultVal, fieldVal); err != nil {
|
||||
slog.Warn("parse default value failed", "key", fi.key, "error", err)
|
||||
continue
|
||||
}
|
||||
// 序列化后写入数据库
|
||||
jsonBytes, err := json.Marshal(fieldVal.Interface())
|
||||
if err != nil {
|
||||
slog.Warn("marshal default value failed", "key", fi.key, "error", err)
|
||||
continue
|
||||
}
|
||||
toInsert = append(toInsert, ConfigItem{
|
||||
Key: fi.key,
|
||||
Value: string(jsonBytes),
|
||||
})
|
||||
}
|
||||
// 没有 default tag 且数据库中不存在,保持零值,不写入数据库
|
||||
}
|
||||
|
||||
// 批量写入默认值到数据库
|
||||
if len(toInsert) > 0 {
|
||||
if err := db.Clauses(clause.OnConflict{
|
||||
Columns: []clause.Column{{Name: "key"}},
|
||||
DoUpdates: clause.AssignmentColumns([]string{"value"}),
|
||||
}).Create(&toInsert).Error; err != nil {
|
||||
slog.Warn("batch insert default config failed", "error", err)
|
||||
}
|
||||
}
|
||||
|
||||
return &t, nil
|
||||
}
|
||||
|
||||
// unmarshalToField 将 JSON 字符串反序列化到字段,支持数值类型转换
|
||||
func unmarshalToField(jsonStr string, fieldVal reflect.Value) error {
|
||||
target := reflect.New(fieldVal.Type()).Interface()
|
||||
if err := json.Unmarshal([]byte(jsonStr), target); err != nil {
|
||||
// 尝试通用解析后转换
|
||||
var generic any
|
||||
if err := json.Unmarshal([]byte(jsonStr), &generic); err != nil {
|
||||
return err
|
||||
}
|
||||
return convertAndSet(generic, fieldVal)
|
||||
}
|
||||
fieldVal.Set(reflect.ValueOf(target).Elem())
|
||||
return nil
|
||||
}
|
||||
|
||||
// parseDefaultToField 解析 default tag 值到字段
|
||||
func parseDefaultToField(defaultVal string, fieldVal reflect.Value) error {
|
||||
kind := fieldVal.Kind()
|
||||
|
||||
switch kind {
|
||||
case reflect.String:
|
||||
fieldVal.SetString(defaultVal)
|
||||
case reflect.Bool:
|
||||
fieldVal.SetBool(defaultVal == "true" || defaultVal == "1")
|
||||
case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
|
||||
var v int64
|
||||
if defaultVal != "" {
|
||||
if _, err := fmt.Sscanf(defaultVal, "%d", &v); err != nil {
|
||||
// 尝试解析浮点数后转换
|
||||
var f float64
|
||||
if _, err := fmt.Sscanf(defaultVal, "%f", &f); err != nil {
|
||||
return err
|
||||
}
|
||||
v = int64(f)
|
||||
}
|
||||
}
|
||||
fieldVal.SetInt(v)
|
||||
case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64:
|
||||
var v uint64
|
||||
if defaultVal != "" {
|
||||
if _, err := fmt.Sscanf(defaultVal, "%d", &v); err != nil {
|
||||
var f float64
|
||||
if _, err := fmt.Sscanf(defaultVal, "%f", &f); err != nil {
|
||||
return err
|
||||
}
|
||||
v = uint64(f)
|
||||
}
|
||||
}
|
||||
fieldVal.SetUint(v)
|
||||
case reflect.Float32, reflect.Float64:
|
||||
var v float64
|
||||
if defaultVal != "" {
|
||||
if _, err := fmt.Sscanf(defaultVal, "%f", &v); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
fieldVal.SetFloat(v)
|
||||
default:
|
||||
// 对于复杂类型,尝试 JSON 解析
|
||||
if defaultVal == "" {
|
||||
return nil // 保持零值
|
||||
}
|
||||
target := reflect.New(fieldVal.Type()).Interface()
|
||||
if err := json.Unmarshal([]byte(defaultVal), target); err != nil {
|
||||
return err
|
||||
}
|
||||
fieldVal.Set(reflect.ValueOf(target).Elem())
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// convertAndSet 通用类型转换并设置字段值
|
||||
func convertAndSet(val any, fieldVal reflect.Value) error {
|
||||
if val == nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
targetType := fieldVal.Type()
|
||||
v := reflect.ValueOf(val)
|
||||
|
||||
// 直接类型匹配
|
||||
if v.Type().AssignableTo(targetType) {
|
||||
fieldVal.Set(v)
|
||||
return nil
|
||||
}
|
||||
|
||||
// 类型可转换
|
||||
if v.Type().ConvertibleTo(targetType) {
|
||||
fieldVal.Set(v.Convert(targetType))
|
||||
return nil
|
||||
}
|
||||
|
||||
// 数值类型特殊处理 (JSON 数字默认解析为 float64)
|
||||
if f, ok := val.(float64); ok {
|
||||
switch fieldVal.Kind() {
|
||||
case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
|
||||
fieldVal.SetInt(int64(f))
|
||||
return nil
|
||||
case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64:
|
||||
fieldVal.SetUint(uint64(f))
|
||||
return nil
|
||||
case reflect.Float32, reflect.Float64:
|
||||
fieldVal.SetFloat(f)
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
// JSON 回环转换
|
||||
b, err := json.Marshal(val)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
target := reflect.New(targetType).Interface()
|
||||
if err := json.Unmarshal(b, target); err != nil {
|
||||
return err
|
||||
}
|
||||
fieldVal.Set(reflect.ValueOf(target).Elem())
|
||||
return nil
|
||||
}
|
||||
|
||||
func GetAll() (map[string]any, error) {
|
||||
var items []ConfigItem
|
||||
result := make(map[string]any)
|
||||
if err := db.Find(&items).Error; err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
for _, item := range items {
|
||||
var parsed any
|
||||
if err := json.Unmarshal([]byte(item.Value), &parsed); err == nil {
|
||||
result[item.Key] = parsed
|
||||
}
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
// Set 设置单个配置
|
||||
func Set(key string, value any) error {
|
||||
oldVal := map[string]any{}
|
||||
{
|
||||
var oldItem ConfigItem
|
||||
if err := db.First(&oldItem, "key = ?", key).Error; err == nil {
|
||||
var parsed any
|
||||
if err := json.Unmarshal([]byte(oldItem.Value), &parsed); err == nil {
|
||||
oldVal[key] = parsed
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
bytes, err := json.Marshal(value)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
item := ConfigItem{
|
||||
Key: key,
|
||||
Value: string(bytes),
|
||||
}
|
||||
|
||||
err = db.Clauses(clause.OnConflict{
|
||||
Columns: []clause.Column{{Name: "key"}},
|
||||
DoUpdates: clause.AssignmentColumns([]string{"value"}),
|
||||
}).Create(&item).Error
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
newVal := map[string]any{key: value}
|
||||
publishEvent(oldVal, newVal)
|
||||
return nil
|
||||
}
|
||||
|
||||
// SetMany 将结构体保存为多个配置项
|
||||
func SetManyAs[T any](config T) error {
|
||||
val := reflect.ValueOf(config)
|
||||
if val.Kind() == reflect.Ptr {
|
||||
val = val.Elem()
|
||||
}
|
||||
|
||||
typ := val.Type()
|
||||
var items []ConfigItem
|
||||
|
||||
for i := 0; i < val.NumField(); i++ {
|
||||
fieldType := typ.Field(i)
|
||||
tag := fieldType.Tag.Get("json")
|
||||
|
||||
if tag == "" || tag == "-" {
|
||||
continue
|
||||
}
|
||||
|
||||
fieldValue := val.Field(i).Interface()
|
||||
|
||||
bytes, err := json.Marshal(fieldValue)
|
||||
if err != nil {
|
||||
return fmt.Errorf("marshal field %s failed: %w", fieldType.Name, err)
|
||||
}
|
||||
|
||||
items = append(items, ConfigItem{
|
||||
Key: tag,
|
||||
Value: string(bytes),
|
||||
})
|
||||
}
|
||||
|
||||
if len(items) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
keys := make([]string, 0, len(items))
|
||||
newVal := make(map[string]any, len(items))
|
||||
for _, it := range items {
|
||||
keys = append(keys, it.Key)
|
||||
var parsed any
|
||||
if err := json.Unmarshal([]byte(it.Value), &parsed); err == nil {
|
||||
newVal[it.Key] = parsed
|
||||
}
|
||||
}
|
||||
|
||||
oldVal := map[string]any{}
|
||||
if len(keys) > 0 {
|
||||
var oldItems []ConfigItem
|
||||
if err := db.Where("key IN ?", keys).Find(&oldItems).Error; err == nil {
|
||||
for _, oi := range oldItems {
|
||||
var parsed any
|
||||
if err := json.Unmarshal([]byte(oi.Value), &parsed); err == nil {
|
||||
oldVal[oi.Key] = parsed
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
err := db.Clauses(clause.OnConflict{
|
||||
Columns: []clause.Column{{Name: "key"}},
|
||||
DoUpdates: clause.AssignmentColumns([]string{"value"}),
|
||||
}).Create(&items).Error
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
publishEvent(oldVal, newVal)
|
||||
return nil
|
||||
}
|
||||
|
||||
func SetMany(cst map[string]any) error {
|
||||
var items []ConfigItem
|
||||
for k, v := range cst {
|
||||
bytes, err := json.Marshal(v)
|
||||
if err != nil {
|
||||
return fmt.Errorf("marshal key %s failed: %w", k, err)
|
||||
}
|
||||
items = append(items, ConfigItem{
|
||||
Key: k,
|
||||
Value: string(bytes),
|
||||
})
|
||||
}
|
||||
if len(items) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
keys := make([]string, 0, len(items))
|
||||
newVal := make(map[string]any, len(items))
|
||||
for _, it := range items {
|
||||
keys = append(keys, it.Key)
|
||||
var parsed any
|
||||
if err := json.Unmarshal([]byte(it.Value), &parsed); err == nil {
|
||||
newVal[it.Key] = parsed
|
||||
}
|
||||
}
|
||||
|
||||
oldVal := map[string]any{}
|
||||
if len(keys) > 0 {
|
||||
var oldItems []ConfigItem
|
||||
if err := db.Where("key IN ?", keys).Find(&oldItems).Error; err == nil {
|
||||
for _, oi := range oldItems {
|
||||
var parsed any
|
||||
if err := json.Unmarshal([]byte(oi.Value), &parsed); err == nil {
|
||||
oldVal[oi.Key] = parsed
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
err := db.Clauses(clause.OnConflict{
|
||||
Columns: []clause.Column{{Name: "key"}},
|
||||
DoUpdates: clause.AssignmentColumns([]string{"value"}),
|
||||
}).Create(&items).Error
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
publishEvent(oldVal, newVal)
|
||||
return nil
|
||||
}
|
||||
|
||||
type ConfigEvent struct {
|
||||
Old map[string]any
|
||||
New map[string]any
|
||||
}
|
||||
|
||||
func (e ConfigEvent) IsChanged(key string) bool {
|
||||
oldVal, oldOk := e.Old[key]
|
||||
newVal, newOk := e.New[key]
|
||||
if !oldOk && !newOk {
|
||||
return false
|
||||
}
|
||||
if oldOk != newOk {
|
||||
return true
|
||||
}
|
||||
return !reflect.DeepEqual(oldVal, newVal)
|
||||
}
|
||||
|
||||
func IsChangedT[T any](e ConfigEvent, key string) (bool, T) {
|
||||
changed := e.IsChanged(key)
|
||||
var zero T
|
||||
|
||||
val, ok := e.New[key]
|
||||
if !ok {
|
||||
val, ok = e.Old[key]
|
||||
if !ok {
|
||||
return changed, zero
|
||||
}
|
||||
}
|
||||
if val == nil {
|
||||
return changed, zero
|
||||
}
|
||||
|
||||
// Fast path: direct assertion.
|
||||
if cast, ok := val.(T); ok {
|
||||
return changed, cast
|
||||
}
|
||||
|
||||
// Try reflection-based conversion (covers numeric conversions, etc.).
|
||||
targetType := reflect.TypeOf((*T)(nil)).Elem()
|
||||
v := reflect.ValueOf(val)
|
||||
if v.IsValid() {
|
||||
if v.Type().AssignableTo(targetType) {
|
||||
return changed, v.Interface().(T)
|
||||
}
|
||||
if v.Type().ConvertibleTo(targetType) {
|
||||
converted := v.Convert(targetType)
|
||||
return changed, converted.Interface().(T)
|
||||
}
|
||||
}
|
||||
|
||||
// Fallback: JSON roundtrip for map/struct and other loosely typed values.
|
||||
if b, err := json.Marshal(val); err == nil {
|
||||
var out T
|
||||
if err := json.Unmarshal(b, &out); err == nil {
|
||||
return changed, out
|
||||
}
|
||||
}
|
||||
|
||||
return changed, zero
|
||||
}
|
||||
|
||||
// ConfigSubscriber handles config events
|
||||
type ConfigSubscriber func(event ConfigEvent)
|
||||
|
||||
var (
|
||||
subscribersMu sync.RWMutex
|
||||
subscribers []ConfigSubscriber
|
||||
)
|
||||
|
||||
// Subscribe registers a subscriber for all config events.
|
||||
func Subscribe(subscriber ConfigSubscriber) {
|
||||
subscribersMu.Lock()
|
||||
defer subscribersMu.Unlock()
|
||||
subscribers = append(subscribers, subscriber)
|
||||
}
|
||||
|
||||
// publishEvent notifies all subscribers of a config change.
|
||||
func publishEvent(oldVal, newVal map[string]any) {
|
||||
subscribersMu.RLock()
|
||||
defer subscribersMu.RUnlock()
|
||||
for _, sub := range subscribers {
|
||||
event := ConfigEvent{Old: oldVal, New: newVal}
|
||||
go sub(event)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,87 @@
|
||||
package config
|
||||
|
||||
import "time"
|
||||
|
||||
type Settings struct {
|
||||
ID uint `json:"id,omitempty"` // 1
|
||||
Sitename string `json:"sitename" default:"Komari"` // 站点名称,默认 "Komari"
|
||||
Description string `json:"description" default:"A simple server monitor tool."` // 站点描述
|
||||
CorsOriginCheckEnabled bool `json:"cors_origin_check_enabled" default:"true"` // 是否启用 API CORS 跨域请求校验,默认 true
|
||||
CorsAllowedOrigins string `json:"cors_allowed_origins" default:""` // API 跨域允许列表
|
||||
WsOriginCheckEnabled bool `json:"ws_origin_check_enabled" default:"true"` // 是否校验 WebSocket Origin
|
||||
WsAllowedOrigins string `json:"ws_allowed_origins" default:""` // WebSocket Origin 允许列表
|
||||
Theme string `json:"theme" default:"default"` // 主题名称,默认 'default'
|
||||
PrivateSite bool `json:"private_site" default:"false"` // 是否为私有站点,默认 false
|
||||
ApiKey string `json:"api_key" default:""` // API 密钥,默认空字符串
|
||||
AutoDiscoveryKey string `json:"auto_discovery_key" default:""` // 自动发现密钥
|
||||
ScriptDomain string `json:"script_domain" default:""` // 自定义脚本域名
|
||||
SendIpAddrToGuest bool `json:"send_ip_addr_to_guest" default:"false"` // 是否向访客页面发送 IP 地址,默认 false
|
||||
EulaAccepted bool `json:"eula_accepted" default:"false"`
|
||||
BaseScriptsURLKey string `json:"base_scripts_url" default:""`
|
||||
// GeoIP 配置
|
||||
GeoIpEnabled bool `json:"geo_ip_enabled" default:"true"`
|
||||
GeoIpProvider string `json:"geo_ip_provider" default:"ipinfo"` // empty, mmdb, ip-api, geojs
|
||||
// Nezha 兼容(Agent gRPC)
|
||||
NezhaCompatEnabled bool `json:"nezha_compat_enabled" default:"false"`
|
||||
NezhaCompatListen string `json:"nezha_compat_listen" default:""` // 例如 0.0.0.0:5555
|
||||
// OAuth 配置
|
||||
OAuthEnabled bool `json:"o_auth_enabled" default:"false"`
|
||||
OAuthProvider string `json:"o_auth_provider" default:"github"`
|
||||
DisablePasswordLogin bool `json:"disable_password_login" default:"false"`
|
||||
CloudflareTunnelToken string `json:"cloudflare_tunnel_token" default:""`
|
||||
// 自定义美化
|
||||
CustomHead string `json:"custom_head" default:""`
|
||||
CustomBody string `json:"custom_body" default:""`
|
||||
// 通知
|
||||
NotificationEnabled bool `json:"notification_enabled" default:"true"` // 通知总开关
|
||||
NotificationMethod string `json:"notification_method" default:"none"`
|
||||
NotificationTemplate string `json:"notification_template" default:"{{emoji}}{{emoji}}{{emoji}}\nEvent: {{event}}\nClients: {{client}}\nMessage: {{message}}\nTime: {{time}}"`
|
||||
ExpireNotificationEnabled bool `json:"expire_notification_enabled" default:"true"` // 是否启用过期通知
|
||||
ExpireNotificationLeadDays int `json:"expire_notification_lead_days" default:"7"` // 过期前多少天通知,默认7天
|
||||
LoginNotification bool `json:"login_notification" default:"true"` // 登录通知
|
||||
TrafficLimitPercentage float64 `json:"traffic_limit_percentage" default:"80.00"` // 流量限制百分比,默认80.00%
|
||||
// Record
|
||||
RecordEnabled bool `json:"record_enabled" default:"true"` // 是否启用记录功能
|
||||
RecordPreserveTime int `json:"record_preserve_time" default:"720"` // 记录保留时间,单位小时,默认30天
|
||||
PingRecordPreserveTime int `json:"ping_record_preserve_time" default:"24"` // Ping 记录保留时间,单位小时,默认1天
|
||||
UpdatedAt time.Time
|
||||
}
|
||||
|
||||
const (
|
||||
SitenameKey = "sitename"
|
||||
DescriptionKey = "description"
|
||||
CorsOriginCheckEnabledKey = "cors_origin_check_enabled"
|
||||
CorsAllowedOriginsKey = "cors_allowed_origins"
|
||||
WsOriginCheckEnabledKey = "ws_origin_check_enabled"
|
||||
WsAllowedOriginsKey = "ws_allowed_origins"
|
||||
ThemeKey = "theme"
|
||||
PrivateSiteKey = "private_site"
|
||||
ApiKeyKey = "api_key"
|
||||
AutoDiscoveryKeyKey = "auto_discovery_key"
|
||||
ScriptDomainKey = "script_domain"
|
||||
SendIpAddrToGuestKey = "send_ip_addr_to_guest"
|
||||
EulaAcceptedKey = "eula_accepted"
|
||||
BaseScriptsURLKey = "base_scripts_url"
|
||||
GeoIpEnabledKey = "geo_ip_enabled"
|
||||
GeoIpProviderKey = "geo_ip_provider"
|
||||
NezhaCompatEnabledKey = "nezha_compat_enabled"
|
||||
NezhaCompatListenKey = "nezha_compat_listen"
|
||||
OAuthEnabledKey = "o_auth_enabled"
|
||||
OAuthProviderKey = "o_auth_provider"
|
||||
DisablePasswordLoginKey = "disable_password_login"
|
||||
CloudflareTunnelTokenKey = "cloudflare_tunnel_token"
|
||||
CustomHeadKey = "custom_head"
|
||||
CustomBodyKey = "custom_body"
|
||||
NotificationEnabledKey = "notification_enabled"
|
||||
NotificationMethodKey = "notification_method"
|
||||
NotificationTemplateKey = "notification_template"
|
||||
ExpireNotificationEnabledKey = "expire_notification_enabled"
|
||||
ExpireNotificationLeadDaysKey = "expire_notification_lead_days"
|
||||
LoginNotificationKey = "login_notification"
|
||||
TrafficLimitPercentageKey = "traffic_limit_percentage"
|
||||
RecordEnabledKey = "record_enabled"
|
||||
RecordPreserveTimeKey = "record_preserve_time"
|
||||
PingRecordPreserveTimeKey = "ping_record_preserve_time"
|
||||
UpdatedAtKey = "updated_at"
|
||||
XtermjsSettingsKey = "xtermjs_settings"
|
||||
)
|
||||
@@ -0,0 +1,345 @@
|
||||
package corn
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"log"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Func 是 corn 调度器执行的任务函数。
|
||||
// 调度器会为每次执行传入可取消的 context,便于任务在重载或关闭时尽快退出。
|
||||
type Func func(ctx context.Context)
|
||||
|
||||
type job struct {
|
||||
cancel context.CancelFunc
|
||||
}
|
||||
|
||||
type schedule interface {
|
||||
Next(time.Time) time.Time
|
||||
}
|
||||
|
||||
type everySchedule struct {
|
||||
interval time.Duration
|
||||
}
|
||||
|
||||
func (s everySchedule) Next(t time.Time) time.Time {
|
||||
return t.Add(s.interval)
|
||||
}
|
||||
|
||||
type cronSchedule struct {
|
||||
seconds map[int]struct{}
|
||||
minutes map[int]struct{}
|
||||
hours map[int]struct{}
|
||||
dom map[int]struct{}
|
||||
months map[int]struct{}
|
||||
dow map[int]struct{}
|
||||
}
|
||||
|
||||
func (s cronSchedule) Next(t time.Time) time.Time {
|
||||
next := t.Truncate(time.Second).Add(time.Second)
|
||||
limit := next.Add(366 * 24 * time.Hour)
|
||||
for next.Before(limit) {
|
||||
if s.match(next) {
|
||||
return next
|
||||
}
|
||||
next = next.Add(time.Second)
|
||||
}
|
||||
return time.Time{}
|
||||
}
|
||||
|
||||
func (s cronSchedule) match(t time.Time) bool {
|
||||
_, okSecond := s.seconds[t.Second()]
|
||||
_, okMinute := s.minutes[t.Minute()]
|
||||
_, okHour := s.hours[t.Hour()]
|
||||
_, okDay := s.dom[t.Day()]
|
||||
_, okMonth := s.months[int(t.Month())]
|
||||
_, okWeek := s.dow[int(t.Weekday())]
|
||||
return okSecond && okMinute && okHour && okDay && okMonth && okWeek
|
||||
}
|
||||
|
||||
type Manager struct {
|
||||
mu sync.Mutex
|
||||
jobs map[string]job
|
||||
}
|
||||
|
||||
var defaultManager = NewManager()
|
||||
|
||||
func NewManager() *Manager {
|
||||
return &Manager{jobs: make(map[string]job)}
|
||||
}
|
||||
|
||||
// AddFunc 按 cron 表达式注册一个任务,fn 会在独立 goroutine 中执行。
|
||||
// 支持 5 字段、6 字段 cron 表达式,以及 @every 1m 这类固定间隔表达式。
|
||||
func AddFunc(name string, spec string, fn func()) error {
|
||||
return AddContextFunc(name, spec, false, func(context.Context) { fn() })
|
||||
}
|
||||
|
||||
// AddContextFunc 按 cron 表达式注册一个任务,支持传递带 context 的 func。
|
||||
func AddContextFunc(name string, spec string, runImmediately bool, fn Func) error {
|
||||
return defaultManager.AddContextFunc(name, spec, runImmediately, fn)
|
||||
}
|
||||
|
||||
func Every(duration time.Duration) string {
|
||||
return "@every " + duration.String()
|
||||
}
|
||||
|
||||
func Remove(name string) {
|
||||
defaultManager.Remove(name)
|
||||
}
|
||||
|
||||
func RemovePrefix(prefix string) {
|
||||
defaultManager.RemovePrefix(prefix)
|
||||
}
|
||||
|
||||
func StopAll() {
|
||||
defaultManager.StopAll()
|
||||
}
|
||||
|
||||
func (m *Manager) AddFunc(name string, spec string, fn func()) error {
|
||||
return m.AddContextFunc(name, spec, false, func(context.Context) { fn() })
|
||||
}
|
||||
|
||||
func (m *Manager) AddContextFunc(name string, spec string, runImmediately bool, fn Func) error {
|
||||
if name == "" {
|
||||
return fmt.Errorf("corn job name is empty")
|
||||
}
|
||||
s, err := Parse(spec)
|
||||
if err != nil {
|
||||
return fmt.Errorf("corn job %q spec is invalid: %w", name, err)
|
||||
}
|
||||
if fn == nil {
|
||||
return fmt.Errorf("corn job %q func is nil", name)
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
m.replace(name, cancel)
|
||||
|
||||
go m.run(ctx, name, s, runImmediately, fn)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (m *Manager) Remove(name string) {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
|
||||
if old, ok := m.jobs[name]; ok {
|
||||
old.cancel()
|
||||
delete(m.jobs, name)
|
||||
}
|
||||
}
|
||||
|
||||
func (m *Manager) RemovePrefix(prefix string) {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
|
||||
for name, old := range m.jobs {
|
||||
if strings.HasPrefix(name, prefix) {
|
||||
old.cancel()
|
||||
delete(m.jobs, name)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (m *Manager) StopAll() {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
|
||||
for name, old := range m.jobs {
|
||||
old.cancel()
|
||||
delete(m.jobs, name)
|
||||
}
|
||||
}
|
||||
|
||||
func (m *Manager) replace(name string, cancel context.CancelFunc) {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
|
||||
if old, ok := m.jobs[name]; ok {
|
||||
old.cancel()
|
||||
}
|
||||
m.jobs[name] = job{cancel: cancel}
|
||||
}
|
||||
|
||||
func (m *Manager) run(ctx context.Context, name string, s schedule, runImmediately bool, fn Func) {
|
||||
if runImmediately {
|
||||
go safeRun(ctx, name, fn)
|
||||
}
|
||||
|
||||
nextTick := s.Next(time.Now())
|
||||
if nextTick.IsZero() {
|
||||
log.Printf("corn job %s has no next run time", name)
|
||||
return
|
||||
}
|
||||
timer := time.NewTimer(time.Until(nextTick))
|
||||
defer timer.Stop()
|
||||
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-timer.C:
|
||||
go safeRun(ctx, name, fn)
|
||||
nextTick = s.Next(nextTick)
|
||||
if nextTick.IsZero() {
|
||||
return
|
||||
}
|
||||
resetTimer(timer, time.Until(nextTick))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func resetTimer(timer *time.Timer, duration time.Duration) {
|
||||
if duration < 0 {
|
||||
duration = 0
|
||||
}
|
||||
timer.Reset(duration)
|
||||
}
|
||||
|
||||
func safeRun(ctx context.Context, name string, fn Func) {
|
||||
defer func() {
|
||||
if r := recover(); r != nil {
|
||||
log.Printf("corn job %s panic: %v", name, r)
|
||||
}
|
||||
}()
|
||||
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
default:
|
||||
fn(ctx)
|
||||
}
|
||||
}
|
||||
|
||||
// Parse 解析 corn 表达式。
|
||||
// 支持:
|
||||
// - 5 字段:minute hour day-of-month month day-of-week
|
||||
// - 6 字段:second minute hour day-of-month month day-of-week
|
||||
// - @every 1m / @every 30s
|
||||
//
|
||||
// 字段支持 *、*/n、a-b、a-b/n、逗号列表和具体数字。
|
||||
func Parse(spec string) (schedule, error) {
|
||||
spec = strings.TrimSpace(spec)
|
||||
if spec == "" {
|
||||
return nil, fmt.Errorf("empty spec")
|
||||
}
|
||||
if strings.HasPrefix(spec, "@every ") {
|
||||
duration, err := time.ParseDuration(strings.TrimSpace(strings.TrimPrefix(spec, "@every ")))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if duration <= 0 {
|
||||
return nil, fmt.Errorf("@every duration must be positive")
|
||||
}
|
||||
return everySchedule{interval: duration}, nil
|
||||
}
|
||||
|
||||
fields := strings.Fields(spec)
|
||||
if len(fields) == 5 {
|
||||
fields = append([]string{"0"}, fields...)
|
||||
}
|
||||
if len(fields) != 6 {
|
||||
return nil, fmt.Errorf("expected 5 or 6 fields, got %d", len(fields))
|
||||
}
|
||||
|
||||
seconds, err := parseField(fields[0], 0, 59)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("second: %w", err)
|
||||
}
|
||||
minutes, err := parseField(fields[1], 0, 59)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("minute: %w", err)
|
||||
}
|
||||
hours, err := parseField(fields[2], 0, 23)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("hour: %w", err)
|
||||
}
|
||||
dom, err := parseField(fields[3], 1, 31)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("day-of-month: %w", err)
|
||||
}
|
||||
months, err := parseField(fields[4], 1, 12)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("month: %w", err)
|
||||
}
|
||||
dow, err := parseField(fields[5], 0, 7)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("day-of-week: %w", err)
|
||||
}
|
||||
if _, ok := dow[7]; ok {
|
||||
dow[0] = struct{}{}
|
||||
delete(dow, 7)
|
||||
}
|
||||
|
||||
return cronSchedule{seconds: seconds, minutes: minutes, hours: hours, dom: dom, months: months, dow: dow}, nil
|
||||
}
|
||||
|
||||
func parseField(field string, min int, max int) (map[int]struct{}, error) {
|
||||
values := make(map[int]struct{})
|
||||
for _, part := range strings.Split(field, ",") {
|
||||
part = strings.TrimSpace(part)
|
||||
if part == "" {
|
||||
return nil, fmt.Errorf("empty part")
|
||||
}
|
||||
|
||||
base := part
|
||||
step := 1
|
||||
if strings.Contains(part, "/") {
|
||||
parts := strings.Split(part, "/")
|
||||
if len(parts) != 2 {
|
||||
return nil, fmt.Errorf("invalid step %q", part)
|
||||
}
|
||||
base = parts[0]
|
||||
parsedStep, err := strconv.Atoi(parts[1])
|
||||
if err != nil || parsedStep <= 0 {
|
||||
return nil, fmt.Errorf("invalid step %q", parts[1])
|
||||
}
|
||||
step = parsedStep
|
||||
}
|
||||
|
||||
start, end, err := parseRange(base, min, max)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for i := start; i <= end; i += step {
|
||||
values[i] = struct{}{}
|
||||
}
|
||||
}
|
||||
return values, nil
|
||||
}
|
||||
|
||||
func parseRange(base string, min int, max int) (int, int, error) {
|
||||
if base == "*" || base == "" {
|
||||
return min, max, nil
|
||||
}
|
||||
if strings.Contains(base, "-") {
|
||||
parts := strings.Split(base, "-")
|
||||
if len(parts) != 2 {
|
||||
return 0, 0, fmt.Errorf("invalid range %q", base)
|
||||
}
|
||||
start, err := strconv.Atoi(parts[0])
|
||||
if err != nil {
|
||||
return 0, 0, fmt.Errorf("invalid range start %q", parts[0])
|
||||
}
|
||||
end, err := strconv.Atoi(parts[1])
|
||||
if err != nil {
|
||||
return 0, 0, fmt.Errorf("invalid range end %q", parts[1])
|
||||
}
|
||||
if start > end || start < min || end > max {
|
||||
return 0, 0, fmt.Errorf("range %q out of bounds %d-%d", base, min, max)
|
||||
}
|
||||
return start, end, nil
|
||||
}
|
||||
|
||||
value, err := strconv.Atoi(base)
|
||||
if err != nil {
|
||||
return 0, 0, fmt.Errorf("invalid value %q", base)
|
||||
}
|
||||
if value < min || value > max {
|
||||
return 0, 0, fmt.Errorf("value %d out of bounds %d-%d", value, min, max)
|
||||
}
|
||||
return value, value, nil
|
||||
}
|
||||
@@ -0,0 +1,270 @@
|
||||
package metric
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"math"
|
||||
"sort"
|
||||
"time"
|
||||
)
|
||||
|
||||
// AggregatePoints groups raw points into time buckets and computes aggregate values.
|
||||
//
|
||||
// AggregatePoints 在内存中按查询配置将原始点分桶并聚合。
|
||||
func AggregatePoints(points []Point, query AggregateQuery) ([]AggregatePoint, error) {
|
||||
if err := query.Validate(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if len(points) == 0 {
|
||||
if query.FillEmpty {
|
||||
return emptyBuckets(query), nil
|
||||
}
|
||||
return []AggregatePoint{}, nil
|
||||
}
|
||||
|
||||
points = sortedPoints(points)
|
||||
groups := make(map[int64][]Point)
|
||||
for _, point := range points {
|
||||
bucket := alignTime(point.Timestamp, query.Interval).UnixNano()
|
||||
groups[bucket] = append(groups[bucket], point)
|
||||
}
|
||||
|
||||
var buckets []int64
|
||||
if query.FillEmpty {
|
||||
for t := alignTime(query.Start, query.Interval); !t.After(query.End); t = t.Add(query.Interval) {
|
||||
buckets = append(buckets, t.UnixNano())
|
||||
}
|
||||
} else {
|
||||
for bucket := range groups {
|
||||
buckets = append(buckets, bucket)
|
||||
}
|
||||
sort.Slice(buckets, func(i, j int) bool { return buckets[i] < buckets[j] })
|
||||
}
|
||||
|
||||
out := make([]AggregatePoint, 0, len(buckets))
|
||||
for _, bucket := range buckets {
|
||||
group := groups[bucket]
|
||||
value, err := aggregateValue(group, query.Aggregation)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out = append(out, AggregatePoint{
|
||||
MetricName: query.MetricName,
|
||||
EntityID: query.EntityID,
|
||||
Bucket: time.Unix(0, bucket).UTC(),
|
||||
Value: value,
|
||||
Count: len(group),
|
||||
})
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// CalculateStats computes summary statistics for a point series.
|
||||
//
|
||||
// CalculateStats 基于一组点计算统计摘要,包括均值、百分位、首尾值和标准差。
|
||||
func CalculateStats(points []Point) (Stats, error) {
|
||||
if len(points) == 0 {
|
||||
// Distinguish "the metric/range yielded no samples" from "the metric
|
||||
// definition does not exist" (which surfaces as ErrNotFound elsewhere).
|
||||
return Stats{}, ErrNoData
|
||||
}
|
||||
points = sortedPoints(points)
|
||||
avg, _ := aggregateValue(points, AggAvg)
|
||||
min, _ := aggregateValue(points, AggMin)
|
||||
max, _ := aggregateValue(points, AggMax)
|
||||
sum, _ := aggregateValue(points, AggSum)
|
||||
p50, _ := aggregateValue(points, AggP50)
|
||||
p95, _ := aggregateValue(points, AggP95)
|
||||
p99, _ := aggregateValue(points, AggP99)
|
||||
first, _ := aggregateValue(points, AggFirst)
|
||||
last, _ := aggregateValue(points, AggLast)
|
||||
rate, _ := aggregateValue(points, AggRate)
|
||||
|
||||
var variance float64
|
||||
for _, p := range points {
|
||||
diff := p.Value - avg
|
||||
variance += diff * diff
|
||||
}
|
||||
|
||||
return Stats{
|
||||
Count: len(points),
|
||||
Min: min,
|
||||
Max: max,
|
||||
Avg: avg,
|
||||
Sum: sum,
|
||||
P50: p50,
|
||||
P95: p95,
|
||||
P99: p99,
|
||||
First: first,
|
||||
Last: last,
|
||||
Rate: rate,
|
||||
Start: points[0].Timestamp,
|
||||
End: points[len(points)-1].Timestamp,
|
||||
StdDev: math.Sqrt(variance / float64(len(points))),
|
||||
}, nil
|
||||
}
|
||||
|
||||
// aggregateValue computes one aggregation over a point group.
|
||||
//
|
||||
// aggregateValue 对一组点执行单个聚合类型的计算。
|
||||
func aggregateValue(points []Point, agg Aggregation) (float64, error) {
|
||||
if len(points) == 0 {
|
||||
return 0, nil
|
||||
}
|
||||
switch agg {
|
||||
case AggAvg:
|
||||
sum, _ := aggregateValue(points, AggSum)
|
||||
return sum / float64(len(points)), nil
|
||||
case AggMin:
|
||||
v := points[0].Value
|
||||
for _, p := range points[1:] {
|
||||
v = math.Min(v, p.Value)
|
||||
}
|
||||
return v, nil
|
||||
case AggMax:
|
||||
v := points[0].Value
|
||||
for _, p := range points[1:] {
|
||||
v = math.Max(v, p.Value)
|
||||
}
|
||||
return v, nil
|
||||
case AggSum:
|
||||
var sum float64
|
||||
for _, p := range points {
|
||||
sum += p.Value
|
||||
}
|
||||
return sum, nil
|
||||
case AggCount:
|
||||
return float64(len(points)), nil
|
||||
case AggFirst:
|
||||
// Callers (AggregatePoints, CalculateStats) pass time-ordered slices.
|
||||
return points[0].Value, nil
|
||||
case AggLast:
|
||||
return points[len(points)-1].Value, nil
|
||||
case AggRate:
|
||||
return counterRate(points), nil
|
||||
case AggStdDev:
|
||||
return stdDevPop(points), nil
|
||||
default:
|
||||
// Any percentile (p50, p95, p99, and arbitrary pXX / pXX.X) is computed
|
||||
// here via linear interpolation over the sorted values. The fixed
|
||||
// AggP50/AggP95/AggP99 constants are just common cases of this.
|
||||
if frac, ok := parsePercentile(agg); ok {
|
||||
return percentile(points, frac), nil
|
||||
}
|
||||
return 0, fmt.Errorf("%w: unsupported aggregation %q", ErrInvalidArgument, agg)
|
||||
}
|
||||
}
|
||||
|
||||
// counterRate computes a per-second rate of change that is resilient to counter
|
||||
// resets. It walks the time-ordered series and sums only positive deltas; a
|
||||
// decrease is treated as a reset (the counter restarted) and contributes zero
|
||||
// rather than a negative spike. For a strictly increasing counter this equals
|
||||
// (last-first)/seconds. For a gauge it yields the total upward movement per
|
||||
// second, which is a stable definition for an otherwise ill-defined quantity.
|
||||
//
|
||||
// counterRate 计算能抵抗计数器重置的每秒变化率。它会遍历按时间排序的序列,
|
||||
// 并且只累加正向增量;当值下降时会被视为重置(计数器重新开始),该段贡献为零,
|
||||
// 而不是产生负向尖峰。对于严格递增的计数器,这等于 (last-first)/seconds。
|
||||
// 对于 gauge,它表示每秒总上升量,为这种本来定义不明确的量提供稳定定义。
|
||||
func counterRate(points []Point) float64 {
|
||||
if len(points) < 2 {
|
||||
return 0
|
||||
}
|
||||
// Input is time-ordered by the callers; avoid an extra sort here.
|
||||
seconds := points[len(points)-1].Timestamp.Sub(points[0].Timestamp).Seconds()
|
||||
if seconds <= 0 {
|
||||
return 0
|
||||
}
|
||||
var increase float64
|
||||
for i := 1; i < len(points); i++ {
|
||||
delta := points[i].Value - points[i-1].Value
|
||||
if delta > 0 {
|
||||
increase += delta
|
||||
}
|
||||
}
|
||||
return increase / seconds
|
||||
}
|
||||
|
||||
// stdDevPop computes the population standard deviation (dividing by N), matching
|
||||
// SQL STDDEV_POP and the StdDev field produced by CalculateStats.
|
||||
//
|
||||
// stdDevPop 计算总体标准差(除以 N),与 SQL 的 STDDEV_POP 以及
|
||||
// CalculateStats 返回的 StdDev 语义保持一致。
|
||||
func stdDevPop(points []Point) float64 {
|
||||
if len(points) == 0 {
|
||||
return 0
|
||||
}
|
||||
var sum float64
|
||||
for _, p := range points {
|
||||
sum += p.Value
|
||||
}
|
||||
mean := sum / float64(len(points))
|
||||
var variance float64
|
||||
for _, p := range points {
|
||||
d := p.Value - mean
|
||||
variance += d * d
|
||||
}
|
||||
return math.Sqrt(variance / float64(len(points)))
|
||||
}
|
||||
|
||||
// percentile computes a quantile from point values.
|
||||
//
|
||||
// percentile 提取点值、排序,并计算指定小数形式的百分位。
|
||||
func percentile(points []Point, p float64) float64 {
|
||||
values := make([]float64, len(points))
|
||||
for i, point := range points {
|
||||
values[i] = point.Value
|
||||
}
|
||||
sort.Float64s(values)
|
||||
return percentileSorted(values, p)
|
||||
}
|
||||
|
||||
// percentileSorted returns the linear-interpolation percentile of an
|
||||
// already-sorted slice. Shared by percentile() and the raw-value paths so the
|
||||
// interpolation method stays identical everywhere.
|
||||
//
|
||||
// percentileSorted 基于已排序切片用线性插值计算百分位,供原始值路径和
|
||||
// percentile 共用,确保所有路径的插值方法一致。
|
||||
func percentileSorted(values []float64, p float64) float64 {
|
||||
if len(values) == 0 {
|
||||
return 0
|
||||
}
|
||||
if len(values) == 1 {
|
||||
return values[0]
|
||||
}
|
||||
pos := p * float64(len(values)-1)
|
||||
lower := int(math.Floor(pos))
|
||||
upper := int(math.Ceil(pos))
|
||||
if lower == upper {
|
||||
return values[lower]
|
||||
}
|
||||
weight := pos - float64(lower)
|
||||
return values[lower]*(1-weight) + values[upper]*weight
|
||||
}
|
||||
|
||||
// alignTime floors a timestamp to the start of its interval bucket.
|
||||
//
|
||||
// alignTime 将时间向下对齐到指定间隔的桶起点。
|
||||
func alignTime(t time.Time, interval time.Duration) time.Time {
|
||||
nano := t.UTC().UnixNano()
|
||||
size := interval.Nanoseconds()
|
||||
// Floor division toward negative infinity so timestamps before the Unix
|
||||
// epoch (negative nanos) align to the bucket start rather than rounding up.
|
||||
// Go's % returns a remainder with the dividend's sign, so normalize it.
|
||||
rem := ((nano % size) + size) % size
|
||||
return time.Unix(0, nano-rem).UTC()
|
||||
}
|
||||
|
||||
// emptyBuckets builds zero-count aggregate buckets for an empty range.
|
||||
//
|
||||
// emptyBuckets 根据查询范围生成空聚合桶,用于 FillEmpty 场景。
|
||||
func emptyBuckets(query AggregateQuery) []AggregatePoint {
|
||||
var out []AggregatePoint
|
||||
for t := alignTime(query.Start, query.Interval); !t.After(query.End); t = t.Add(query.Interval) {
|
||||
out = append(out, AggregatePoint{
|
||||
MetricName: query.MetricName,
|
||||
EntityID: query.EntityID,
|
||||
Bucket: t,
|
||||
})
|
||||
}
|
||||
return out
|
||||
}
|
||||
@@ -0,0 +1,485 @@
|
||||
package metric
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"fmt"
|
||||
"net/url"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Config describes database, pooling, migration, retention, and rollup settings.
|
||||
//
|
||||
// Config 描述 Store 的数据库后端、连接池、迁移、保留和 rollup 配置。
|
||||
type Config struct {
|
||||
// Driver selects the database backend.
|
||||
//
|
||||
// Driver 选择数据库后端。
|
||||
Driver Driver
|
||||
// DSN is the database connection string when DB is not supplied.
|
||||
//
|
||||
// DSN 是未传入 DB 时使用的数据库连接字符串。
|
||||
DSN string
|
||||
|
||||
// DB can be supplied by the host app when it owns the connection pool.
|
||||
// When DB is set, DSN is ignored and Close leaves the supplied DB open.
|
||||
//
|
||||
// 当宿主程序自己管理连接池时可以传入 DB;设置 DB 后会忽略 DSN,
|
||||
// Close 也不会关闭这个外部传入的连接池。
|
||||
DB *sql.DB
|
||||
|
||||
// TablePrefix prefixes every table managed by the store.
|
||||
//
|
||||
// TablePrefix 是 Store 管理的所有表名前缀。
|
||||
TablePrefix string
|
||||
// DefaultRetentionDays is used when a metric definition omits retention.
|
||||
//
|
||||
// DefaultRetentionDays 在指标定义未设置保留时间时使用。
|
||||
DefaultRetentionDays int
|
||||
// AutoMigrate controls whether Open creates or updates the schema.
|
||||
//
|
||||
// AutoMigrate 控制 Open 是否创建或更新表结构。
|
||||
AutoMigrate bool
|
||||
// MaxOpenConns sets the primary pool's maximum open connections.
|
||||
//
|
||||
// MaxOpenConns 设置主连接池的最大打开连接数。
|
||||
MaxOpenConns int
|
||||
// MaxIdleConns sets the primary pool's maximum idle connections.
|
||||
//
|
||||
// MaxIdleConns 设置主连接池的最大空闲连接数。
|
||||
MaxIdleConns int
|
||||
// ConnMaxLifetime sets the maximum lifetime for pooled connections.
|
||||
//
|
||||
// ConnMaxLifetime 设置池化连接的最大生命周期。
|
||||
ConnMaxLifetime time.Duration
|
||||
// ConnectTimeout bounds the initial database ping in Open.
|
||||
//
|
||||
// ConnectTimeout 限制 Open 中首次 ping 数据库的时间。
|
||||
ConnectTimeout time.Duration
|
||||
// SQLite holds SQLite-specific options.
|
||||
//
|
||||
// SQLite 保存 SQLite 专用选项。
|
||||
SQLite SQLiteOptions
|
||||
|
||||
// RollupPolicy configures downsampling tiers and tiered retention. When it
|
||||
// defines tiers, Store.Compact materializes them and enforces every
|
||||
// retention window. The zero value disables rollups (raw points only).
|
||||
//
|
||||
// RollupPolicy 配置降采样层级和分层保留时间;定义层级后,
|
||||
// Store.Compact 会生成 rollup 并执行保留策略。零值表示禁用 rollup。
|
||||
RollupPolicy RollupPolicy
|
||||
}
|
||||
|
||||
// SQLiteOptions contains SQLite-specific performance and read-concurrency settings.
|
||||
//
|
||||
// SQLiteOptions 保存 SQLite 专用的性能和并发读取选项。
|
||||
type SQLiteOptions struct {
|
||||
// PerformanceProfile selects the SQLite synchronous durability preset.
|
||||
//
|
||||
// PerformanceProfile 选择 SQLite synchronous 持久化预设。
|
||||
PerformanceProfile SQLitePerformanceProfile
|
||||
// BusyTimeout is applied to SQLite busy_timeout.
|
||||
//
|
||||
// BusyTimeout 会应用到 SQLite busy_timeout。
|
||||
BusyTimeout time.Duration
|
||||
// CacheSizeKB sets the SQLite page cache size in KB.
|
||||
//
|
||||
// CacheSizeKB 设置 SQLite 页缓存大小,单位为 KB。
|
||||
CacheSizeKB int
|
||||
// PageSize sets SQLite page_size when positive.
|
||||
//
|
||||
// PageSize 为正数时设置 SQLite page_size。
|
||||
PageSize int
|
||||
// TempStoreMemory enables memory-backed temporary storage.
|
||||
//
|
||||
// TempStoreMemory 启用基于内存的临时存储。
|
||||
TempStoreMemory bool
|
||||
// MMapSizeBytes sets SQLite mmap_size in bytes.
|
||||
//
|
||||
// MMapSizeBytes 设置 SQLite mmap_size,单位为字节。
|
||||
MMapSizeBytes int64
|
||||
// WALAutoCheckpoint sets the SQLite WAL auto-checkpoint page count.
|
||||
//
|
||||
// WALAutoCheckpoint 设置 SQLite WAL 自动 checkpoint 页数。
|
||||
WALAutoCheckpoint int
|
||||
// ReadPoolSize, when > 0, opens a second read-only connection pool with
|
||||
// this many connections for SELECT-style calls. SQLite serializes writes,
|
||||
// but WAL lets readers run concurrently, so a read pool lifts read
|
||||
// throughput while keeping writes on the single primary connection.
|
||||
// 0 (default) means all calls share the single primary pool, preserving
|
||||
// the previous single-connection behavior.
|
||||
//
|
||||
// ReadPoolSize 大于 0 时会为 SELECT 类调用打开独立只读连接池。
|
||||
// SQLite 写入会被串行化,但 WAL 允许读取并发执行,因此读池可以在写入仍走
|
||||
// 单主连接的同时提升读取吞吐。0(默认值)表示所有调用共享单个主连接池,
|
||||
// 保留之前的单连接行为。
|
||||
ReadPoolSize int
|
||||
}
|
||||
|
||||
// SQLitePerformanceProfile names SQLite durability and performance presets.
|
||||
//
|
||||
// SQLitePerformanceProfile 表示 SQLite synchronous 等持久化策略预设。
|
||||
type SQLitePerformanceProfile string
|
||||
|
||||
const (
|
||||
// SQLiteProfileDefault uses the package default SQLite profile.
|
||||
//
|
||||
// SQLiteProfileDefault 使用包默认的 SQLite 配置档。
|
||||
SQLiteProfileDefault SQLitePerformanceProfile = ""
|
||||
// SQLiteProfileBalanced uses NORMAL synchronous mode.
|
||||
//
|
||||
// SQLiteProfileBalanced 使用 NORMAL synchronous 模式。
|
||||
SQLiteProfileBalanced SQLitePerformanceProfile = "balanced"
|
||||
// SQLiteProfilePerformance favors throughput over durability.
|
||||
//
|
||||
// SQLiteProfilePerformance 优先吞吐而非持久性。
|
||||
SQLiteProfilePerformance SQLitePerformanceProfile = "performance"
|
||||
// SQLiteProfileDurable favors durability over write throughput.
|
||||
//
|
||||
// SQLiteProfileDurable 优先持久性而非写入吞吐。
|
||||
SQLiteProfileDurable SQLitePerformanceProfile = "durable"
|
||||
)
|
||||
|
||||
// Option mutates a Config value.
|
||||
//
|
||||
// Option 是用于修改 Config 的函数式选项。
|
||||
type Option func(*Config)
|
||||
|
||||
// DefaultConfig returns the default configuration for a backend.
|
||||
//
|
||||
// DefaultConfig 返回指定后端的默认配置。
|
||||
func DefaultConfig(driver Driver, dsn string) Config {
|
||||
return Config{
|
||||
Driver: driver,
|
||||
DSN: dsn,
|
||||
TablePrefix: "metric_",
|
||||
DefaultRetentionDays: 90,
|
||||
AutoMigrate: true,
|
||||
MaxOpenConns: 25,
|
||||
MaxIdleConns: 5,
|
||||
ConnMaxLifetime: time.Hour,
|
||||
ConnectTimeout: 10 * time.Second,
|
||||
SQLite: SQLiteOptions{
|
||||
PerformanceProfile: SQLiteProfileBalanced,
|
||||
BusyTimeout: 5 * time.Second,
|
||||
CacheSizeKB: 64 * 1024,
|
||||
TempStoreMemory: true,
|
||||
MMapSizeBytes: 256 * 1024 * 1024,
|
||||
WALAutoCheckpoint: 1000,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
// Backend builds a generic backend configuration and applies options.
|
||||
//
|
||||
// Backend 构造通用后端配置,并应用额外选项。
|
||||
func Backend(driver Driver, dsn string, opts ...Option) Config {
|
||||
cfg := DefaultConfig(driver, dsn)
|
||||
applyOptions(&cfg, opts...)
|
||||
return cfg
|
||||
}
|
||||
|
||||
// SQLite builds a SQLite backend configuration.
|
||||
//
|
||||
// SQLite 构造 SQLite 后端配置。
|
||||
func SQLite(dsn string, opts ...Option) Config {
|
||||
cfg := SQLiteConfig(dsn)
|
||||
applyOptions(&cfg, opts...)
|
||||
return cfg
|
||||
}
|
||||
|
||||
// SQLiteInDir builds a managed SQLite file configuration rooted at a directory.
|
||||
//
|
||||
// SQLiteInDir 构造由 package 管理目录的 SQLite 文件数据库配置。
|
||||
func SQLiteInDir(dir string, opts ...Option) Config {
|
||||
cfg := SQLiteConfig(sqliteFileDSN(filepath.Join(dir, "metrics.db")))
|
||||
cfg.MaxOpenConns = 1
|
||||
cfg.MaxIdleConns = 1
|
||||
applyOptions(&cfg, opts...)
|
||||
return cfg
|
||||
}
|
||||
|
||||
// MySQL builds a MySQL backend configuration.
|
||||
//
|
||||
// MySQL 构造 MySQL 后端配置。
|
||||
func MySQL(dsn string, opts ...Option) Config {
|
||||
cfg := MySQLConfig(dsn)
|
||||
applyOptions(&cfg, opts...)
|
||||
return cfg
|
||||
}
|
||||
|
||||
// PostgreSQL builds a PostgreSQL backend configuration.
|
||||
//
|
||||
// PostgreSQL 构造 PostgreSQL 后端配置。
|
||||
func PostgreSQL(dsn string, opts ...Option) Config {
|
||||
cfg := PostgreSQLConfig(dsn)
|
||||
applyOptions(&cfg, opts...)
|
||||
return cfg
|
||||
}
|
||||
|
||||
// SQLiteConfig returns the default SQLite configuration.
|
||||
//
|
||||
// SQLiteConfig 返回 SQLite 后端的默认配置。
|
||||
func SQLiteConfig(dsn string) Config {
|
||||
cfg := DefaultConfig(DriverSQLite, dsn)
|
||||
if cfg.DSN == "" {
|
||||
cfg.DSN = "file:metric?mode=memory&cache=shared"
|
||||
}
|
||||
// SQLite serializes writes, so a single connection is the safe default and
|
||||
// avoids "database is locked" contention. Callers can still override with
|
||||
// WithMaxOpenConns/WithMaxIdleConns. Setting it here (rather than inferring
|
||||
// it later from the 25/5 defaults) means an explicit WithMaxOpenConns(25)
|
||||
// is honored instead of being mistaken for "unset".
|
||||
cfg.MaxOpenConns = 1
|
||||
cfg.MaxIdleConns = 1
|
||||
return cfg
|
||||
}
|
||||
|
||||
// MySQLConfig returns the default MySQL configuration.
|
||||
//
|
||||
// MySQLConfig 返回 MySQL 后端的默认配置。
|
||||
func MySQLConfig(dsn string) Config {
|
||||
return DefaultConfig(DriverMySQL, dsn)
|
||||
}
|
||||
|
||||
// PostgreSQLConfig returns the default PostgreSQL configuration.
|
||||
//
|
||||
// PostgreSQLConfig 返回 PostgreSQL 后端的默认配置。
|
||||
func PostgreSQLConfig(dsn string) Config {
|
||||
return DefaultConfig(DriverPostgreSQL, dsn)
|
||||
}
|
||||
|
||||
// WithDB sets a caller-owned database connection pool.
|
||||
//
|
||||
// WithDB 设置由调用方提供的数据库连接池。
|
||||
func WithDB(db *sql.DB) Option {
|
||||
return func(c *Config) {
|
||||
c.DB = db
|
||||
}
|
||||
}
|
||||
|
||||
// WithTablePrefix sets the table-name prefix used by metric tables.
|
||||
//
|
||||
// WithTablePrefix 设置 metric 私有表的表名前缀。
|
||||
func WithTablePrefix(prefix string) Option {
|
||||
return func(c *Config) {
|
||||
c.TablePrefix = prefix
|
||||
}
|
||||
}
|
||||
|
||||
// WithDefaultRetention sets the default metric retention in days.
|
||||
//
|
||||
// WithDefaultRetention 设置指标默认保留天数。
|
||||
func WithDefaultRetention(days int) Option {
|
||||
return func(c *Config) {
|
||||
c.DefaultRetentionDays = days
|
||||
}
|
||||
}
|
||||
|
||||
// WithAutoMigrate controls whether Open creates the schema automatically.
|
||||
//
|
||||
// WithAutoMigrate 控制 Open 时是否自动创建表结构。
|
||||
func WithAutoMigrate(enabled bool) Option {
|
||||
return func(c *Config) {
|
||||
c.AutoMigrate = enabled
|
||||
}
|
||||
}
|
||||
|
||||
// WithMaxOpenConns sets the maximum number of open database connections.
|
||||
//
|
||||
// WithMaxOpenConns 设置底层数据库连接池的最大打开连接数。
|
||||
func WithMaxOpenConns(n int) Option {
|
||||
return func(c *Config) {
|
||||
c.MaxOpenConns = n
|
||||
}
|
||||
}
|
||||
|
||||
// WithMaxIdleConns sets the maximum number of idle database connections.
|
||||
//
|
||||
// WithMaxIdleConns 设置底层数据库连接池的最大空闲连接数。
|
||||
func WithMaxIdleConns(n int) Option {
|
||||
return func(c *Config) {
|
||||
c.MaxIdleConns = n
|
||||
}
|
||||
}
|
||||
|
||||
// WithConnMaxLifetime sets the maximum lifetime for pooled connections.
|
||||
//
|
||||
// WithConnMaxLifetime 设置连接最大复用时间。
|
||||
func WithConnMaxLifetime(d time.Duration) Option {
|
||||
return func(c *Config) {
|
||||
c.ConnMaxLifetime = d
|
||||
}
|
||||
}
|
||||
|
||||
// WithConnectTimeout sets the timeout used while opening the store.
|
||||
//
|
||||
// WithConnectTimeout 设置打开 Store 时 ping 数据库的超时时间。
|
||||
func WithConnectTimeout(d time.Duration) Option {
|
||||
return func(c *Config) {
|
||||
c.ConnectTimeout = d
|
||||
}
|
||||
}
|
||||
|
||||
// WithSQLiteProfile sets the SQLite durability and performance profile.
|
||||
//
|
||||
// WithSQLiteProfile 设置 SQLite 持久化性能预设。
|
||||
func WithSQLiteProfile(profile SQLitePerformanceProfile) Option {
|
||||
return func(c *Config) {
|
||||
c.SQLite.PerformanceProfile = profile
|
||||
}
|
||||
}
|
||||
|
||||
// WithSQLiteBusyTimeout sets SQLite busy_timeout.
|
||||
//
|
||||
// WithSQLiteBusyTimeout 设置 SQLite busy_timeout。
|
||||
func WithSQLiteBusyTimeout(d time.Duration) Option {
|
||||
return func(c *Config) {
|
||||
c.SQLite.BusyTimeout = d
|
||||
}
|
||||
}
|
||||
|
||||
// WithSQLiteCacheSizeKB sets the SQLite page cache size in KB.
|
||||
//
|
||||
// WithSQLiteCacheSizeKB 设置 SQLite 页缓存大小,单位为 KB。
|
||||
func WithSQLiteCacheSizeKB(kb int) Option {
|
||||
return func(c *Config) {
|
||||
c.SQLite.CacheSizeKB = kb
|
||||
}
|
||||
}
|
||||
|
||||
// WithSQLiteMMapSize sets SQLite mmap_size.
|
||||
//
|
||||
// WithSQLiteMMapSize 设置 SQLite mmap_size。
|
||||
func WithSQLiteMMapSize(bytes int64) Option {
|
||||
return func(c *Config) {
|
||||
c.SQLite.MMapSizeBytes = bytes
|
||||
}
|
||||
}
|
||||
|
||||
// WithSQLitePageSize sets SQLite page_size.
|
||||
//
|
||||
// WithSQLitePageSize 设置 SQLite page_size。
|
||||
func WithSQLitePageSize(bytes int) Option {
|
||||
return func(c *Config) {
|
||||
c.SQLite.PageSize = bytes
|
||||
}
|
||||
}
|
||||
|
||||
// WithSQLiteTempStoreMemory controls whether SQLite uses memory for temporary storage.
|
||||
//
|
||||
// WithSQLiteTempStoreMemory 设置 SQLite 是否使用内存临时存储。
|
||||
func WithSQLiteTempStoreMemory(enabled bool) Option {
|
||||
return func(c *Config) {
|
||||
c.SQLite.TempStoreMemory = enabled
|
||||
}
|
||||
}
|
||||
|
||||
// WithSQLiteWALAutoCheckpoint sets the SQLite WAL auto-checkpoint page count.
|
||||
//
|
||||
// WithSQLiteWALAutoCheckpoint 设置 SQLite WAL 自动 checkpoint 页数。
|
||||
func WithSQLiteWALAutoCheckpoint(pages int) Option {
|
||||
return func(c *Config) {
|
||||
c.SQLite.WALAutoCheckpoint = pages
|
||||
}
|
||||
}
|
||||
|
||||
// WithSQLiteReadPool enables a dedicated read-only connection pool of n
|
||||
// connections for SQLite. Writes stay on the single primary connection
|
||||
// (SQLite serializes them); reads fan out across the pool, which WAL mode
|
||||
// allows to run concurrently. Pass n <= 1 to disable (the default).
|
||||
//
|
||||
// WithSQLiteReadPool 为 SQLite 启用包含 n 个连接的独立只读连接池。写入仍走
|
||||
// 单主连接(SQLite 会串行化写入);读取会分散到该连接池中,WAL 模式允许它们
|
||||
// 并发执行。传入 n <= 1 会禁用该功能(默认行为)。
|
||||
func WithSQLiteReadPool(n int) Option {
|
||||
return func(c *Config) {
|
||||
c.SQLite.ReadPoolSize = n
|
||||
}
|
||||
}
|
||||
|
||||
// WithRollupPolicy sets the downsampling/retention ladder. Compact uses it to
|
||||
// build rollup tiers (each progressively coarser and longer-lived) and to age
|
||||
// out raw points and expired tiers.
|
||||
//
|
||||
// WithRollupPolicy 设置降采样与保留时间阶梯;Compact 会据此构建逐级
|
||||
// 更粗、保留更久的 rollup,并清理过期原始点和过期层级。
|
||||
func WithRollupPolicy(p RollupPolicy) Option {
|
||||
return func(c *Config) {
|
||||
c.RollupPolicy = p
|
||||
}
|
||||
}
|
||||
|
||||
// applyOptions applies configuration options in order.
|
||||
//
|
||||
// applyOptions 将一组选项依次应用到 Config。
|
||||
func applyOptions(cfg *Config, opts ...Option) {
|
||||
for _, opt := range opts {
|
||||
if opt != nil {
|
||||
opt(cfg)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Validate checks whether the value is well formed.
|
||||
//
|
||||
// Validate 检查 Config 的后端、表名前缀、保留时间和 rollup 策略是否合法。
|
||||
func (c Config) Validate() error {
|
||||
switch c.Driver {
|
||||
case DriverSQLite, DriverMySQL, DriverPostgreSQL:
|
||||
default:
|
||||
return fmt.Errorf("%w: unsupported driver %q", ErrInvalidArgument, c.Driver)
|
||||
}
|
||||
if c.DB == nil && strings.TrimSpace(c.DSN) == "" {
|
||||
return fmt.Errorf("%w: dsn is required when db is not supplied", ErrInvalidArgument)
|
||||
}
|
||||
if c.TablePrefix == "" {
|
||||
return fmt.Errorf("%w: table prefix cannot be empty", ErrInvalidArgument)
|
||||
}
|
||||
for _, r := range c.TablePrefix {
|
||||
if !(r == '_' || r >= '0' && r <= '9' || r >= 'A' && r <= 'Z' || r >= 'a' && r <= 'z') {
|
||||
return fmt.Errorf("%w: table prefix must contain only letters, digits, and underscores", ErrInvalidArgument)
|
||||
}
|
||||
}
|
||||
if c.DefaultRetentionDays <= 0 {
|
||||
return fmt.Errorf("%w: default retention days must be positive", ErrInvalidArgument)
|
||||
}
|
||||
if err := c.RollupPolicy.Validate(); err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// driverName returns the database/sql driver name for the configured backend.
|
||||
//
|
||||
// driverName 返回 database/sql 注册使用的驱动名称。
|
||||
func (c Config) driverName() string {
|
||||
switch c.Driver {
|
||||
case DriverSQLite:
|
||||
return "sqlite3"
|
||||
case DriverMySQL:
|
||||
return "mysql"
|
||||
case DriverPostgreSQL:
|
||||
return "pgx"
|
||||
default:
|
||||
return string(c.Driver)
|
||||
}
|
||||
}
|
||||
|
||||
// sqliteFileDSN converts a filesystem path into a SQLite file DSN.
|
||||
//
|
||||
// sqliteFileDSN 将文件路径转换为 SQLite file: DSN。
|
||||
func sqliteFileDSN(path string) string {
|
||||
return "file:" + filepath.ToSlash(path) + "?cache=shared&mode=rwc"
|
||||
}
|
||||
|
||||
// appendSQLiteDSNParam appends a query parameter to a SQLite DSN.
|
||||
//
|
||||
// appendSQLiteDSNParam 向 SQLite DSN 追加查询参数。
|
||||
func appendSQLiteDSNParam(dsn, key, value string) string {
|
||||
if strings.Contains(dsn, "?") {
|
||||
return dsn + "&" + url.QueryEscape(key) + "=" + url.QueryEscape(value)
|
||||
}
|
||||
return dsn + "?" + url.QueryEscape(key) + "=" + url.QueryEscape(value)
|
||||
}
|
||||
@@ -0,0 +1,68 @@
|
||||
package metric
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// TestBackendBuildersApplyOptions verifies backend builders apply options.
|
||||
//
|
||||
// TestBackendBuildersApplyOptions 验证各后端构造器会正确应用选项。
|
||||
func TestBackendBuildersApplyOptions(t *testing.T) {
|
||||
cfg := SQLite(
|
||||
"file:metrics.db?cache=shared",
|
||||
WithTablePrefix("x_metric_"),
|
||||
WithDefaultRetention(180),
|
||||
WithAutoMigrate(false),
|
||||
WithMaxOpenConns(10),
|
||||
WithMaxIdleConns(2),
|
||||
WithConnMaxLifetime(30*time.Minute),
|
||||
WithConnectTimeout(2*time.Second),
|
||||
)
|
||||
|
||||
if cfg.Driver != DriverSQLite {
|
||||
t.Fatalf("expected sqlite driver, got %q", cfg.Driver)
|
||||
}
|
||||
if cfg.TablePrefix != "x_metric_" {
|
||||
t.Fatalf("table prefix option was not applied: %q", cfg.TablePrefix)
|
||||
}
|
||||
if cfg.DefaultRetentionDays != 180 {
|
||||
t.Fatalf("default retention option was not applied: %d", cfg.DefaultRetentionDays)
|
||||
}
|
||||
if cfg.AutoMigrate {
|
||||
t.Fatalf("auto migrate option was not applied")
|
||||
}
|
||||
if cfg.MaxOpenConns != 10 || cfg.MaxIdleConns != 2 {
|
||||
t.Fatalf("pool options were not applied: %#v", cfg)
|
||||
}
|
||||
if cfg.ConnMaxLifetime != 30*time.Minute || cfg.ConnectTimeout != 2*time.Second {
|
||||
t.Fatalf("timeout options were not applied: %#v", cfg)
|
||||
}
|
||||
}
|
||||
|
||||
// TestSQLiteInDirBuildsFileConfig verifies SQLiteInDir file configuration.
|
||||
//
|
||||
// TestSQLiteInDirBuildsFileConfig 验证 SQLiteInDir 会生成预期的文件数据库配置。
|
||||
func TestSQLiteInDirBuildsFileConfig(t *testing.T) {
|
||||
cfg := SQLiteInDir(
|
||||
"data/metrics",
|
||||
WithSQLiteProfile(SQLiteProfilePerformance),
|
||||
WithSQLiteCacheSizeKB(128*1024),
|
||||
)
|
||||
|
||||
if cfg.Driver != DriverSQLite {
|
||||
t.Fatalf("expected sqlite driver, got %q", cfg.Driver)
|
||||
}
|
||||
if cfg.MaxOpenConns != 1 || cfg.MaxIdleConns != 1 {
|
||||
t.Fatalf("sqlite directory config should default to a single connection: %#v", cfg)
|
||||
}
|
||||
if cfg.SQLite.PerformanceProfile != SQLiteProfilePerformance {
|
||||
t.Fatalf("sqlite profile option was not applied: %q", cfg.SQLite.PerformanceProfile)
|
||||
}
|
||||
if cfg.SQLite.CacheSizeKB != 128*1024 {
|
||||
t.Fatalf("sqlite cache option was not applied: %d", cfg.SQLite.CacheSizeKB)
|
||||
}
|
||||
if cfg.DSN != "file:data/metrics/metrics.db?cache=shared&mode=rwc" {
|
||||
t.Fatalf("unexpected sqlite dir dsn: %q", cfg.DSN)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,444 @@
|
||||
package metric
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"fmt"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// dialect abstracts SQL differences between supported databases.
|
||||
//
|
||||
// dialect 抽象不同数据库后端的 SQL 差异。
|
||||
type dialect interface {
|
||||
placeholder(n int) string
|
||||
// jsonPlaceholder returns the bind placeholder for a JSON column value.
|
||||
// PostgreSQL needs an explicit ::jsonb cast because the driver sends Go
|
||||
// strings with a text OID, which the server refuses to assign to a jsonb
|
||||
// column. SQLite and MySQL store JSON as text/native and need no cast.
|
||||
//
|
||||
// jsonPlaceholder 返回 JSON 列值的绑定占位符。PostgreSQL 需要显式的
|
||||
// ::jsonb 转换,因为驱动会以 text OID 发送 Go 字符串,服务器拒绝把它赋给
|
||||
// jsonb 列。SQLite 和 MySQL 将 JSON 存为文本或原生类型,不需要转换。
|
||||
jsonPlaceholder(n int) string
|
||||
jsonType() string
|
||||
autoIncrementPrimaryKey() string
|
||||
nowExpr() string
|
||||
insertDefinitionSQL(t tables) string
|
||||
upsertPointSQL(t tables, rowCount int) string
|
||||
// jsonExtractEquals renders a boolean predicate comparing the text value at
|
||||
// key inside a JSON column to a bind placeholder. The key is interpolated
|
||||
// into the SQL (callers must restrict it to a safe charset); the compared
|
||||
// value is always bound.
|
||||
//
|
||||
// jsonExtractEquals 渲染一个布尔谓词,用于比较 JSON 列内 key 对应的文本值和
|
||||
// 绑定占位符。key 会插入 SQL 中(调用方必须限制到安全字符集);被比较的值
|
||||
// 始终通过绑定参数传入。
|
||||
jsonExtractEquals(column, key, placeholder string) string
|
||||
// blobType is the column type for the rollup t-digest sketch.
|
||||
//
|
||||
// blobType 是 rollup t-digest sketch 的列类型。
|
||||
blobType() string
|
||||
// upsertRollupSQL builds a single-row upsert for one rollup bucket cell.
|
||||
//
|
||||
// upsertRollupSQL 为单个 rollup 桶单元构造单行 upsert。
|
||||
upsertRollupSQL(t tables) string
|
||||
// compactTxOptions returns the transaction options used for the compaction
|
||||
// transaction. PostgreSQL/MySQL escalate to SERIALIZABLE so the raw scan and
|
||||
// the raw deletion share one snapshot and a point inserted between them
|
||||
// cannot be deleted without first being rolled up. SQLite returns nil
|
||||
// (its default) because a single connection already serializes writes.
|
||||
//
|
||||
// compactTxOptions 返回 compaction 事务使用的事务选项。PostgreSQL/MySQL 会
|
||||
// 提升到 SERIALIZABLE,使 raw 扫描和 raw 删除共享同一个快照,让两者之间写入
|
||||
// 的点不会在尚未进入 rollup 前被删除。SQLite 返回 nil(默认),因为单连接
|
||||
// 已经串行化写入。
|
||||
compactTxOptions() *sql.TxOptions
|
||||
}
|
||||
|
||||
// tables stores the physical table names used by a Store.
|
||||
//
|
||||
// tables 保存 Store 使用的实际表名。
|
||||
type tables struct {
|
||||
// definitions is the metric definitions table.
|
||||
//
|
||||
// definitions 是指标定义表。
|
||||
definitions string
|
||||
// points is the raw points table.
|
||||
//
|
||||
// points 是原始采样点表。
|
||||
points string
|
||||
// rollups is the downsampled rollups table.
|
||||
//
|
||||
// rollups 是降采样 rollup 表。
|
||||
rollups string
|
||||
}
|
||||
|
||||
// newDialect returns the SQL dialect implementation for a backend.
|
||||
//
|
||||
// newDialect 根据数据库后端创建对应 SQL 方言实现。
|
||||
func newDialect(driver Driver) dialect {
|
||||
switch driver {
|
||||
case DriverPostgreSQL:
|
||||
return postgresDialect{}
|
||||
case DriverMySQL:
|
||||
return mysqlDialect{}
|
||||
default:
|
||||
return sqliteDialect{}
|
||||
}
|
||||
}
|
||||
|
||||
// sqliteDialect implements SQL rendering for SQLite.
|
||||
//
|
||||
// sqliteDialect 实现 SQLite 方言。
|
||||
type sqliteDialect struct{}
|
||||
|
||||
// placeholder returns a bind placeholder for this SQL dialect.
|
||||
//
|
||||
// placeholder 返回 SQLite 的绑定参数占位符。
|
||||
func (sqliteDialect) placeholder(int) string { return "?" }
|
||||
|
||||
// jsonPlaceholder returns a bind placeholder for JSON values.
|
||||
//
|
||||
// jsonPlaceholder 返回 SQLite JSON 值的绑定参数占位符。
|
||||
func (sqliteDialect) jsonPlaceholder(int) string { return "?" }
|
||||
|
||||
// jsonType returns the SQL column type used for JSON values.
|
||||
//
|
||||
// jsonType 返回 SQLite 使用的 JSON 存储类型。
|
||||
func (sqliteDialect) jsonType() string { return "TEXT" }
|
||||
|
||||
// autoIncrementPrimaryKey returns the SQL definition for an auto-incrementing primary key.
|
||||
//
|
||||
// autoIncrementPrimaryKey 返回 SQLite 自增主键定义。
|
||||
func (sqliteDialect) autoIncrementPrimaryKey() string { return "INTEGER PRIMARY KEY AUTOINCREMENT" }
|
||||
|
||||
// nowExpr returns the SQL expression for the current time.
|
||||
//
|
||||
// nowExpr 返回 SQLite 当前时间表达式。
|
||||
func (sqliteDialect) nowExpr() string { return "CURRENT_TIMESTAMP" }
|
||||
|
||||
// insertDefinitionSQL builds SQL for inserting or updating a metric definition.
|
||||
//
|
||||
// insertDefinitionSQL 构造 SQLite 指标定义 upsert SQL。
|
||||
func (sqliteDialect) insertDefinitionSQL(t tables) string {
|
||||
return fmt.Sprintf(`INSERT INTO %s
|
||||
(name, type, unit, description, retention_days, metadata, created_at, updated_at)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?)
|
||||
ON CONFLICT(name) DO UPDATE SET
|
||||
type = excluded.type,
|
||||
unit = excluded.unit,
|
||||
description = excluded.description,
|
||||
retention_days = excluded.retention_days,
|
||||
metadata = excluded.metadata,
|
||||
updated_at = excluded.updated_at`, t.definitions)
|
||||
}
|
||||
|
||||
// upsertPointSQL builds SQL for inserting or updating metric points.
|
||||
//
|
||||
// upsertPointSQL 构造 SQLite 采样点批量 upsert SQL。
|
||||
func (sqliteDialect) upsertPointSQL(t tables, rowCount int) string {
|
||||
return buildInsertPointsSQL(t.points, sqliteDialect{}, rowCount, `ON CONFLICT(metric_name, entity_id, tags_hash, ts_nano) DO UPDATE SET
|
||||
value = excluded.value,
|
||||
tags = excluded.tags,
|
||||
labels = excluded.labels,
|
||||
created_at = excluded.created_at`)
|
||||
}
|
||||
|
||||
// mysqlDialect implements SQL rendering for MySQL.
|
||||
//
|
||||
// mysqlDialect 实现 MySQL 方言。
|
||||
type mysqlDialect struct{}
|
||||
|
||||
// placeholder returns a bind placeholder for this SQL dialect.
|
||||
//
|
||||
// placeholder 返回 MySQL 的绑定参数占位符。
|
||||
func (mysqlDialect) placeholder(int) string { return "?" }
|
||||
|
||||
// jsonPlaceholder returns a bind placeholder for JSON values.
|
||||
//
|
||||
// jsonPlaceholder 返回 MySQL JSON 值的绑定参数占位符。
|
||||
func (mysqlDialect) jsonPlaceholder(int) string { return "?" }
|
||||
|
||||
// jsonType returns the SQL column type used for JSON values.
|
||||
//
|
||||
// jsonType 返回 MySQL 使用的 JSON 存储类型。
|
||||
func (mysqlDialect) jsonType() string { return "JSON" }
|
||||
|
||||
// autoIncrementPrimaryKey returns the SQL definition for an auto-incrementing primary key.
|
||||
//
|
||||
// autoIncrementPrimaryKey 返回 MySQL 自增主键定义。
|
||||
func (mysqlDialect) autoIncrementPrimaryKey() string { return "BIGINT AUTO_INCREMENT PRIMARY KEY" }
|
||||
|
||||
// nowExpr returns the SQL expression for the current time.
|
||||
//
|
||||
// nowExpr 返回 MySQL 当前时间表达式。
|
||||
func (mysqlDialect) nowExpr() string { return "CURRENT_TIMESTAMP" }
|
||||
|
||||
// insertDefinitionSQL builds SQL for inserting or updating a metric definition.
|
||||
//
|
||||
// insertDefinitionSQL 构造 MySQL 指标定义 upsert SQL。
|
||||
func (mysqlDialect) insertDefinitionSQL(t tables) string {
|
||||
return fmt.Sprintf(`INSERT INTO %s
|
||||
(name, type, unit, description, retention_days, metadata, created_at, updated_at)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?)
|
||||
ON DUPLICATE KEY UPDATE
|
||||
type = VALUES(type),
|
||||
unit = VALUES(unit),
|
||||
description = VALUES(description),
|
||||
retention_days = VALUES(retention_days),
|
||||
metadata = VALUES(metadata),
|
||||
updated_at = VALUES(updated_at)`, t.definitions)
|
||||
}
|
||||
|
||||
// upsertPointSQL builds SQL for inserting or updating metric points.
|
||||
//
|
||||
// upsertPointSQL 构造 MySQL 采样点批量 upsert SQL。
|
||||
func (mysqlDialect) upsertPointSQL(t tables, rowCount int) string {
|
||||
return buildInsertPointsSQL(t.points, mysqlDialect{}, rowCount, `ON DUPLICATE KEY UPDATE
|
||||
value = VALUES(value),
|
||||
tags = VALUES(tags),
|
||||
labels = VALUES(labels),
|
||||
created_at = VALUES(created_at)`)
|
||||
}
|
||||
|
||||
// postgresDialect implements SQL rendering for PostgreSQL.
|
||||
//
|
||||
// postgresDialect 实现 PostgreSQL 方言。
|
||||
type postgresDialect struct{}
|
||||
|
||||
// placeholder returns a bind placeholder for this SQL dialect.
|
||||
//
|
||||
// placeholder 返回 PostgreSQL 的编号绑定参数占位符。
|
||||
func (postgresDialect) placeholder(n int) string { return fmt.Sprintf("$%d", n) }
|
||||
|
||||
// jsonPlaceholder returns a bind placeholder for JSON values.
|
||||
//
|
||||
// jsonPlaceholder 返回 PostgreSQL JSONB 值的绑定参数占位符。
|
||||
func (postgresDialect) jsonPlaceholder(n int) string { return fmt.Sprintf("$%d::jsonb", n) }
|
||||
|
||||
// jsonType returns the SQL column type used for JSON values.
|
||||
//
|
||||
// jsonType 返回 PostgreSQL 使用的 JSONB 存储类型。
|
||||
func (postgresDialect) jsonType() string { return "JSONB" }
|
||||
|
||||
// autoIncrementPrimaryKey returns the SQL definition for an auto-incrementing primary key.
|
||||
//
|
||||
// autoIncrementPrimaryKey 返回 PostgreSQL 自增主键定义。
|
||||
func (postgresDialect) autoIncrementPrimaryKey() string { return "BIGSERIAL PRIMARY KEY" }
|
||||
|
||||
// nowExpr returns the SQL expression for the current time.
|
||||
//
|
||||
// nowExpr 返回 PostgreSQL 当前时间表达式。
|
||||
func (postgresDialect) nowExpr() string { return "CURRENT_TIMESTAMP" }
|
||||
|
||||
// insertDefinitionSQL builds SQL for inserting or updating a metric definition.
|
||||
//
|
||||
// insertDefinitionSQL 构造 PostgreSQL 指标定义 upsert SQL。
|
||||
func (postgresDialect) insertDefinitionSQL(t tables) string {
|
||||
return fmt.Sprintf(`INSERT INTO %s
|
||||
(name, type, unit, description, retention_days, metadata, created_at, updated_at)
|
||||
VALUES ($1, $2, $3, $4, $5, $6::jsonb, $7, $8)
|
||||
ON CONFLICT(name) DO UPDATE SET
|
||||
type = EXCLUDED.type,
|
||||
unit = EXCLUDED.unit,
|
||||
description = EXCLUDED.description,
|
||||
retention_days = EXCLUDED.retention_days,
|
||||
metadata = EXCLUDED.metadata,
|
||||
updated_at = EXCLUDED.updated_at`, t.definitions)
|
||||
}
|
||||
|
||||
// upsertPointSQL builds SQL for inserting or updating metric points.
|
||||
//
|
||||
// upsertPointSQL 构造 PostgreSQL 采样点批量 upsert SQL。
|
||||
func (postgresDialect) upsertPointSQL(t tables, rowCount int) string {
|
||||
return buildInsertPointsSQL(t.points, postgresDialect{}, rowCount, `ON CONFLICT(metric_name, entity_id, tags_hash, ts_nano) DO UPDATE SET
|
||||
value = EXCLUDED.value,
|
||||
tags = EXCLUDED.tags,
|
||||
labels = EXCLUDED.labels,
|
||||
created_at = EXCLUDED.created_at`)
|
||||
}
|
||||
|
||||
// buildInsertPointsSQL builds a multi-row insert statement for metric points.
|
||||
//
|
||||
// buildInsertPointsSQL 构造多行采样点 INSERT/UPSERT SQL。
|
||||
func buildInsertPointsSQL(table string, d dialect, rowCount int, suffix string) string {
|
||||
var b strings.Builder
|
||||
b.WriteString("INSERT INTO ")
|
||||
b.WriteString(table)
|
||||
b.WriteString(" (metric_name, entity_id, tags_hash, ts_nano, value, tags, labels, created_at) VALUES ")
|
||||
arg := 1
|
||||
rows := make([]string, rowCount)
|
||||
for i := 0; i < rowCount; i++ {
|
||||
parts := make([]string, 8)
|
||||
for j := range parts {
|
||||
// Columns are (metric_name, entity_id, tags_hash, ts_nano, value, tags,
|
||||
// labels, created_at); positions 5 and 6 (0-indexed) are JSON values.
|
||||
// tags_hash makes the tag set part of a point's identity, so points
|
||||
// that share entity+timestamp but differ in tags no longer collide.
|
||||
if j == 5 || j == 6 {
|
||||
parts[j] = d.jsonPlaceholder(arg)
|
||||
} else {
|
||||
parts[j] = d.placeholder(arg)
|
||||
}
|
||||
arg++
|
||||
}
|
||||
rows[i] = "(" + strings.Join(parts, ", ") + ")"
|
||||
}
|
||||
b.WriteString(strings.Join(rows, ", "))
|
||||
if suffix != "" {
|
||||
b.WriteByte(' ')
|
||||
b.WriteString(suffix)
|
||||
}
|
||||
return b.String()
|
||||
}
|
||||
|
||||
// tableName combines a table prefix and logical table name.
|
||||
//
|
||||
// tableName 用表名前缀和逻辑表名生成实际表名。
|
||||
func tableName(prefix, name string) string {
|
||||
return prefix + name
|
||||
}
|
||||
|
||||
// insertDefinitionOnlySQL builds a plain INSERT (no upsert clause) for a metric
|
||||
// definition. CreateMetric uses it so a duplicate name surfaces as a unique
|
||||
// constraint violation instead of silently overwriting the existing row.
|
||||
//
|
||||
// insertDefinitionOnlySQL 构造普通 INSERT(不带 upsert),供
|
||||
// CreateMetric 保持“只创建”语义;重复名称会暴露为唯一约束错误。
|
||||
func insertDefinitionOnlySQL(d dialect, t tables) string {
|
||||
cols := "(name, type, unit, description, retention_days, metadata, created_at, updated_at)"
|
||||
ph := []string{
|
||||
d.placeholder(1),
|
||||
d.placeholder(2),
|
||||
d.placeholder(3),
|
||||
d.placeholder(4),
|
||||
d.placeholder(5),
|
||||
d.jsonPlaceholder(6),
|
||||
d.placeholder(7),
|
||||
d.placeholder(8),
|
||||
}
|
||||
return fmt.Sprintf("INSERT INTO %s %s VALUES (%s)", t.definitions, cols, strings.Join(ph, ", "))
|
||||
}
|
||||
|
||||
// sqlSingleQuote escapes a string for safe inclusion inside a single-quoted SQL
|
||||
// string literal by doubling embedded single quotes.
|
||||
//
|
||||
// sqlSingleQuote 通过把单引号加倍,安全地转义可放入 SQL 单引号字符串
|
||||
// 字面量的内容。
|
||||
func sqlSingleQuote(s string) string {
|
||||
return strings.ReplaceAll(s, "'", "''")
|
||||
}
|
||||
|
||||
// sqlJSONPathQuoted builds a quoted JSON path member access ($."key") for the
|
||||
// given object key and returns it ready to embed inside a single-quoted SQL
|
||||
// string literal. Using the quoted form ($."key") rather than the bare form
|
||||
// ($.key) is required for keys that contain dots, hyphens, spaces or other
|
||||
// characters the JSON path grammar would otherwise interpret (e.g. a dot is a
|
||||
// member separator, so the bare path "$.region.zone" would look up nested
|
||||
// member "zone" of object "region" instead of the flat key "region.zone").
|
||||
//
|
||||
// Escaping is applied in two layers, innermost first:
|
||||
// 1. JSON path string escaping inside the double quotes: backslash and double
|
||||
// quote are backslash-escaped.
|
||||
// 2. SQL string-literal escaping of the whole path: single quotes doubled.
|
||||
//
|
||||
// sqlJSONPathQuoted 为给定对象键构造带引号的 JSON path 成员访问($."key"),
|
||||
// 并返回可嵌入 SQL 单引号字符串字面量的结果。必须使用带引号形式($."key"),
|
||||
// 而不是裸形式($.key),因为包含点号、连字符、空格或其他 JSON path 语法字符的
|
||||
// key 会被误解;例如点号是成员分隔符,所以裸路径 "$.region.zone" 会查找对象
|
||||
// "region" 的嵌套成员 "zone",而不是平铺 key "region.zone"。
|
||||
//
|
||||
// 转义按从内到外两层应用:
|
||||
// 1. 双引号内的 JSON path 字符串转义:反斜杠和双引号使用反斜杠转义。
|
||||
// 2. 整个 path 的 SQL 字符串字面量转义:单引号加倍。
|
||||
func sqlJSONPathQuoted(key string) string {
|
||||
escaped := strings.ReplaceAll(key, "\\", "\\\\")
|
||||
escaped = strings.ReplaceAll(escaped, "\"", "\\\"")
|
||||
return sqlSingleQuote("$.\"" + escaped + "\"")
|
||||
}
|
||||
|
||||
// jsonExtractEquals renders a JSON equality predicate.
|
||||
//
|
||||
// jsonExtractEquals 构造 SQLite JSON 字段等值过滤表达式。
|
||||
func (sqliteDialect) jsonExtractEquals(column, key, placeholder string) string {
|
||||
return fmt.Sprintf("json_extract(%s, '%s') = %s", column, sqlJSONPathQuoted(key), placeholder)
|
||||
}
|
||||
|
||||
// jsonExtractEquals renders a JSON equality predicate.
|
||||
//
|
||||
// jsonExtractEquals 构造 MySQL JSON 字段等值过滤表达式。
|
||||
func (mysqlDialect) jsonExtractEquals(column, key, placeholder string) string {
|
||||
return fmt.Sprintf("JSON_UNQUOTE(JSON_EXTRACT(%s, '%s')) = %s", column, sqlJSONPathQuoted(key), placeholder)
|
||||
}
|
||||
|
||||
// jsonExtractEquals renders a JSON equality predicate.
|
||||
//
|
||||
// jsonExtractEquals 构造 PostgreSQL JSONB 字段等值过滤表达式。
|
||||
func (postgresDialect) jsonExtractEquals(column, key, placeholder string) string {
|
||||
// PostgreSQL's ->> takes the key as a plain text literal (not a JSON path),
|
||||
// so a dot or hyphen in the key is harmless; only single quotes need SQL
|
||||
// escaping.
|
||||
return fmt.Sprintf("%s->>'%s' = %s", column, sqlSingleQuote(key), placeholder)
|
||||
}
|
||||
|
||||
// sqlAggValueExpr returns the SQL value expression that computes agg over the
|
||||
// "value" column for the given driver, and whether that aggregation can be
|
||||
// pushed down to this backend. Aggregations that need the ordered raw series
|
||||
// (first/last/rate) are never pushed down here. Percentiles and population
|
||||
// standard deviation are pushed down only where the backend has a portable,
|
||||
// semantics-matching function:
|
||||
//
|
||||
// - STDDEV_POP: MySQL and PostgreSQL (matches the in-memory population stddev,
|
||||
// which divides by N). SQLite has no built-in, so it falls back to memory.
|
||||
// - percentile_cont(p) WITHIN GROUP (ORDER BY value): PostgreSQL only. This
|
||||
// matches the in-memory linear-interpolation percentile for ANY p (p50,
|
||||
// p95, p99, p99.9, ...). MySQL lacks a portable continuous-percentile
|
||||
// function and SQLite has none, so both fall back to memory.
|
||||
//
|
||||
// The simple reductions (avg/min/max/sum/count) are portable everywhere.
|
||||
//
|
||||
// sqlAggValueExpr 返回给定驱动上对 "value" 列计算 agg 的 SQL 值表达式,
|
||||
// 以及该聚合是否能下推到该后端。需要有序原始序列的聚合(first/last/rate)
|
||||
// 永远不会在这里下推。百分位和总体标准差只会在后端提供可移植且语义匹配的函数时
|
||||
// 下推:
|
||||
//
|
||||
// - STDDEV_POP:MySQL 和 PostgreSQL 支持(匹配内存中的总体标准差,
|
||||
// 即除以 N)。SQLite 没有内置函数,因此回退到内存计算。
|
||||
// - percentile_cont(p) WITHIN GROUP (ORDER BY value):仅 PostgreSQL 支持。
|
||||
// 它匹配内存中任意 p(p50、p95、p99、p99.9 等)的线性插值百分位。
|
||||
// MySQL 没有可移植的连续百分位函数,SQLite 也没有,因此两者都回退到内存计算。
|
||||
//
|
||||
// 简单归约(avg/min/max/sum/count)在所有后端都可移植。
|
||||
func sqlAggValueExpr(driver Driver, agg Aggregation) (string, bool) {
|
||||
switch agg {
|
||||
case AggAvg:
|
||||
return "AVG(value)", true
|
||||
case AggMin:
|
||||
return "MIN(value)", true
|
||||
case AggMax:
|
||||
return "MAX(value)", true
|
||||
case AggSum:
|
||||
return "SUM(value)", true
|
||||
case AggCount:
|
||||
return "COUNT(*)", true
|
||||
case AggStdDev:
|
||||
switch driver {
|
||||
case DriverMySQL, DriverPostgreSQL:
|
||||
return "STDDEV_POP(value)", true
|
||||
default:
|
||||
return "", false
|
||||
}
|
||||
default:
|
||||
// Any percentile (fixed p50/p95/p99 and arbitrary pXX / pXX.X all land
|
||||
// here). PostgreSQL has a portable continuous percentile that matches the
|
||||
// in-memory linear-interpolation percentile; other backends fall back to
|
||||
// the in-memory path.
|
||||
if frac, ok := percentileFractionString(agg); ok {
|
||||
if driver == DriverPostgreSQL {
|
||||
return fmt.Sprintf("percentile_cont(%s) WITHIN GROUP (ORDER BY value)", frac), true
|
||||
}
|
||||
return "", false
|
||||
}
|
||||
return "", false
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,160 @@
|
||||
package metric
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"fmt"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// rollupColumns is the fixed column list for the rollups table, shared by the
|
||||
// upsert builder and the row scanner so they cannot drift apart. tags_hash is a
|
||||
// stable fingerprint of the (canonical) tag set; tags carries the tag map itself
|
||||
// so reads can return it and tag filters can be pushed down. Including the tag
|
||||
// dimension in the key is what keeps each distinct tag combination (e.g. a GPU
|
||||
// device_index) as its own rollup series instead of being merged together.
|
||||
//
|
||||
// rollupColumns 是 rollups 表固定列清单,由 upsert 构造器和行扫描器共用,
|
||||
// 避免二者漂移。tags_hash 是(规范化)标签集合的稳定指纹;tags 携带标签 map
|
||||
// 本身,因此读取时可以返回它,也能下推标签过滤。把标签维度纳入 key,正是让
|
||||
// 每个不同标签组合(例如 GPU 的 device_index)保留为自己的 rollup 序列、
|
||||
// 而不是被合并在一起的原因。
|
||||
const rollupColumns = "metric_name, entity_id, tags_hash, tags, resolution_nano, bucket_nano, " +
|
||||
"count, sum, sum_sq, min_val, max_val, first_val, first_ts, last_val, last_ts, digest, created_at"
|
||||
|
||||
// rollupColumnCount is the number of columns in rollupColumns.
|
||||
//
|
||||
// rollupColumnCount 是 rollupColumns 中的列数量。
|
||||
const rollupColumnCount = 17
|
||||
|
||||
// rollupTagsArgIndex is the 1-based position of the `tags` JSON column in
|
||||
// rollupColumns. PostgreSQL needs a ::jsonb cast on that bind placeholder.
|
||||
//
|
||||
// rollupTagsArgIndex 是 tags JSON 列在 rollupColumns 中的 1 基位置;
|
||||
// PostgreSQL 需要在对应占位符上追加 ::jsonb 转换。
|
||||
const rollupTagsArgIndex = 4
|
||||
|
||||
// blobType returns the column type used to store the t-digest sketch.
|
||||
//
|
||||
// blobType 返回用于保存 t-digest 摘要的二进制列类型。
|
||||
func (sqliteDialect) blobType() string { return "BLOB" }
|
||||
|
||||
// blobType returns the binary column type used for t-digest blobs.
|
||||
//
|
||||
// blobType 返回 MySQL 用于保存 t-digest 摘要的二进制列类型。
|
||||
func (mysqlDialect) blobType() string { return "LONGBLOB" }
|
||||
|
||||
// blobType returns the binary column type used for t-digest blobs.
|
||||
//
|
||||
// blobType 返回 PostgreSQL 用于保存 t-digest 摘要的二进制列类型。
|
||||
func (postgresDialect) blobType() string { return "BYTEA" }
|
||||
|
||||
// compactTxOptions returns nil so SQLite uses its default isolation. A single
|
||||
// write connection already serializes the compaction transaction against
|
||||
// concurrent writers, so the raw scan and the raw deletion observe a stable
|
||||
// view without escalating the isolation level.
|
||||
//
|
||||
// compactTxOptions 返回 nil,让 SQLite 使用默认隔离级别。单个写连接已经把
|
||||
// compaction 事务与并发写入串行化,因此 raw 扫描和 raw 删除无需提升隔离级别
|
||||
// 即可观察到稳定视图。
|
||||
func (sqliteDialect) compactTxOptions() *sql.TxOptions { return nil }
|
||||
|
||||
// compactTxOptions escalates to SERIALIZABLE so a point inserted between the raw
|
||||
// scan and the raw deletion cannot be deleted without first being rolled up.
|
||||
// InnoDB's SERIALIZABLE turns the scan into a locking read that blocks such a
|
||||
// concurrent insert in the scanned range until the compaction commits.
|
||||
//
|
||||
// compactTxOptions 提升到 SERIALIZABLE,使在 raw 扫描和 raw 删除之间写入的点
|
||||
// 不会在尚未进入 rollup 前被删除。InnoDB 的 SERIALIZABLE 会把扫描变为加锁读,
|
||||
// 在 compaction 提交前阻塞扫描范围内的此类并发插入。
|
||||
func (mysqlDialect) compactTxOptions() *sql.TxOptions {
|
||||
return &sql.TxOptions{Isolation: sql.LevelSerializable}
|
||||
}
|
||||
|
||||
// compactTxOptions escalates to SERIALIZABLE so the raw scan and the raw
|
||||
// deletion share one snapshot. A concurrent insert into the scanned range that
|
||||
// would otherwise be deleted without a rollup instead triggers a serialization
|
||||
// failure that CompactMetric retries on a fresh snapshot.
|
||||
//
|
||||
// compactTxOptions 提升到 SERIALIZABLE,使 raw 扫描和 raw 删除共享同一个快照。
|
||||
// 对扫描范围内的并发插入(否则会被删除却没有进入 rollup)会触发序列化失败,
|
||||
// 由 CompactMetric 在新快照上重试。
|
||||
func (postgresDialect) compactTxOptions() *sql.TxOptions {
|
||||
return &sql.TxOptions{Isolation: sql.LevelSerializable}
|
||||
}
|
||||
|
||||
// upsertRollupSQL builds a single-row upsert for a rollup cell, keyed by
|
||||
// (metric_name, entity_id, tags_hash, resolution_nano, bucket_nano). Compact
|
||||
// recomputes a bucket wholesale from its inputs each run, so the conflict action
|
||||
// replaces every summary column rather than trying to merge in SQL.
|
||||
//
|
||||
// upsertRollupSQL 构造单个 rollup 单元的 upsert SQL,其 key 为
|
||||
// (metric_name, entity_id, tags_hash, resolution_nano, bucket_nano)。Compact
|
||||
// 每次都会从输入整体重新计算一个桶,因此冲突动作会替换每个摘要列,
|
||||
// 而不是尝试在 SQL 中合并。
|
||||
func (d sqliteDialect) upsertRollupSQL(t tables) string {
|
||||
return buildUpsertRollupSQL(t.rollups, d, sqliteRollupConflict)
|
||||
}
|
||||
|
||||
// upsertRollupSQL builds backend-specific SQL for upserting a rollup cell.
|
||||
//
|
||||
// upsertRollupSQL 构造 MySQL rollup 单元 upsert SQL。
|
||||
func (d mysqlDialect) upsertRollupSQL(t tables) string {
|
||||
return buildUpsertRollupSQL(t.rollups, d, mysqlRollupConflict)
|
||||
}
|
||||
|
||||
// upsertRollupSQL builds backend-specific SQL for upserting a rollup cell.
|
||||
//
|
||||
// upsertRollupSQL 构造 PostgreSQL rollup 单元 upsert SQL。
|
||||
func (d postgresDialect) upsertRollupSQL(t tables) string {
|
||||
return buildUpsertRollupSQL(t.rollups, d, postgresRollupConflict)
|
||||
}
|
||||
|
||||
// sqliteRollupConflict is the SQLite conflict clause for replacing rollup cells.
|
||||
//
|
||||
// sqliteRollupConflict 是 SQLite 用于替换 rollup 单元的冲突处理子句。
|
||||
const sqliteRollupConflict = `ON CONFLICT(metric_name, entity_id, tags_hash, resolution_nano, bucket_nano) DO UPDATE SET
|
||||
tags = excluded.tags,
|
||||
count = excluded.count, sum = excluded.sum, sum_sq = excluded.sum_sq,
|
||||
min_val = excluded.min_val, max_val = excluded.max_val,
|
||||
first_val = excluded.first_val, first_ts = excluded.first_ts,
|
||||
last_val = excluded.last_val, last_ts = excluded.last_ts,
|
||||
digest = excluded.digest, created_at = excluded.created_at`
|
||||
|
||||
// postgresRollupConflict is the PostgreSQL conflict clause for replacing rollup cells.
|
||||
//
|
||||
// postgresRollupConflict 是 PostgreSQL 用于替换 rollup 单元的冲突处理子句。
|
||||
const postgresRollupConflict = `ON CONFLICT(metric_name, entity_id, tags_hash, resolution_nano, bucket_nano) DO UPDATE SET
|
||||
tags = EXCLUDED.tags,
|
||||
count = EXCLUDED.count, sum = EXCLUDED.sum, sum_sq = EXCLUDED.sum_sq,
|
||||
min_val = EXCLUDED.min_val, max_val = EXCLUDED.max_val,
|
||||
first_val = EXCLUDED.first_val, first_ts = EXCLUDED.first_ts,
|
||||
last_val = EXCLUDED.last_val, last_ts = EXCLUDED.last_ts,
|
||||
digest = EXCLUDED.digest, created_at = EXCLUDED.created_at`
|
||||
|
||||
// mysqlRollupConflict is the MySQL conflict clause for replacing rollup cells.
|
||||
//
|
||||
// mysqlRollupConflict 是 MySQL 用于替换 rollup 单元的冲突处理子句。
|
||||
const mysqlRollupConflict = `ON DUPLICATE KEY UPDATE
|
||||
tags = VALUES(tags),
|
||||
count = VALUES(count), sum = VALUES(sum), sum_sq = VALUES(sum_sq),
|
||||
min_val = VALUES(min_val), max_val = VALUES(max_val),
|
||||
first_val = VALUES(first_val), first_ts = VALUES(first_ts),
|
||||
last_val = VALUES(last_val), last_ts = VALUES(last_ts),
|
||||
digest = VALUES(digest), created_at = VALUES(created_at)`
|
||||
|
||||
// buildUpsertRollupSQL builds a single-row rollup upsert statement.
|
||||
//
|
||||
// buildUpsertRollupSQL 构造单行 rollup upsert SQL。
|
||||
func buildUpsertRollupSQL(table string, d dialect, conflict string) string {
|
||||
ph := make([]string, rollupColumnCount)
|
||||
for i := 0; i < rollupColumnCount; i++ {
|
||||
if i+1 == rollupTagsArgIndex {
|
||||
// The tags column is JSON; PostgreSQL requires an explicit ::jsonb cast.
|
||||
ph[i] = d.jsonPlaceholder(i + 1)
|
||||
} else {
|
||||
ph[i] = d.placeholder(i + 1)
|
||||
}
|
||||
}
|
||||
return fmt.Sprintf("INSERT INTO %s (%s) VALUES (%s) %s",
|
||||
table, rollupColumns, strings.Join(ph, ", "), conflict)
|
||||
}
|
||||
@@ -0,0 +1,60 @@
|
||||
package metric
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// TestDialectsGenerateBackendSpecificSQL verifies backend-specific SQL rendering.
|
||||
//
|
||||
// TestDialectsGenerateBackendSpecificSQL 验证不同数据库后端生成各自的 SQL。
|
||||
func TestDialectsGenerateBackendSpecificSQL(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
driver Driver
|
||||
placeholder string
|
||||
upsert string
|
||||
jsonType string
|
||||
}{
|
||||
{
|
||||
name: "sqlite",
|
||||
driver: DriverSQLite,
|
||||
placeholder: "?",
|
||||
upsert: "ON CONFLICT(metric_name, entity_id, tags_hash, ts_nano)",
|
||||
jsonType: "TEXT",
|
||||
},
|
||||
{
|
||||
name: "mysql",
|
||||
driver: DriverMySQL,
|
||||
placeholder: "?",
|
||||
upsert: "ON DUPLICATE KEY UPDATE",
|
||||
jsonType: "JSON",
|
||||
},
|
||||
{
|
||||
name: "postgresql",
|
||||
driver: DriverPostgreSQL,
|
||||
placeholder: "$1",
|
||||
upsert: "ON CONFLICT(metric_name, entity_id, tags_hash, ts_nano)",
|
||||
jsonType: "JSONB",
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
d := newDialect(tt.driver)
|
||||
if got := d.placeholder(1); got != tt.placeholder {
|
||||
t.Fatalf("placeholder: expected %q, got %q", tt.placeholder, got)
|
||||
}
|
||||
if got := d.jsonType(); got != tt.jsonType {
|
||||
t.Fatalf("json type: expected %q, got %q", tt.jsonType, got)
|
||||
}
|
||||
sql := d.upsertPointSQL(tables{points: "metric_points"}, 2)
|
||||
if !strings.Contains(sql, "metric_points") {
|
||||
t.Fatalf("upsert sql should reference metric_points: %s", sql)
|
||||
}
|
||||
if !strings.Contains(sql, tt.upsert) {
|
||||
t.Fatalf("upsert sql should contain %q: %s", tt.upsert, sql)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,26 @@
|
||||
package metric
|
||||
|
||||
import "errors"
|
||||
|
||||
var (
|
||||
// ErrInvalidArgument reports an invalid caller-supplied argument.
|
||||
//
|
||||
// ErrInvalidArgument 表示调用方传入了无效参数。
|
||||
ErrInvalidArgument = errors.New("metric: invalid argument")
|
||||
// ErrNotFound reports a missing metric or record.
|
||||
//
|
||||
// ErrNotFound 表示指标或记录不存在。
|
||||
ErrNotFound = errors.New("metric: not found")
|
||||
// ErrNoData reports that a valid query range contained no samples.
|
||||
//
|
||||
// ErrNoData 表示有效查询范围内没有采样。
|
||||
ErrNoData = errors.New("metric: no data in range")
|
||||
// ErrAlreadyExists reports that a create-only operation found an existing row.
|
||||
//
|
||||
// ErrAlreadyExists 表示只创建操作遇到了已存在的行。
|
||||
ErrAlreadyExists = errors.New("metric: already exists")
|
||||
// ErrClosed reports that the store has already been closed.
|
||||
//
|
||||
// ErrClosed 表示 Store 已经关闭。
|
||||
ErrClosed = errors.New("metric: store is closed")
|
||||
)
|
||||
@@ -0,0 +1,347 @@
|
||||
package metric_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"log"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/komari-monitor/komari/pkg/metric"
|
||||
)
|
||||
|
||||
// Example_rollupTags demonstrates tagged rollups with automatic series routing.
|
||||
//
|
||||
// Example_rollupTags 演示带标签的 rollup 以及 Series 自动路由读取。
|
||||
func Example_rollupTags() {
|
||||
ctx := context.Background()
|
||||
base := time.Date(2026, 6, 18, 10, 0, 0, 0, time.UTC)
|
||||
|
||||
store, err := metric.Open(ctx, metric.SQLite(
|
||||
"file:metric-example?mode=memory&cache=shared",
|
||||
metric.WithRollupPolicy(metric.RollupPolicy{
|
||||
RawRetention: 2 * time.Minute,
|
||||
Tiers: []metric.RollupTier{
|
||||
{Interval: time.Minute, Retention: 24 * time.Hour},
|
||||
},
|
||||
}),
|
||||
))
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
defer store.Close()
|
||||
|
||||
if err := store.CreateMetric(ctx, metric.Definition{
|
||||
Name: "gpu.usage",
|
||||
Type: metric.TypeGauge,
|
||||
Unit: "%",
|
||||
}); err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
|
||||
var points []metric.Point
|
||||
for i := 0; i < 10; i++ {
|
||||
ts := base.Add(time.Duration(i) * time.Second)
|
||||
points = append(points,
|
||||
metric.Point{
|
||||
MetricName: "gpu.usage",
|
||||
EntityID: "host-1",
|
||||
Timestamp: ts,
|
||||
Value: float64(10 + i),
|
||||
Tags: map[string]string{"device_index": "0"},
|
||||
},
|
||||
metric.Point{
|
||||
MetricName: "gpu.usage",
|
||||
EntityID: "host-1",
|
||||
Timestamp: ts,
|
||||
Value: float64(80 + i),
|
||||
Tags: map[string]string{"device_index": "1"},
|
||||
},
|
||||
)
|
||||
}
|
||||
if err := store.WriteBatch(ctx, points); err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
|
||||
// Compact builds one rollup series per tag set. Raw points older than the
|
||||
// policy's RawRetention are deleted after their rollups are materialized.
|
||||
now := base.Add(time.Hour)
|
||||
if _, err := store.Compact(ctx, now); err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
|
||||
series, err := store.Series(ctx, metric.AggregateQuery{
|
||||
Query: metric.Query{
|
||||
MetricName: "gpu.usage",
|
||||
EntityID: "host-1",
|
||||
Start: base,
|
||||
End: base.Add(time.Minute),
|
||||
Tags: map[string]string{"device_index": "0"},
|
||||
},
|
||||
Aggregation: metric.AggAvg,
|
||||
Interval: time.Minute,
|
||||
}, now)
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
|
||||
fmt.Printf("device 0: count=%d avg=%.1f\n", series[0].Count, series[0].Value)
|
||||
|
||||
// Output:
|
||||
// device 0: count=10 avg=14.5
|
||||
}
|
||||
|
||||
// Example_agentFleetLifecycle demonstrates a realistic Komari-style fleet:
|
||||
// 50 agents report system metrics every second, ping tasks are represented as a
|
||||
// tagged ping metric, and lifecycle events clean up task and agent series.
|
||||
//
|
||||
// Example_agentFleetLifecycle 演示贴近 Komari 的实际场景:50 个 agent 每秒上报系统
|
||||
// 指标,ping task 用带标签的 ping 指标表示,并通过生命周期事件清理 task 和 agent 序列。
|
||||
func Example_agentFleetLifecycle() {
|
||||
ctx := context.Background()
|
||||
base := time.Date(2026, 6, 18, 10, 0, 0, 0, time.UTC)
|
||||
|
||||
store, err := metric.Open(ctx, metric.SQLite(
|
||||
"file:metric-agent-fleet?mode=memory&cache=shared",
|
||||
metric.WithRollupPolicy(metric.RollupPolicy{
|
||||
RawRetention: 2 * time.Minute,
|
||||
Tiers: []metric.RollupTier{
|
||||
{Interval: time.Minute, Retention: 24 * time.Hour},
|
||||
{Interval: 5 * time.Minute, Retention: 30 * 24 * time.Hour},
|
||||
},
|
||||
}),
|
||||
))
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
defer store.Close()
|
||||
|
||||
for _, def := range []metric.Definition{
|
||||
{Name: "cpu.usage", Type: metric.TypeGauge, Unit: "%"},
|
||||
{Name: "mem.usage", Type: metric.TypeGauge, Unit: "%"},
|
||||
{Name: "disk.usage", Type: metric.TypeGauge, Unit: "%"},
|
||||
{Name: "swap.usage", Type: metric.TypeGauge, Unit: "%"},
|
||||
{Name: "net.in.bytes", Type: metric.TypeCounter, Unit: "bytes"},
|
||||
{Name: "net.out.bytes", Type: metric.TypeCounter, Unit: "bytes"},
|
||||
{Name: "process.count", Type: metric.TypeGauge, Unit: "count"},
|
||||
{Name: "gpu.usage", Type: metric.TypeGauge, Unit: "%"},
|
||||
{Name: "ping.latency_ms", Type: metric.TypeGauge, Unit: "ms"},
|
||||
} {
|
||||
if err := store.UpsertMetric(ctx, def); err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
agents := makeAgents(50)
|
||||
pingTasks := []string{"pingtask1", "pingtask2"}
|
||||
for sec := 0; sec < 120; sec++ {
|
||||
ts := base.Add(time.Duration(sec) * time.Second)
|
||||
|
||||
switch sec {
|
||||
case 45:
|
||||
// A ping task is deleted by the admin. Because task_id is a tag, one
|
||||
// DeleteSeries removes that task's records for every agent, including
|
||||
// already materialized rollups.
|
||||
if _, err := store.DeleteSeries(ctx, metric.Query{
|
||||
MetricName: "ping.latency_ms",
|
||||
Tags: map[string]string{"task_id": "pingtask2"},
|
||||
}); err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
pingTasks = []string{"pingtask1"}
|
||||
case 70:
|
||||
// The admin clears all ping task records while keeping the task
|
||||
// definitions themselves. Deleting the metric data is the simplest
|
||||
// reset; UpsertMetric recreates the definition before new samples land.
|
||||
if err := store.DeleteMetric(ctx, "ping.latency_ms"); err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
if err := store.UpsertMetric(ctx, metric.Definition{
|
||||
Name: "ping.latency_ms", Type: metric.TypeGauge, Unit: "ms",
|
||||
}); err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
case 85:
|
||||
// An agent is removed. DeleteEntity clears every metric series for
|
||||
// that agent, raw and rollup, across the whole store.
|
||||
if _, err := store.DeleteEntity(ctx, "agent-007"); err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
agents = removeAgent(agents, "agent-007")
|
||||
case 95:
|
||||
// A new agent is added. No schema change is needed; the first write
|
||||
// creates its series naturally.
|
||||
agents = append(agents, "agent-051")
|
||||
}
|
||||
|
||||
points := make([]metric.Point, 0, len(agents)*(7+2+len(pingTasks)))
|
||||
for _, agentID := range agents {
|
||||
points = append(points, systemPoints(agentID, ts, sec)...)
|
||||
for _, gpu := range []string{"0", "1"} {
|
||||
points = append(points, metric.Point{
|
||||
MetricName: "gpu.usage",
|
||||
EntityID: agentID,
|
||||
Timestamp: ts,
|
||||
Value: gpuUsage(agentID, gpu, sec),
|
||||
Tags: map[string]string{"device_index": gpu},
|
||||
})
|
||||
}
|
||||
for _, taskID := range pingTasks {
|
||||
points = append(points, metric.Point{
|
||||
MetricName: "ping.latency_ms",
|
||||
EntityID: agentID,
|
||||
Timestamp: ts,
|
||||
Value: pingLatency(agentID, taskID, sec),
|
||||
Tags: map[string]string{"task_id": taskID},
|
||||
})
|
||||
}
|
||||
}
|
||||
if err := store.WriteBatch(ctx, points); err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
if sec%30 == 29 {
|
||||
if _, err := store.Compact(ctx, ts); err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
now := base.Add(2 * time.Hour)
|
||||
if _, err := store.Compact(ctx, now); err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
|
||||
activeCPU, err := store.Series(ctx, metric.AggregateQuery{
|
||||
Query: metric.Query{
|
||||
MetricName: "cpu.usage",
|
||||
Start: base,
|
||||
End: base.Add(2 * time.Minute),
|
||||
},
|
||||
Aggregation: metric.AggAvg,
|
||||
Interval: time.Minute,
|
||||
}, now)
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
deletedAgentCPU, err := store.Series(ctx, metric.AggregateQuery{
|
||||
Query: metric.Query{
|
||||
MetricName: "cpu.usage",
|
||||
EntityID: "agent-007",
|
||||
Start: base,
|
||||
End: base.Add(2 * time.Minute),
|
||||
},
|
||||
Aggregation: metric.AggAvg,
|
||||
Interval: time.Minute,
|
||||
}, now)
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
task1, err := store.Series(ctx, metric.AggregateQuery{
|
||||
Query: metric.Query{
|
||||
MetricName: "ping.latency_ms",
|
||||
Start: base,
|
||||
End: base.Add(2 * time.Minute),
|
||||
Tags: map[string]string{"task_id": "pingtask1"},
|
||||
},
|
||||
Aggregation: metric.AggAvg,
|
||||
Interval: time.Minute,
|
||||
}, now)
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
task2, err := store.Series(ctx, metric.AggregateQuery{
|
||||
Query: metric.Query{
|
||||
MetricName: "ping.latency_ms",
|
||||
Start: base,
|
||||
End: base.Add(2 * time.Minute),
|
||||
Tags: map[string]string{"task_id": "pingtask2"},
|
||||
},
|
||||
Aggregation: metric.AggAvg,
|
||||
Interval: time.Minute,
|
||||
}, now)
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
|
||||
fmt.Printf("agents now=%d\n", len(agents))
|
||||
fmt.Printf("fleet cpu buckets=%d\n", len(nonEmpty(activeCPU)))
|
||||
fmt.Printf("deleted agent buckets=%d\n", len(nonEmpty(deletedAgentCPU)))
|
||||
fmt.Printf("pingtask1 buckets=%d\n", len(nonEmpty(task1)))
|
||||
fmt.Printf("pingtask2 buckets=%d\n", len(nonEmpty(task2)))
|
||||
|
||||
// Output:
|
||||
// agents now=50
|
||||
// fleet cpu buckets=2
|
||||
// deleted agent buckets=0
|
||||
// pingtask1 buckets=1
|
||||
// pingtask2 buckets=0
|
||||
}
|
||||
|
||||
func makeAgents(n int) []string {
|
||||
out := make([]string, 0, n)
|
||||
for i := 1; i <= n; i++ {
|
||||
out = append(out, fmt.Sprintf("agent-%03d", i))
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func removeAgent(agents []string, deleted string) []string {
|
||||
out := agents[:0]
|
||||
for _, agentID := range agents {
|
||||
if agentID != deleted {
|
||||
out = append(out, agentID)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func systemPoints(agentID string, ts time.Time, sec int) []metric.Point {
|
||||
id := numericSuffix(agentID)
|
||||
return []metric.Point{
|
||||
{MetricName: "cpu.usage", EntityID: agentID, Timestamp: ts, Value: float64((id*3 + sec) % 100)},
|
||||
{MetricName: "mem.usage", EntityID: agentID, Timestamp: ts, Value: 35 + float64((id+sec)%40)},
|
||||
{MetricName: "disk.usage", EntityID: agentID, Timestamp: ts, Value: 50 + float64(id%20)/2},
|
||||
{MetricName: "swap.usage", EntityID: agentID, Timestamp: ts, Value: float64((id + sec/10) % 12)},
|
||||
{MetricName: "net.in.bytes", EntityID: agentID, Timestamp: ts, Value: float64(sec * (1000 + id))},
|
||||
{MetricName: "net.out.bytes", EntityID: agentID, Timestamp: ts, Value: float64(sec * (700 + id))},
|
||||
{MetricName: "process.count", EntityID: agentID, Timestamp: ts, Value: float64(90 + id%25)},
|
||||
}
|
||||
}
|
||||
|
||||
func gpuUsage(agentID, deviceIndex string, sec int) float64 {
|
||||
id := numericSuffix(agentID)
|
||||
device := 0
|
||||
if deviceIndex == "1" {
|
||||
device = 37
|
||||
}
|
||||
return float64((id*5 + device + sec*2) % 100)
|
||||
}
|
||||
|
||||
func pingLatency(agentID, taskID string, sec int) float64 {
|
||||
id := numericSuffix(agentID)
|
||||
offset := 0
|
||||
if strings.HasSuffix(taskID, "2") {
|
||||
offset = 35
|
||||
}
|
||||
return float64(20 + (id+sec+offset)%80)
|
||||
}
|
||||
|
||||
func numericSuffix(agentID string) int {
|
||||
var n int
|
||||
for _, r := range agentID {
|
||||
if r >= '0' && r <= '9' {
|
||||
n = n*10 + int(r-'0')
|
||||
}
|
||||
}
|
||||
return n
|
||||
}
|
||||
|
||||
func nonEmpty(points []metric.AggregatePoint) []metric.AggregatePoint {
|
||||
out := points[:0]
|
||||
for _, p := range points {
|
||||
if p.Count > 0 {
|
||||
out = append(out, p)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
@@ -0,0 +1,565 @@
|
||||
package metric
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"math"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// newMemStore opens an isolated in-memory store for tests.
|
||||
//
|
||||
// newMemStore 打开一个用于测试的隔离内存 Store。
|
||||
func newMemStore(t *testing.T) *Store {
|
||||
t.Helper()
|
||||
store, err := Open(context.Background(), SQLite("file:imp-test?mode=memory&cache=shared"))
|
||||
if err != nil {
|
||||
t.Fatalf("open store: %v", err)
|
||||
}
|
||||
t.Cleanup(func() { store.Close() })
|
||||
return store
|
||||
}
|
||||
|
||||
// TestCreateMetricRejectsDuplicate verifies create-only metric semantics.
|
||||
//
|
||||
// TestCreateMetricRejectsDuplicate 验证 CreateMetric 遇到重复指标时会拒绝。
|
||||
func TestCreateMetricRejectsDuplicate(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
s := newMemStore(t)
|
||||
def := Definition{Name: "dup.metric", Type: TypeGauge}
|
||||
if err := s.CreateMetric(ctx, def); err != nil {
|
||||
t.Fatalf("first create: %v", err)
|
||||
}
|
||||
err := s.CreateMetric(ctx, def)
|
||||
if !errors.Is(err, ErrAlreadyExists) {
|
||||
t.Fatalf("expected ErrAlreadyExists on duplicate, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// TestUpsertMetricOverwrites verifies upsert updates mutable definition fields.
|
||||
//
|
||||
// TestUpsertMetricOverwrites 验证 UpsertMetric 会更新指标定义的可变字段。
|
||||
func TestUpsertMetricOverwrites(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
s := newMemStore(t)
|
||||
if err := s.CreateMetric(ctx, Definition{Name: "m", Type: TypeGauge, Unit: "ms"}); err != nil {
|
||||
t.Fatalf("create: %v", err)
|
||||
}
|
||||
if err := s.UpsertMetric(ctx, Definition{Name: "m", Type: TypeCounter, Unit: "count"}); err != nil {
|
||||
t.Fatalf("upsert: %v", err)
|
||||
}
|
||||
got, err := s.GetMetric(ctx, "m")
|
||||
if err != nil {
|
||||
t.Fatalf("get: %v", err)
|
||||
}
|
||||
if got.Type != TypeCounter || got.Unit != "count" {
|
||||
t.Fatalf("upsert did not overwrite: %#v", got)
|
||||
}
|
||||
}
|
||||
|
||||
// TestTagFilterPushdownWithPaging verifies tag filtering happens before paging.
|
||||
//
|
||||
// TestTagFilterPushdownWithPaging 验证标签过滤会先于分页在 SQL 中执行。
|
||||
func TestTagFilterPushdownWithPaging(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
s := newMemStore(t)
|
||||
if err := s.CreateMetric(ctx, Definition{Name: "req", Type: TypeCounter}); err != nil {
|
||||
t.Fatalf("create: %v", err)
|
||||
}
|
||||
base := time.Date(2026, 6, 18, 0, 0, 0, 0, time.UTC)
|
||||
var batch []Point
|
||||
for i := 0; i < 6; i++ {
|
||||
env := "prod"
|
||||
if i%2 == 1 {
|
||||
env = "stage"
|
||||
}
|
||||
batch = append(batch, Point{
|
||||
MetricName: "req", EntityID: "n1",
|
||||
Timestamp: base.Add(time.Duration(i) * time.Minute),
|
||||
Value: float64(i),
|
||||
Tags: map[string]string{"env": env},
|
||||
})
|
||||
}
|
||||
if err := s.WriteBatch(ctx, batch); err != nil {
|
||||
t.Fatalf("write: %v", err)
|
||||
}
|
||||
// 3 prod points (i=0,2,4); with Limit=2 the SQL LIMIT must apply to the
|
||||
// tag-filtered set, returning the first two prod points in time order.
|
||||
got, err := s.Query(ctx, Query{
|
||||
MetricName: "req", EntityID: "n1",
|
||||
Start: base, End: base.Add(time.Hour),
|
||||
Tags: map[string]string{"env": "prod"},
|
||||
Limit: 2,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("query: %v", err)
|
||||
}
|
||||
if len(got) != 2 {
|
||||
t.Fatalf("expected 2 paged prod points, got %d", len(got))
|
||||
}
|
||||
if got[0].Value != 0 || got[1].Value != 2 {
|
||||
t.Fatalf("unexpected paged values: %#v", got)
|
||||
}
|
||||
}
|
||||
|
||||
// TestSQLAggregateMatchesInMemory compares SQL and in-memory aggregation.
|
||||
//
|
||||
// TestSQLAggregateMatchesInMemory 对比 SQL 下推聚合和内存聚合结果。
|
||||
func TestSQLAggregateMatchesInMemory(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
s := newMemStore(t)
|
||||
if err := s.CreateMetric(ctx, Definition{Name: "g", Type: TypeGauge}); err != nil {
|
||||
t.Fatalf("create: %v", err)
|
||||
}
|
||||
base := time.Date(2026, 6, 18, 0, 0, 0, 0, time.UTC)
|
||||
var batch []Point
|
||||
vals := []float64{5, 15, 25, 35, 100, 200}
|
||||
for i, v := range vals {
|
||||
batch = append(batch, Point{
|
||||
MetricName: "g", EntityID: "n1",
|
||||
Timestamp: base.Add(time.Duration(i) * time.Minute),
|
||||
Value: v,
|
||||
})
|
||||
}
|
||||
if err := s.WriteBatch(ctx, batch); err != nil {
|
||||
t.Fatalf("write: %v", err)
|
||||
}
|
||||
q := AggregateQuery{
|
||||
Query: Query{MetricName: "g", EntityID: "n1", Start: base, End: base.Add(time.Hour)},
|
||||
Aggregation: AggAvg,
|
||||
Interval: 3 * time.Minute,
|
||||
}
|
||||
// SQL pushdown path.
|
||||
sqlRes, err := s.Aggregate(ctx, q)
|
||||
if err != nil {
|
||||
t.Fatalf("sql aggregate: %v", err)
|
||||
}
|
||||
// In-memory reference over the same raw points.
|
||||
pts, err := s.Query(ctx, q.Query)
|
||||
if err != nil {
|
||||
t.Fatalf("query: %v", err)
|
||||
}
|
||||
memRes, err := AggregatePoints(pts, q)
|
||||
if err != nil {
|
||||
t.Fatalf("mem aggregate: %v", err)
|
||||
}
|
||||
if len(sqlRes) != len(memRes) {
|
||||
t.Fatalf("bucket count mismatch: sql=%d mem=%d", len(sqlRes), len(memRes))
|
||||
}
|
||||
for i := range sqlRes {
|
||||
if !sqlRes[i].Bucket.Equal(memRes[i].Bucket) || sqlRes[i].Value != memRes[i].Value || sqlRes[i].Count != memRes[i].Count {
|
||||
t.Fatalf("bucket %d mismatch: sql=%#v mem=%#v", i, sqlRes[i], memRes[i])
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestCounterRateHandlesReset verifies reset-aware counter rate calculation.
|
||||
//
|
||||
// TestCounterRateHandlesReset 验证计数器重置时速率计算仍然稳定。
|
||||
func TestCounterRateHandlesReset(t *testing.T) {
|
||||
base := time.Date(2026, 6, 18, 0, 0, 0, 0, time.UTC)
|
||||
// Counter goes 0 -> 10 -> reset -> 5; naive (last-first)/sec would give a
|
||||
// positive-but-wrong 5/30s. Correct counter rate sums positive deltas:
|
||||
// (10-0) + (5-0 after reset) = 15 over 30s = 0.5/s.
|
||||
pts := []Point{
|
||||
{Timestamp: base, Value: 0},
|
||||
{Timestamp: base.Add(10 * time.Second), Value: 10},
|
||||
{Timestamp: base.Add(20 * time.Second), Value: 0},
|
||||
{Timestamp: base.Add(30 * time.Second), Value: 5},
|
||||
}
|
||||
rate := counterRate(pts)
|
||||
if rate != 0.5 {
|
||||
t.Fatalf("expected reset-aware rate 0.5/s, got %v", rate)
|
||||
}
|
||||
}
|
||||
|
||||
// TestAlignTimeNegativeTimestamp verifies pre-epoch bucket alignment.
|
||||
//
|
||||
// TestAlignTimeNegativeTimestamp 验证 Unix epoch 之前的时间也能正确对齐桶。
|
||||
func TestAlignTimeNegativeTimestamp(t *testing.T) {
|
||||
interval := time.Minute
|
||||
// 30s before the epoch should align down to -60s, not up to 0.
|
||||
tm := time.Unix(-30, 0).UTC()
|
||||
got := alignTime(tm, interval)
|
||||
want := time.Unix(-60, 0).UTC()
|
||||
if !got.Equal(want) {
|
||||
t.Fatalf("alignTime negative: got %v want %v", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
// TestLatestReturnsMostRecent verifies latest-point ordering.
|
||||
//
|
||||
// TestLatestReturnsMostRecent 验证 Latest 返回最新采样点。
|
||||
func TestLatestReturnsMostRecent(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
s := newMemStore(t)
|
||||
if err := s.CreateMetric(ctx, Definition{Name: "l", Type: TypeGauge}); err != nil {
|
||||
t.Fatalf("create: %v", err)
|
||||
}
|
||||
base := time.Date(2026, 6, 18, 0, 0, 0, 0, time.UTC)
|
||||
for i := 0; i < 5; i++ {
|
||||
if err := s.Write(ctx, Point{MetricName: "l", EntityID: "n1", Timestamp: base.Add(time.Duration(i) * time.Minute), Value: float64(i)}); err != nil {
|
||||
t.Fatalf("write: %v", err)
|
||||
}
|
||||
}
|
||||
got, err := s.Latest(ctx, "l", "n1", 2)
|
||||
if err != nil {
|
||||
t.Fatalf("latest: %v", err)
|
||||
}
|
||||
if len(got) != 2 || got[0].Value != 4 || got[1].Value != 3 {
|
||||
t.Fatalf("unexpected latest: %#v", got)
|
||||
}
|
||||
}
|
||||
|
||||
// TestStatsDistinguishesNoDataFromUnknownMetric verifies stats error semantics.
|
||||
//
|
||||
// TestStatsDistinguishesNoDataFromUnknownMetric 验证 Stats 能区分无数据和未知指标。
|
||||
func TestStatsDistinguishesNoDataFromUnknownMetric(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
s := newMemStore(t)
|
||||
base := time.Date(2026, 6, 18, 0, 0, 0, 0, time.UTC)
|
||||
|
||||
// Unknown metric -> ErrNotFound.
|
||||
_, err := s.Stats(ctx, Query{MetricName: "nope", Start: base, End: base.Add(time.Hour)})
|
||||
if !errors.Is(err, ErrNotFound) {
|
||||
t.Fatalf("unknown metric: expected ErrNotFound, got %v", err)
|
||||
}
|
||||
|
||||
// Known metric but empty window -> ErrNoData.
|
||||
if err := s.CreateMetric(ctx, Definition{Name: "known", Type: TypeGauge}); err != nil {
|
||||
t.Fatalf("create: %v", err)
|
||||
}
|
||||
_, err = s.Stats(ctx, Query{MetricName: "known", Start: base, End: base.Add(time.Hour)})
|
||||
if !errors.Is(err, ErrNoData) {
|
||||
t.Fatalf("empty window: expected ErrNoData, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// TestStdDevPopMatchesCalculateStats verifies population standard deviation.
|
||||
//
|
||||
// TestStdDevPopMatchesCalculateStats 验证总体标准差与统计摘要一致。
|
||||
func TestStdDevPopMatchesCalculateStats(t *testing.T) {
|
||||
base := time.Date(2026, 6, 18, 0, 0, 0, 0, time.UTC)
|
||||
var pts []Point
|
||||
for i, v := range []float64{10, 20, 30, 40, 50} {
|
||||
pts = append(pts, Point{Timestamp: base.Add(time.Duration(i) * time.Minute), Value: v})
|
||||
}
|
||||
st, err := CalculateStats(pts)
|
||||
if err != nil {
|
||||
t.Fatalf("stats: %v", err)
|
||||
}
|
||||
got, _ := aggregateValue(pts, AggStdDev)
|
||||
if math.Abs(got-st.StdDev) > 1e-9 {
|
||||
t.Fatalf("AggStdDev %v != Stats.StdDev %v", got, st.StdDev)
|
||||
}
|
||||
// Known value: population stddev of 10..50 step 10 is sqrt(200) ~= 14.142135.
|
||||
if math.Abs(got-14.142135623730951) > 1e-9 {
|
||||
t.Fatalf("unexpected population stddev: %v", got)
|
||||
}
|
||||
}
|
||||
|
||||
// TestAggregateStdDevSQLiteUsesMemoryPath verifies SQLite stddev fallback.
|
||||
//
|
||||
// TestAggregateStdDevSQLiteUsesMemoryPath 验证 SQLite 标准差聚合会回退到内存路径。
|
||||
func TestAggregateStdDevSQLiteUsesMemoryPath(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
s := newMemStore(t)
|
||||
if err := s.CreateMetric(ctx, Definition{Name: "sd", Type: TypeGauge}); err != nil {
|
||||
t.Fatalf("create: %v", err)
|
||||
}
|
||||
base := time.Date(2026, 6, 18, 0, 0, 0, 0, time.UTC)
|
||||
var batch []Point
|
||||
for i, v := range []float64{10, 20, 30} {
|
||||
batch = append(batch, Point{MetricName: "sd", EntityID: "n1", Timestamp: base.Add(time.Duration(i) * time.Minute), Value: v})
|
||||
}
|
||||
if err := s.WriteBatch(ctx, batch); err != nil {
|
||||
t.Fatalf("write: %v", err)
|
||||
}
|
||||
res, err := s.Aggregate(ctx, AggregateQuery{
|
||||
Query: Query{MetricName: "sd", EntityID: "n1", Start: base, End: base.Add(time.Hour)},
|
||||
Aggregation: AggStdDev,
|
||||
Interval: time.Hour,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("aggregate stddev: %v", err)
|
||||
}
|
||||
if len(res) != 1 {
|
||||
t.Fatalf("expected 1 bucket, got %d", len(res))
|
||||
}
|
||||
// Population stddev of {10,20,30} = sqrt(200/3) ~= 8.16496.
|
||||
if math.Abs(res[0].Value-8.16496580927726) > 1e-9 {
|
||||
t.Fatalf("unexpected stddev bucket value: %v", res[0].Value)
|
||||
}
|
||||
}
|
||||
|
||||
// TestSQLAggValueExprPushdownMatrix verifies aggregation pushdown support.
|
||||
//
|
||||
// TestSQLAggValueExprPushdownMatrix 验证各后端支持的聚合下推矩阵。
|
||||
func TestSQLAggValueExprPushdownMatrix(t *testing.T) {
|
||||
cases := []struct {
|
||||
driver Driver
|
||||
agg Aggregation
|
||||
ok bool
|
||||
}{
|
||||
{DriverSQLite, AggAvg, true},
|
||||
{DriverSQLite, AggStdDev, false}, // no STDDEV_POP in sqlite -> memory
|
||||
{DriverSQLite, AggP95, false}, // no percentile in sqlite -> memory
|
||||
{DriverMySQL, AggStdDev, true}, // STDDEV_POP
|
||||
{DriverMySQL, AggP95, false}, // no portable continuous percentile
|
||||
{DriverPostgreSQL, AggStdDev, true},
|
||||
{DriverPostgreSQL, AggP50, true}, // percentile_cont
|
||||
{DriverPostgreSQL, AggP99, true},
|
||||
{DriverPostgreSQL, AggFirst, false}, // needs ordered raw series
|
||||
}
|
||||
for _, c := range cases {
|
||||
expr, ok := sqlAggValueExpr(c.driver, c.agg)
|
||||
if ok != c.ok {
|
||||
t.Fatalf("%s/%s: pushdown=%v want %v (expr=%q)", c.driver, c.agg, ok, c.ok, expr)
|
||||
}
|
||||
if ok && expr == "" {
|
||||
t.Fatalf("%s/%s: pushdown ok but empty expr", c.driver, c.agg)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestSQLiteReadPoolOpens verifies SQLite read-pool creation.
|
||||
//
|
||||
// TestSQLiteReadPoolOpens 验证 SQLite 只读连接池会按配置打开。
|
||||
func TestSQLiteReadPoolOpens(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
dir := filepath.Join(t.TempDir(), "rp")
|
||||
store, err := Open(ctx, SQLiteInDir(dir, WithSQLiteReadPool(4)))
|
||||
if err != nil {
|
||||
t.Fatalf("open with read pool: %v", err)
|
||||
}
|
||||
t.Cleanup(func() { store.Close() })
|
||||
if store.readDB == nil {
|
||||
t.Fatalf("expected a dedicated read pool to be opened")
|
||||
}
|
||||
if store.reader() != store.readDB {
|
||||
t.Fatalf("reader() should return the dedicated read pool")
|
||||
}
|
||||
// Round-trip a write (primary) and a read (read pool).
|
||||
if err := store.CreateMetric(ctx, Definition{Name: "rp", Type: TypeGauge}); err != nil {
|
||||
t.Fatalf("create: %v", err)
|
||||
}
|
||||
base := time.Date(2026, 6, 18, 0, 0, 0, 0, time.UTC)
|
||||
if err := store.Write(ctx, Point{MetricName: "rp", EntityID: "n1", Timestamp: base, Value: 7}); err != nil {
|
||||
t.Fatalf("write: %v", err)
|
||||
}
|
||||
got, err := store.Latest(ctx, "rp", "n1", 1)
|
||||
if err != nil {
|
||||
t.Fatalf("latest: %v", err)
|
||||
}
|
||||
if len(got) != 1 || got[0].Value != 7 {
|
||||
t.Fatalf("read pool round-trip failed: %#v", got)
|
||||
}
|
||||
}
|
||||
|
||||
// TestMemoryDSNSkipsReadPool verifies memory SQLite skips read pools.
|
||||
//
|
||||
// TestMemoryDSNSkipsReadPool 验证内存 SQLite 不会打开独立读池。
|
||||
func TestMemoryDSNSkipsReadPool(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
store, err := Open(ctx, SQLite("file:rp-mem?mode=memory&cache=shared", WithSQLiteReadPool(4)))
|
||||
if err != nil {
|
||||
t.Fatalf("open: %v", err)
|
||||
}
|
||||
t.Cleanup(func() { store.Close() })
|
||||
if store.readDB != nil {
|
||||
t.Fatalf("in-memory database must not open a second read pool")
|
||||
}
|
||||
}
|
||||
|
||||
// TestWriteBatchAtomicAcrossChunks verifies chunked writes are atomic.
|
||||
//
|
||||
// TestWriteBatchAtomicAcrossChunks 验证分块批量写入仍保持整体原子性。
|
||||
func TestWriteBatchAtomicAcrossChunks(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
s := newMemStore(t)
|
||||
if err := s.CreateMetric(ctx, Definition{Name: "atom", Type: TypeGauge}); err != nil {
|
||||
t.Fatalf("create: %v", err)
|
||||
}
|
||||
base := time.Date(2026, 6, 18, 0, 0, 0, 0, time.UTC)
|
||||
// 1500 valid points, then one invalid (missing entity) to force a mid-batch
|
||||
// failure on the second chunk. The whole batch must roll back.
|
||||
var batch []Point
|
||||
for i := 0; i < 1500; i++ {
|
||||
batch = append(batch, Point{MetricName: "atom", EntityID: "n1", Timestamp: base.Add(time.Duration(i) * time.Millisecond), Value: float64(i)})
|
||||
}
|
||||
batch = append(batch, Point{MetricName: "atom", EntityID: "", Timestamp: base.Add(2 * time.Second), Value: 1}) // invalid
|
||||
if err := s.WriteBatch(ctx, batch); err == nil {
|
||||
t.Fatalf("expected WriteBatch to fail on invalid point")
|
||||
}
|
||||
// Nothing should have been committed.
|
||||
pts, err := s.Query(ctx, Query{MetricName: "atom", EntityID: "n1", Start: base, End: base.Add(time.Hour)})
|
||||
if err != nil {
|
||||
t.Fatalf("query: %v", err)
|
||||
}
|
||||
if len(pts) != 0 {
|
||||
t.Fatalf("expected 0 committed points after rolled-back batch, got %d", len(pts))
|
||||
}
|
||||
}
|
||||
|
||||
// TestAggregateBucketPagingSQLPath verifies bucket paging in SQL aggregation.
|
||||
//
|
||||
// TestAggregateBucketPagingSQLPath 验证 SQL 聚合路径按桶分页。
|
||||
func TestAggregateBucketPagingSQLPath(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
s := newMemStore(t)
|
||||
if err := s.CreateMetric(ctx, Definition{Name: "bp", Type: TypeGauge}); err != nil {
|
||||
t.Fatalf("create: %v", err)
|
||||
}
|
||||
base := time.Date(2026, 6, 18, 0, 0, 0, 0, time.UTC)
|
||||
// 6 points across 6 one-minute buckets, values 0..5.
|
||||
var batch []Point
|
||||
for i := 0; i < 6; i++ {
|
||||
batch = append(batch, Point{MetricName: "bp", EntityID: "n1", Timestamp: base.Add(time.Duration(i) * time.Minute), Value: float64(i)})
|
||||
}
|
||||
if err := s.WriteBatch(ctx, batch); err != nil {
|
||||
t.Fatalf("write: %v", err)
|
||||
}
|
||||
// AggAvg -> SQL pushdown path. One bucket per minute. Page: offset 1, limit 2.
|
||||
res, err := s.Aggregate(ctx, AggregateQuery{
|
||||
Query: Query{MetricName: "bp", EntityID: "n1", Start: base, End: base.Add(time.Hour)},
|
||||
Aggregation: AggAvg,
|
||||
Interval: time.Minute,
|
||||
BucketLimit: 2,
|
||||
BucketOffset: 1,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("aggregate: %v", err)
|
||||
}
|
||||
if len(res) != 2 {
|
||||
t.Fatalf("expected 2 paged buckets, got %d", len(res))
|
||||
}
|
||||
// Buckets are values 0..5; offset 1 skips the value-0 bucket, so we expect 1,2.
|
||||
if res[0].Value != 1 || res[1].Value != 2 {
|
||||
t.Fatalf("unexpected paged bucket values: %#v", res)
|
||||
}
|
||||
}
|
||||
|
||||
// TestAggregateBucketPagingMemoryPathMatchesSQL compares bucket paging paths.
|
||||
//
|
||||
// TestAggregateBucketPagingMemoryPathMatchesSQL 对比内存聚合和 SQL 聚合的桶分页语义。
|
||||
func TestAggregateBucketPagingMemoryPathMatchesSQL(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
s := newMemStore(t)
|
||||
if err := s.CreateMetric(ctx, Definition{Name: "bp2", Type: TypeGauge}); err != nil {
|
||||
t.Fatalf("create: %v", err)
|
||||
}
|
||||
base := time.Date(2026, 6, 18, 0, 0, 0, 0, time.UTC)
|
||||
var batch []Point
|
||||
for i := 0; i < 6; i++ {
|
||||
batch = append(batch, Point{MetricName: "bp2", EntityID: "n1", Timestamp: base.Add(time.Duration(i) * time.Minute), Value: float64(i * 10)})
|
||||
}
|
||||
if err := s.WriteBatch(ctx, batch); err != nil {
|
||||
t.Fatalf("write: %v", err)
|
||||
}
|
||||
q := Query{MetricName: "bp2", EntityID: "n1", Start: base, End: base.Add(time.Hour)}
|
||||
// AggMin (SQL pushdown) and AggP50 (memory fallback on sqlite) with identical
|
||||
// bucket paging must select the same bucket window (the same timestamps).
|
||||
sqlPaged, err := s.Aggregate(ctx, AggregateQuery{Query: q, Aggregation: AggMin, Interval: time.Minute, BucketLimit: 3, BucketOffset: 2})
|
||||
if err != nil {
|
||||
t.Fatalf("sql aggregate: %v", err)
|
||||
}
|
||||
memPaged, err := s.Aggregate(ctx, AggregateQuery{Query: q, Aggregation: AggP50, Interval: time.Minute, BucketLimit: 3, BucketOffset: 2})
|
||||
if err != nil {
|
||||
t.Fatalf("mem aggregate: %v", err)
|
||||
}
|
||||
if len(sqlPaged) != 3 || len(memPaged) != 3 {
|
||||
t.Fatalf("expected 3 buckets each, got sql=%d mem=%d", len(sqlPaged), len(memPaged))
|
||||
}
|
||||
for i := range sqlPaged {
|
||||
if !sqlPaged[i].Bucket.Equal(memPaged[i].Bucket) {
|
||||
t.Fatalf("bucket window mismatch at %d: sql=%v mem=%v", i, sqlPaged[i].Bucket, memPaged[i].Bucket)
|
||||
}
|
||||
}
|
||||
// Per-bucket single point, so min == p50 == the value.
|
||||
for i := range sqlPaged {
|
||||
if sqlPaged[i].Value != memPaged[i].Value {
|
||||
t.Fatalf("value mismatch at %d: sql=%v mem=%v", i, sqlPaged[i].Value, memPaged[i].Value)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestAggregateIgnoresRawPointLimit verifies raw limits do not affect aggregation.
|
||||
//
|
||||
// TestAggregateIgnoresRawPointLimit 验证原始点分页参数不会影响聚合输入。
|
||||
func TestAggregateIgnoresRawPointLimit(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
s := newMemStore(t)
|
||||
if err := s.CreateMetric(ctx, Definition{Name: "ig", Type: TypeGauge}); err != nil {
|
||||
t.Fatalf("create: %v", err)
|
||||
}
|
||||
base := time.Date(2026, 6, 18, 0, 0, 0, 0, time.UTC)
|
||||
var batch []Point
|
||||
for i := 0; i < 6; i++ {
|
||||
batch = append(batch, Point{MetricName: "ig", EntityID: "n1", Timestamp: base.Add(time.Duration(i) * time.Minute), Value: float64(i)})
|
||||
}
|
||||
if err := s.WriteBatch(ctx, batch); err != nil {
|
||||
t.Fatalf("write: %v", err)
|
||||
}
|
||||
// The embedded Query.Limit must be ignored for aggregation: a single 1-hour
|
||||
// bucket should aggregate ALL 6 points (count 6), not just 2.
|
||||
res, err := s.Aggregate(ctx, AggregateQuery{
|
||||
Query: Query{MetricName: "ig", EntityID: "n1", Start: base, End: base.Add(time.Hour), Limit: 2},
|
||||
Aggregation: AggP95, // memory path, where the bug would have surfaced
|
||||
Interval: time.Hour,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("aggregate: %v", err)
|
||||
}
|
||||
if len(res) != 1 || res[0].Count != 6 {
|
||||
t.Fatalf("embedded Limit leaked into aggregation: %#v", res)
|
||||
}
|
||||
}
|
||||
|
||||
// TestJSONTagKeyWithSpecialChars verifies JSON tag keys with special characters.
|
||||
//
|
||||
// TestJSONTagKeyWithSpecialChars 验证包含特殊字符的标签键可正确查询。
|
||||
func TestJSONTagKeyWithSpecialChars(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
s := newMemStore(t)
|
||||
if err := s.CreateMetric(ctx, Definition{Name: "tk", Type: TypeGauge}); err != nil {
|
||||
t.Fatalf("create: %v", err)
|
||||
}
|
||||
base := time.Date(2026, 6, 18, 0, 0, 0, 0, time.UTC)
|
||||
// Keys containing a dot, a hyphen, a space, and a quote.
|
||||
tags := map[string]string{
|
||||
"region.zone": "ap-1",
|
||||
"a-b": "yes",
|
||||
"with space": "ok",
|
||||
}
|
||||
if err := s.Write(ctx, Point{MetricName: "tk", EntityID: "n1", Timestamp: base, Value: 1, Tags: tags}); err != nil {
|
||||
t.Fatalf("write: %v", err)
|
||||
}
|
||||
// A second point that should NOT match the filter below.
|
||||
if err := s.Write(ctx, Point{MetricName: "tk", EntityID: "n1", Timestamp: base.Add(time.Minute), Value: 2, Tags: map[string]string{"region.zone": "eu-1"}}); err != nil {
|
||||
t.Fatalf("write 2: %v", err)
|
||||
}
|
||||
|
||||
for _, tc := range []struct {
|
||||
key, val string
|
||||
want int
|
||||
}{
|
||||
{"region.zone", "ap-1", 1}, // dotted key must match the flat key, not nested
|
||||
{"a-b", "yes", 1}, // hyphen
|
||||
{"with space", "ok", 1}, // space
|
||||
{"region.zone", "eu-1", 1}, // the other point
|
||||
} {
|
||||
got, err := s.Query(ctx, Query{
|
||||
MetricName: "tk", EntityID: "n1",
|
||||
Start: base.Add(-time.Second), End: base.Add(time.Hour),
|
||||
Tags: map[string]string{tc.key: tc.val},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("query %s=%s: %v", tc.key, tc.val, err)
|
||||
}
|
||||
if len(got) != tc.want {
|
||||
t.Fatalf("tag %q=%q: expected %d points, got %d", tc.key, tc.val, tc.want, len(got))
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,83 @@
|
||||
package metric
|
||||
|
||||
import (
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
)
|
||||
|
||||
// encodeMap encodes a string map as JSON.
|
||||
//
|
||||
// encodeMap 将字符串 map 编码为 JSON 字符串,nil 会按空对象处理。
|
||||
func encodeMap(m map[string]string) (string, error) {
|
||||
if m == nil {
|
||||
m = map[string]string{}
|
||||
}
|
||||
b, err := json.Marshal(m)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return string(b), nil
|
||||
}
|
||||
|
||||
// decodeMap decodes a scanned JSON value into a string map.
|
||||
//
|
||||
// decodeMap 将数据库扫描出的 JSON 值还原为字符串 map。
|
||||
func decodeMap(v any) (map[string]string, error) {
|
||||
switch x := v.(type) {
|
||||
case nil:
|
||||
return map[string]string{}, nil
|
||||
case string:
|
||||
return decodeMapString(x)
|
||||
case []byte:
|
||||
return decodeMapString(string(x))
|
||||
default:
|
||||
return nil, fmt.Errorf("unsupported json value type %T", v)
|
||||
}
|
||||
}
|
||||
|
||||
// decodeMapString decodes a JSON object string into a string map.
|
||||
//
|
||||
// decodeMapString 将 JSON 字符串解码为字符串 map。
|
||||
func decodeMapString(s string) (map[string]string, error) {
|
||||
if s == "" {
|
||||
return map[string]string{}, nil
|
||||
}
|
||||
var out map[string]string
|
||||
if err := json.Unmarshal([]byte(s), &out); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if out == nil {
|
||||
out = map[string]string{}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// canonicalTags returns a deterministic JSON encoding of a tag map. encodeMap
|
||||
// already relies on encoding/json, which sorts map keys, so equal tag maps
|
||||
// always produce identical bytes — the property tagsFingerprint depends on.
|
||||
//
|
||||
// canonicalTags 返回标签 map 的确定性 JSON 编码;相同标签集合总会得到
|
||||
// 相同字节序列,tagsFingerprint 依赖这个性质。
|
||||
func canonicalTags(m map[string]string) (string, error) {
|
||||
return encodeMap(m)
|
||||
}
|
||||
|
||||
// tagsFingerprint returns a stable hex fingerprint of a tag set, used as the
|
||||
// rollups table's tags_hash key column. Equal tag maps (regardless of Go map
|
||||
// iteration order) hash identically, so each distinct tag combination becomes
|
||||
// its own rollup series; an empty/nil tag map hashes to the fingerprint of "{}".
|
||||
//
|
||||
// tagsFingerprint 为标签集合生成稳定的十六进制指纹,用作 rollups 表的
|
||||
// tags_hash key 列。相同标签 map(无论 Go map 迭代顺序如何)都会得到相同 hash,
|
||||
// 因此每种不同标签组合都会成为自己的 rollup 序列;空或 nil 标签 map 会得到 "{}"
|
||||
// 的指纹。
|
||||
func tagsFingerprint(m map[string]string) (hash string, canonical string, err error) {
|
||||
canonical, err = canonicalTags(m)
|
||||
if err != nil {
|
||||
return "", "", err
|
||||
}
|
||||
sum := sha256.Sum256([]byte(canonical))
|
||||
return hex.EncodeToString(sum[:]), canonical, nil
|
||||
}
|
||||
@@ -0,0 +1,182 @@
|
||||
package metric
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
)
|
||||
|
||||
// Migrate creates the tables and indexes used by the metric store.
|
||||
//
|
||||
// Migrate 创建 metric 包需要的表和索引。
|
||||
func (s *Store) Migrate(ctx context.Context) error {
|
||||
if err := s.ensureOpen(); err != nil {
|
||||
return err
|
||||
}
|
||||
d := s.dialect
|
||||
jsonType := d.jsonType()
|
||||
pk := d.autoIncrementPrimaryKey()
|
||||
blob := d.blobType()
|
||||
|
||||
statements := []string{
|
||||
fmt.Sprintf(`CREATE TABLE IF NOT EXISTS %s (
|
||||
name VARCHAR(191) PRIMARY KEY,
|
||||
type VARCHAR(32) NOT NULL,
|
||||
unit VARCHAR(64) NOT NULL DEFAULT '',
|
||||
description TEXT NOT NULL DEFAULT '',
|
||||
retention_days INTEGER NOT NULL DEFAULT 0,
|
||||
metadata %s NOT NULL,
|
||||
created_at BIGINT NOT NULL,
|
||||
updated_at BIGINT NOT NULL
|
||||
)`, s.tables.definitions, jsonType),
|
||||
fmt.Sprintf(`CREATE TABLE IF NOT EXISTS %s (
|
||||
id %s,
|
||||
metric_name VARCHAR(191) NOT NULL,
|
||||
entity_id VARCHAR(191) NOT NULL,
|
||||
tags_hash VARCHAR(64) NOT NULL,
|
||||
ts_nano BIGINT NOT NULL,
|
||||
value DOUBLE PRECISION NOT NULL,
|
||||
tags %s NOT NULL,
|
||||
labels %s NOT NULL,
|
||||
created_at BIGINT NOT NULL,
|
||||
UNIQUE(metric_name, entity_id, tags_hash, ts_nano)
|
||||
)`, s.tables.points, pk, jsonType, jsonType),
|
||||
fmt.Sprintf(`CREATE INDEX IF NOT EXISTS %s_points_metric_entity_time_idx ON %s (metric_name, entity_id, ts_nano)`, s.cfg.TablePrefix, s.tables.points),
|
||||
fmt.Sprintf(`CREATE INDEX IF NOT EXISTS %s_points_metric_time_idx ON %s (metric_name, ts_nano)`, s.cfg.TablePrefix, s.tables.points),
|
||||
fmt.Sprintf(`CREATE INDEX IF NOT EXISTS %s_points_entity_time_idx ON %s (entity_id, ts_nano)`, s.cfg.TablePrefix, s.tables.points),
|
||||
fmt.Sprintf(`CREATE TABLE IF NOT EXISTS %s (
|
||||
id %s,
|
||||
metric_name VARCHAR(191) NOT NULL,
|
||||
entity_id VARCHAR(191) NOT NULL,
|
||||
tags_hash VARCHAR(64) NOT NULL,
|
||||
tags %s NOT NULL,
|
||||
resolution_nano BIGINT NOT NULL,
|
||||
bucket_nano BIGINT NOT NULL,
|
||||
count BIGINT NOT NULL,
|
||||
sum DOUBLE PRECISION NOT NULL,
|
||||
sum_sq DOUBLE PRECISION NOT NULL,
|
||||
min_val DOUBLE PRECISION NOT NULL,
|
||||
max_val DOUBLE PRECISION NOT NULL,
|
||||
first_val DOUBLE PRECISION NOT NULL,
|
||||
first_ts BIGINT NOT NULL,
|
||||
last_val DOUBLE PRECISION NOT NULL,
|
||||
last_ts BIGINT NOT NULL,
|
||||
digest %s,
|
||||
created_at BIGINT NOT NULL,
|
||||
UNIQUE(metric_name, entity_id, tags_hash, resolution_nano, bucket_nano)
|
||||
)`, s.tables.rollups, pk, jsonType, blob),
|
||||
fmt.Sprintf(`CREATE INDEX IF NOT EXISTS %s_rollups_lookup_idx ON %s (metric_name, entity_id, tags_hash, resolution_nano, bucket_nano)`, s.cfg.TablePrefix, s.tables.rollups),
|
||||
fmt.Sprintf(`CREATE INDEX IF NOT EXISTS %s_rollups_res_time_idx ON %s (metric_name, resolution_nano, bucket_nano)`, s.cfg.TablePrefix, s.tables.rollups),
|
||||
}
|
||||
|
||||
if s.cfg.Driver == DriverMySQL {
|
||||
statements = []string{
|
||||
fmt.Sprintf(`CREATE TABLE IF NOT EXISTS %s (
|
||||
name VARCHAR(191) PRIMARY KEY,
|
||||
type VARCHAR(32) NOT NULL,
|
||||
unit VARCHAR(64) NOT NULL DEFAULT '',
|
||||
description TEXT NOT NULL,
|
||||
retention_days INT NOT NULL DEFAULT 0,
|
||||
metadata %s NOT NULL,
|
||||
created_at BIGINT NOT NULL,
|
||||
updated_at BIGINT NOT NULL
|
||||
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci`, s.tables.definitions, jsonType),
|
||||
fmt.Sprintf(`CREATE TABLE IF NOT EXISTS %s (
|
||||
id %s,
|
||||
metric_name VARCHAR(191) NOT NULL,
|
||||
entity_id VARCHAR(191) NOT NULL,
|
||||
tags_hash VARCHAR(64) NOT NULL,
|
||||
ts_nano BIGINT NOT NULL,
|
||||
value DOUBLE NOT NULL,
|
||||
tags %s NOT NULL,
|
||||
labels %s NOT NULL,
|
||||
created_at BIGINT NOT NULL,
|
||||
UNIQUE KEY uq_metric_entity_time (metric_name, entity_id, tags_hash, ts_nano),
|
||||
INDEX idx_metric_entity_time (metric_name, entity_id, ts_nano),
|
||||
INDEX idx_metric_time (metric_name, ts_nano),
|
||||
INDEX idx_entity_time (entity_id, ts_nano)
|
||||
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci`, s.tables.points, pk, jsonType, jsonType),
|
||||
fmt.Sprintf(`CREATE TABLE IF NOT EXISTS %s (
|
||||
id %s,
|
||||
metric_name VARCHAR(191) NOT NULL,
|
||||
entity_id VARCHAR(191) NOT NULL,
|
||||
tags_hash VARCHAR(64) NOT NULL,
|
||||
tags %s NOT NULL,
|
||||
resolution_nano BIGINT NOT NULL,
|
||||
bucket_nano BIGINT NOT NULL,
|
||||
count BIGINT NOT NULL,
|
||||
sum DOUBLE NOT NULL,
|
||||
sum_sq DOUBLE NOT NULL,
|
||||
min_val DOUBLE NOT NULL,
|
||||
max_val DOUBLE NOT NULL,
|
||||
first_val DOUBLE NOT NULL,
|
||||
first_ts BIGINT NOT NULL,
|
||||
last_val DOUBLE NOT NULL,
|
||||
last_ts BIGINT NOT NULL,
|
||||
digest %s,
|
||||
created_at BIGINT NOT NULL,
|
||||
UNIQUE KEY uq_rollup (metric_name, entity_id, tags_hash, resolution_nano, bucket_nano),
|
||||
INDEX idx_rollup_res_time (metric_name, resolution_nano, bucket_nano)
|
||||
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci`, s.tables.rollups, pk, jsonType, blob),
|
||||
}
|
||||
}
|
||||
|
||||
if s.cfg.Driver == DriverPostgreSQL {
|
||||
statements = []string{
|
||||
fmt.Sprintf(`CREATE TABLE IF NOT EXISTS %s (
|
||||
name VARCHAR(191) PRIMARY KEY,
|
||||
type VARCHAR(32) NOT NULL,
|
||||
unit VARCHAR(64) NOT NULL DEFAULT '',
|
||||
description TEXT NOT NULL DEFAULT '',
|
||||
retention_days INTEGER NOT NULL DEFAULT 0,
|
||||
metadata %s NOT NULL,
|
||||
created_at BIGINT NOT NULL,
|
||||
updated_at BIGINT NOT NULL
|
||||
)`, s.tables.definitions, jsonType),
|
||||
fmt.Sprintf(`CREATE TABLE IF NOT EXISTS %s (
|
||||
id %s,
|
||||
metric_name VARCHAR(191) NOT NULL,
|
||||
entity_id VARCHAR(191) NOT NULL,
|
||||
tags_hash VARCHAR(64) NOT NULL,
|
||||
ts_nano BIGINT NOT NULL,
|
||||
value DOUBLE PRECISION NOT NULL,
|
||||
tags %s NOT NULL,
|
||||
labels %s NOT NULL,
|
||||
created_at BIGINT NOT NULL,
|
||||
UNIQUE(metric_name, entity_id, tags_hash, ts_nano)
|
||||
)`, s.tables.points, pk, jsonType, jsonType),
|
||||
fmt.Sprintf(`CREATE INDEX IF NOT EXISTS %s_points_metric_entity_time_idx ON %s (metric_name, entity_id, ts_nano)`, s.cfg.TablePrefix, s.tables.points),
|
||||
fmt.Sprintf(`CREATE INDEX IF NOT EXISTS %s_points_metric_time_idx ON %s (metric_name, ts_nano)`, s.cfg.TablePrefix, s.tables.points),
|
||||
fmt.Sprintf(`CREATE INDEX IF NOT EXISTS %s_points_entity_time_idx ON %s (entity_id, ts_nano)`, s.cfg.TablePrefix, s.tables.points),
|
||||
fmt.Sprintf(`CREATE TABLE IF NOT EXISTS %s (
|
||||
id %s,
|
||||
metric_name VARCHAR(191) NOT NULL,
|
||||
entity_id VARCHAR(191) NOT NULL,
|
||||
tags_hash VARCHAR(64) NOT NULL,
|
||||
tags %s NOT NULL,
|
||||
resolution_nano BIGINT NOT NULL,
|
||||
bucket_nano BIGINT NOT NULL,
|
||||
count BIGINT NOT NULL,
|
||||
sum DOUBLE PRECISION NOT NULL,
|
||||
sum_sq DOUBLE PRECISION NOT NULL,
|
||||
min_val DOUBLE PRECISION NOT NULL,
|
||||
max_val DOUBLE PRECISION NOT NULL,
|
||||
first_val DOUBLE PRECISION NOT NULL,
|
||||
first_ts BIGINT NOT NULL,
|
||||
last_val DOUBLE PRECISION NOT NULL,
|
||||
last_ts BIGINT NOT NULL,
|
||||
digest %s,
|
||||
created_at BIGINT NOT NULL,
|
||||
UNIQUE(metric_name, entity_id, tags_hash, resolution_nano, bucket_nano)
|
||||
)`, s.tables.rollups, pk, jsonType, blob),
|
||||
fmt.Sprintf(`CREATE INDEX IF NOT EXISTS %s_rollups_lookup_idx ON %s (metric_name, entity_id, tags_hash, resolution_nano, bucket_nano)`, s.cfg.TablePrefix, s.tables.rollups),
|
||||
fmt.Sprintf(`CREATE INDEX IF NOT EXISTS %s_rollups_res_time_idx ON %s (metric_name, resolution_nano, bucket_nano)`, s.cfg.TablePrefix, s.tables.rollups),
|
||||
}
|
||||
}
|
||||
|
||||
for _, stmt := range statements {
|
||||
if _, err := s.db.ExecContext(ctx, stmt); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,76 @@
|
||||
package metric
|
||||
|
||||
import (
|
||||
"strconv"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// Pxx builds the Aggregation for an arbitrary percentile. The argument is a
|
||||
// percentage in (0,100): Pxx(99.9) -> "p99.9", Pxx(50) -> "p50". The fixed
|
||||
// AggP50/AggP95/AggP99 constants are just the common cases of this same string
|
||||
// form, so they keep working unchanged.
|
||||
//
|
||||
// This is what turns the package from "p50/p95/p99 only" into "any percentile":
|
||||
// callers can ask for p75, p90, p99.99, etc., and every path (in-memory,
|
||||
// SQL pushdown, and rollup-via-t-digest) understands it.
|
||||
//
|
||||
// Pxx 根据任意百分位构造 Aggregation。参数是 (0,100) 内的百分比:
|
||||
// Pxx(99.9) -> "p99.9",Pxx(50) -> "p50"。固定的 AggP50/AggP95/AggP99
|
||||
// 常量只是同一字符串形式的常用情况,因此会保持原有行为。
|
||||
//
|
||||
// 这让 package 从“只支持 p50/p95/p99”变成“支持任意百分位”:调用方可以请求
|
||||
// p75、p90、p99.99 等,并且每条路径(内存、SQL 下推、基于 t-digest 的 rollup)
|
||||
// 都能理解它。
|
||||
func Pxx(p float64) Aggregation {
|
||||
// Trim trailing zeros so Pxx(95) == AggP95 ("p95"), not "p95.000000".
|
||||
s := strconv.FormatFloat(p, 'f', -1, 64)
|
||||
return Aggregation("p" + s)
|
||||
}
|
||||
|
||||
// parsePercentile reports whether agg names a percentile and, if so, returns
|
||||
// the corresponding fraction in [0,1]. "p99.9" -> 0.999. Out-of-range
|
||||
// percentages (<=0 or >=100) are rejected so validation can reject them.
|
||||
//
|
||||
// parsePercentile 判断 agg 是否命名了百分位;如果是,则返回对应的 [0,1] 小数。
|
||||
// 例如 "p99.9" -> 0.999。越界百分比(<=0 或 >=100)会被拒绝,以便校验逻辑
|
||||
// 能拒绝它们。
|
||||
func parsePercentile(agg Aggregation) (float64, bool) {
|
||||
s := string(agg)
|
||||
if len(s) < 2 || (s[0] != 'p' && s[0] != 'P') {
|
||||
return 0, false
|
||||
}
|
||||
pct, err := strconv.ParseFloat(s[1:], 64)
|
||||
if err != nil {
|
||||
return 0, false
|
||||
}
|
||||
if pct <= 0 || pct >= 100 {
|
||||
return 0, false
|
||||
}
|
||||
return pct / 100, true
|
||||
}
|
||||
|
||||
// isPercentile reports whether agg is any percentile aggregation.
|
||||
//
|
||||
// isPercentile 判断聚合类型是否为任意百分位聚合。
|
||||
func isPercentile(agg Aggregation) bool {
|
||||
_, ok := parsePercentile(agg)
|
||||
return ok
|
||||
}
|
||||
|
||||
// percentileFractionString renders the fraction for SQL percentile_cont, e.g.
|
||||
// "p99.9" -> "0.999". Trailing zeros are trimmed for stable SQL text.
|
||||
//
|
||||
// percentileFractionString 把百分位聚合转换为 SQL percentile_cont 需要的
|
||||
// 小数字符串,并去掉尾随零以保持 SQL 文本稳定。
|
||||
func percentileFractionString(agg Aggregation) (string, bool) {
|
||||
f, ok := parsePercentile(agg)
|
||||
if !ok {
|
||||
return "", false
|
||||
}
|
||||
s := strconv.FormatFloat(f, 'f', -1, 64)
|
||||
if !strings.Contains(s, ".") {
|
||||
// f is in (0,1) so this should not happen, but guard anyway.
|
||||
s = strconv.FormatFloat(f, 'f', 1, 64)
|
||||
}
|
||||
return s, true
|
||||
}
|
||||
@@ -0,0 +1,184 @@
|
||||
package metric
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// TestPostgreSQLIntegration runs the PostgreSQL integration test when configured.
|
||||
//
|
||||
// TestPostgreSQLIntegration 在配置 DSN 后运行 PostgreSQL 集成测试。
|
||||
func TestPostgreSQLIntegration(t *testing.T) {
|
||||
dsn := os.Getenv("METRIC_POSTGRES_DSN")
|
||||
if dsn == "" {
|
||||
t.Skip("METRIC_POSTGRES_DSN is not set")
|
||||
}
|
||||
|
||||
runSQLIntegration(t, "postgres", PostgreSQL(dsn), true)
|
||||
}
|
||||
|
||||
// TestMySQLIntegration runs the MySQL integration test when configured.
|
||||
//
|
||||
// TestMySQLIntegration 在配置 DSN 后运行 MySQL 集成测试。
|
||||
func TestMySQLIntegration(t *testing.T) {
|
||||
dsn := os.Getenv("METRIC_MYSQL_DSN")
|
||||
if dsn == "" {
|
||||
t.Skip("METRIC_MYSQL_DSN is not set")
|
||||
}
|
||||
|
||||
runSQLIntegration(t, "mysql", MySQL(dsn), false)
|
||||
}
|
||||
|
||||
// runSQLIntegration exercises the SQL store against an external database.
|
||||
//
|
||||
// runSQLIntegration 在外部数据库上执行通用 SQL 集成测试流程。
|
||||
func runSQLIntegration(t *testing.T, name string, cfg Config, expectSQLPercentile bool) {
|
||||
t.Helper()
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||||
defer cancel()
|
||||
|
||||
prefix := fmt.Sprintf("it_%d_", time.Now().UnixNano())
|
||||
cfg.TablePrefix = prefix
|
||||
store, err := Open(ctx, cfg)
|
||||
if err != nil {
|
||||
t.Fatalf("open %s store: %v", name, err)
|
||||
}
|
||||
defer store.Close()
|
||||
defer dropIntegrationTables(t, store, prefix)
|
||||
|
||||
if err := store.CreateMetric(ctx, Definition{
|
||||
Name: "http.latency",
|
||||
Type: TypeGauge,
|
||||
Unit: "ms",
|
||||
Description: "HTTP latency",
|
||||
Metadata: map[string]string{"source": "integration"},
|
||||
}); err != nil {
|
||||
t.Fatalf("create metric: %v", err)
|
||||
}
|
||||
if err := store.CreateMetric(ctx, Definition{Name: "http.latency", Type: TypeGauge}); err == nil {
|
||||
t.Fatalf("duplicate CreateMetric should fail")
|
||||
}
|
||||
if err := store.UpsertMetric(ctx, Definition{Name: "http.latency", Type: TypeGauge, Unit: "milliseconds"}); err != nil {
|
||||
t.Fatalf("upsert metric: %v", err)
|
||||
}
|
||||
def, err := store.GetMetric(ctx, "http.latency")
|
||||
if err != nil {
|
||||
t.Fatalf("get metric: %v", err)
|
||||
}
|
||||
if def.Unit != "milliseconds" {
|
||||
t.Fatalf("upsert did not update unit: %#v", def)
|
||||
}
|
||||
|
||||
base := time.Date(2026, 6, 20, 1, 0, 0, 0, time.UTC)
|
||||
points := []Point{
|
||||
{MetricName: "http.latency", EntityID: "api-1", Timestamp: base, Value: 10, Tags: map[string]string{"region.zone": "ap-1", "route-name": "/v1/nodes"}},
|
||||
{MetricName: "http.latency", EntityID: "api-1", Timestamp: base.Add(time.Minute), Value: 20, Tags: map[string]string{"region.zone": "ap-1", "route-name": "/v1/nodes"}},
|
||||
{MetricName: "http.latency", EntityID: "api-1", Timestamp: base.Add(2 * time.Minute), Value: 30, Tags: map[string]string{"region.zone": "ap-1", "route-name": "/v1/nodes"}},
|
||||
{MetricName: "http.latency", EntityID: "api-1", Timestamp: base.Add(3 * time.Minute), Value: 40, Tags: map[string]string{"region.zone": "eu-1", "route-name": "/v1/nodes"}},
|
||||
}
|
||||
if err := store.WriteBatch(ctx, points); err != nil {
|
||||
t.Fatalf("write batch: %v", err)
|
||||
}
|
||||
|
||||
got, err := store.Query(ctx, Query{
|
||||
MetricName: "http.latency",
|
||||
EntityID: "api-1",
|
||||
Start: base.Add(-time.Second),
|
||||
End: base.Add(5 * time.Minute),
|
||||
Tags: map[string]string{"region.zone": "ap-1", "route-name": "/v1/nodes"},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("query with json tags: %v", err)
|
||||
}
|
||||
if len(got) != 3 {
|
||||
t.Fatalf("expected 3 ap-1 points, got %d: %#v", len(got), got)
|
||||
}
|
||||
|
||||
stats, err := store.Stats(ctx, Query{
|
||||
MetricName: "http.latency",
|
||||
EntityID: "api-1",
|
||||
Start: base,
|
||||
End: base.Add(5 * time.Minute),
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("stats: %v", err)
|
||||
}
|
||||
if stats.Count != 4 || stats.Avg != 25 {
|
||||
t.Fatalf("unexpected stats: %#v", stats)
|
||||
}
|
||||
|
||||
avgBuckets, err := store.Aggregate(ctx, AggregateQuery{
|
||||
Query: Query{
|
||||
MetricName: "http.latency",
|
||||
EntityID: "api-1",
|
||||
Start: base,
|
||||
End: base.Add(5 * time.Minute),
|
||||
},
|
||||
Aggregation: AggAvg,
|
||||
Interval: 2 * time.Minute,
|
||||
BucketLimit: 1,
|
||||
BucketOffset: 1,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("avg aggregate: %v", err)
|
||||
}
|
||||
if len(avgBuckets) != 1 || avgBuckets[0].Value != 35 || avgBuckets[0].Count != 2 {
|
||||
t.Fatalf("unexpected bucket-paged avg aggregate: %#v", avgBuckets)
|
||||
}
|
||||
|
||||
p95Buckets, err := store.Aggregate(ctx, AggregateQuery{
|
||||
Query: Query{
|
||||
MetricName: "http.latency",
|
||||
EntityID: "api-1",
|
||||
Start: base,
|
||||
End: base.Add(5 * time.Minute),
|
||||
},
|
||||
Aggregation: AggP95,
|
||||
Interval: 10 * time.Minute,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("p95 aggregate: %v", err)
|
||||
}
|
||||
if len(p95Buckets) != 1 || p95Buckets[0].Count != 4 || p95Buckets[0].Value <= 38 {
|
||||
t.Fatalf("unexpected p95 aggregate: %#v", p95Buckets)
|
||||
}
|
||||
if _, ok := sqlAggValueExpr(cfg.Driver, AggP95); ok != expectSQLPercentile {
|
||||
t.Fatalf("%s p95 pushdown expectation mismatch: got %v want %v", name, ok, expectSQLPercentile)
|
||||
}
|
||||
|
||||
latest, err := store.Latest(ctx, "http.latency", "api-1", 1)
|
||||
if err != nil {
|
||||
t.Fatalf("latest: %v", err)
|
||||
}
|
||||
if len(latest) != 1 || latest[0].Value != 40 {
|
||||
t.Fatalf("unexpected latest point: %#v", latest)
|
||||
}
|
||||
|
||||
deleted, err := store.DeleteBefore(ctx, "http.latency", base.Add(2*time.Minute))
|
||||
if err != nil {
|
||||
t.Fatalf("delete before: %v", err)
|
||||
}
|
||||
if deleted != 2 {
|
||||
t.Fatalf("expected 2 deleted points, got %d", deleted)
|
||||
}
|
||||
}
|
||||
|
||||
// dropIntegrationTables drops integration-test tables.
|
||||
//
|
||||
// dropIntegrationTables 删除集成测试创建的表。
|
||||
func dropIntegrationTables(t *testing.T, store *Store, prefix string) {
|
||||
t.Helper()
|
||||
if strings.TrimSpace(prefix) == "" {
|
||||
t.Fatal("refusing to drop tables with empty prefix")
|
||||
}
|
||||
for _, name := range []string{prefix + "rollups", prefix + "points", prefix + "definitions"} {
|
||||
if _, err := store.db.Exec("DROP TABLE IF EXISTS " + name); err != nil {
|
||||
t.Fatalf("drop integration table %s: %v", name, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,320 @@
|
||||
package metric
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"math"
|
||||
"time"
|
||||
)
|
||||
|
||||
// RollupTier describes one downsampled resolution: raw points (or the next
|
||||
// finer tier) are aggregated into buckets Interval wide, and those buckets are
|
||||
// kept for Retention. A policy lists tiers from finest to coarsest, e.g.
|
||||
//
|
||||
// 1m kept 7d -> 5m kept 30d -> 1h kept 1y
|
||||
//
|
||||
// As data ages it falls off the finer tiers and survives only in coarser ones,
|
||||
// so storage shrinks with age — the "data gets sparser as it gets older"
|
||||
// behavior of a downsampling TSDB.
|
||||
//
|
||||
// RollupTier 描述一个降采样分辨率:原始点(或下一层更细的层级)会被聚合进
|
||||
// Interval 宽的桶,而这些桶会保留 Retention。策略会按从细到粗排列层级,例如:
|
||||
//
|
||||
// 1m 保留 7d -> 5m 保留 30d -> 1h 保留 1y
|
||||
//
|
||||
// 随着数据变旧,它会从更细层级中过期,只在更粗层级中保留下来,因此存储量会随
|
||||
// 数据年龄下降,也就是降采样 TSDB 中“数据越旧越稀疏”的行为。
|
||||
type RollupTier struct {
|
||||
// Interval is the bucket width for this rollup tier.
|
||||
//
|
||||
// Interval 是该 rollup 层的桶宽。
|
||||
Interval time.Duration `json:"interval"`
|
||||
// Retention is how long buckets in this tier are kept.
|
||||
//
|
||||
// Retention 是该层桶的保留时间。
|
||||
Retention time.Duration `json:"retention"`
|
||||
}
|
||||
|
||||
// RollupPolicy is the full retention ladder for a store: how long raw points
|
||||
// live, then a chain of progressively coarser tiers. Compact materializes the
|
||||
// tiers and enforces every retention window.
|
||||
//
|
||||
// RollupPolicy 描述完整保留阶梯:原始点保留多久,以及后续逐级变粗的
|
||||
// rollup 层;Compact 会物化这些层并执行保留窗口。
|
||||
type RollupPolicy struct {
|
||||
// RawRetention is how long raw points are kept before Compact deletes them
|
||||
// (after they have been rolled into the finest tier). Zero means "never
|
||||
// delete raw" — rollups are still built, but raw is retained.
|
||||
//
|
||||
// RawRetention 是 Compact 删除原始点之前保留它们的时间(在它们已进入最细
|
||||
// rollup 层之后)。零值表示“永不删除原始点”;rollup 仍会构建,但原始点保留。
|
||||
RawRetention time.Duration `json:"raw_retention"`
|
||||
// Tiers are ordered finest-first. Each Interval must be a positive integer
|
||||
// multiple of the previous tier's Interval (so a coarse bucket is composed
|
||||
// of whole finer buckets), and each Retention must be >= the previous
|
||||
// tier's Retention (coarse data outlives fine data).
|
||||
//
|
||||
// Tiers 按从细到粗排序。每个 Interval 必须是前一层 Interval 的正整数倍
|
||||
// (这样粗桶由完整细桶组成),每个 Retention 必须 >= 前一层 Retention
|
||||
// (粗数据比细数据活得更久)。
|
||||
Tiers []RollupTier `json:"tiers"`
|
||||
// Compression tunes the per-bucket t-digest (size vs. percentile accuracy).
|
||||
// <=1 uses the default (100).
|
||||
//
|
||||
// Compression 调整每个桶的 t-digest(大小与百分位精度之间的取舍)。
|
||||
// <=1 时使用默认值(100)。
|
||||
Compression float64 `json:"compression"`
|
||||
}
|
||||
|
||||
// Enabled reports whether the policy actually defines any rollup tiers.
|
||||
//
|
||||
// Enabled 表示策略是否实际定义了 rollup 层级。
|
||||
func (p RollupPolicy) Enabled() bool { return len(p.Tiers) > 0 }
|
||||
|
||||
// compression returns the effective t-digest compression for a rollup policy.
|
||||
//
|
||||
// compression 返回 rollup 策略实际使用的 t-digest 压缩参数。
|
||||
func (p RollupPolicy) compression() float64 {
|
||||
if p.Compression <= 1 {
|
||||
return defaultTDigestCompression
|
||||
}
|
||||
return p.Compression
|
||||
}
|
||||
|
||||
func (p RollupPolicy) rawCutoff(now time.Time) time.Time {
|
||||
if p.RawRetention <= 0 || len(p.Tiers) == 0 {
|
||||
return time.Time{}
|
||||
}
|
||||
cutoff := now.UTC().Add(-p.RawRetention)
|
||||
interval := p.Tiers[0].Interval.Nanoseconds()
|
||||
return time.Unix(0, floorDivNano(cutoff.UnixNano(), interval)).UTC()
|
||||
}
|
||||
|
||||
// Validate enforces the structural rules that make cascading composition and
|
||||
// retention well-defined.
|
||||
//
|
||||
// Validate 检查策略结构是否满足级联合成和保留语义所需的约束。
|
||||
func (p RollupPolicy) Validate() error {
|
||||
if len(p.Tiers) == 0 {
|
||||
return nil // a store with no tiers simply does no rollup work
|
||||
}
|
||||
if p.RawRetention < 0 {
|
||||
return fmt.Errorf("%w: raw retention cannot be negative", ErrInvalidArgument)
|
||||
}
|
||||
var prev RollupTier
|
||||
for i, tr := range p.Tiers {
|
||||
if tr.Interval <= 0 {
|
||||
return fmt.Errorf("%w: tier %d interval must be positive", ErrInvalidArgument, i)
|
||||
}
|
||||
if tr.Retention <= 0 {
|
||||
return fmt.Errorf("%w: tier %d retention must be positive", ErrInvalidArgument, i)
|
||||
}
|
||||
if i == 0 {
|
||||
// The finest tier rolls up raw points. Require raw retention (when
|
||||
// set) to cover at least two of its buckets so the in-progress and
|
||||
// most-recent sealed bucket still have their raw backing when Compact
|
||||
// runs and deletes old raw.
|
||||
if p.RawRetention > 0 && p.RawRetention < 2*tr.Interval {
|
||||
return fmt.Errorf("%w: raw retention must be >= 2x the finest tier interval", ErrInvalidArgument)
|
||||
}
|
||||
} else {
|
||||
if tr.Interval <= prev.Interval {
|
||||
return fmt.Errorf("%w: tier %d interval must be larger than tier %d", ErrInvalidArgument, i, i-1)
|
||||
}
|
||||
if tr.Interval%prev.Interval != 0 {
|
||||
return fmt.Errorf("%w: tier %d interval must be a multiple of tier %d interval", ErrInvalidArgument, i, i-1)
|
||||
}
|
||||
if tr.Retention < prev.Retention {
|
||||
return fmt.Errorf("%w: tier %d retention must be >= tier %d retention", ErrInvalidArgument, i, i-1)
|
||||
}
|
||||
}
|
||||
prev = tr
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// rollupBucket is the in-memory accumulator for one (metric, entity,
|
||||
// resolution, bucket) cell. It carries exactly the summaries that can be
|
||||
// re-aggregated losslessly when composing coarser tiers — count/sum/sumSq for
|
||||
// avg & population stddev, min/max, first/last by timestamp — plus a t-digest
|
||||
// so arbitrary percentiles survive downsampling with bounded error.
|
||||
//
|
||||
// rollupBucket 是单个(指标、实体、分辨率、桶)单元的内存累加器。它只携带在
|
||||
// 合成更粗层级时可无损重新聚合的摘要:用于平均值和总体标准差的 count/sum/sumSq,
|
||||
// min/max,按时间记录的 first/last,外加一个 t-digest,让任意百分位能以
|
||||
// 有界误差在降采样后保留下来。
|
||||
type rollupBucket struct {
|
||||
// count is the total number of raw points represented.
|
||||
//
|
||||
// count 是该桶代表的原始点总数。
|
||||
count int64
|
||||
// sum is the sum of represented values.
|
||||
//
|
||||
// sum 是该桶代表值的总和。
|
||||
sum float64
|
||||
// sumSq is the sum of squared values for population stddev.
|
||||
//
|
||||
// sumSq 是用于总体标准差的平方和。
|
||||
sumSq float64
|
||||
// min is the minimum represented value.
|
||||
//
|
||||
// min 是该桶代表的最小值。
|
||||
min float64
|
||||
// max is the maximum represented value.
|
||||
//
|
||||
// max 是该桶代表的最大值。
|
||||
max float64
|
||||
// firstVal is the value with the earliest timestamp.
|
||||
//
|
||||
// firstVal 是时间戳最早的值。
|
||||
firstVal float64
|
||||
// firstTS is the earliest timestamp in nanoseconds.
|
||||
//
|
||||
// firstTS 是最早时间戳的纳秒值。
|
||||
firstTS int64
|
||||
// lastVal is the value with the latest timestamp.
|
||||
//
|
||||
// lastVal 是时间戳最晚的值。
|
||||
lastVal float64
|
||||
// lastTS is the latest timestamp in nanoseconds.
|
||||
//
|
||||
// lastTS 是最晚时间戳的纳秒值。
|
||||
lastTS int64
|
||||
// digest estimates percentiles for represented values.
|
||||
//
|
||||
// digest 用于估算该桶代表值的百分位。
|
||||
digest *TDigest
|
||||
// tagsHash carries the stable tag-set fingerprint for this rollup cell.
|
||||
//
|
||||
// tagsHash 携带该 rollup 单元的稳定标签集合指纹。
|
||||
tagsHash string
|
||||
// tagsJSON carries the canonical tag map written back to the tags column.
|
||||
//
|
||||
// tagsJSON 携带写回 tags 列的规范标签 map。
|
||||
tagsJSON string
|
||||
}
|
||||
|
||||
// newRollupBucket creates an empty rollup accumulator.
|
||||
//
|
||||
// newRollupBucket 创建一个空的 rollup 累加器。
|
||||
func newRollupBucket(compression float64) *rollupBucket {
|
||||
return &rollupBucket{
|
||||
min: 0,
|
||||
max: 0,
|
||||
firstTS: 0,
|
||||
lastTS: 0,
|
||||
digest: NewTDigest(compression),
|
||||
}
|
||||
}
|
||||
|
||||
// addPoint folds a raw observation into the bucket.
|
||||
//
|
||||
// addPoint 将一个原始观测值合入当前桶。
|
||||
func (b *rollupBucket) addPoint(value float64, tsNano int64) {
|
||||
if b.count == 0 {
|
||||
b.min, b.max = value, value
|
||||
b.firstVal, b.firstTS = value, tsNano
|
||||
b.lastVal, b.lastTS = value, tsNano
|
||||
} else {
|
||||
if value < b.min {
|
||||
b.min = value
|
||||
}
|
||||
if value > b.max {
|
||||
b.max = value
|
||||
}
|
||||
if tsNano < b.firstTS {
|
||||
b.firstVal, b.firstTS = value, tsNano
|
||||
}
|
||||
if tsNano > b.lastTS {
|
||||
b.lastVal, b.lastTS = value, tsNano
|
||||
}
|
||||
}
|
||||
b.count++
|
||||
b.sum += value
|
||||
b.sumSq += value * value
|
||||
b.digest.Add(value, 1)
|
||||
}
|
||||
|
||||
// mergeStored folds a finer rollup row (already-summarized) into this coarser
|
||||
// bucket. This is the cascade step: tier i+1 buckets are built by merging the
|
||||
// tier i rows they span.
|
||||
//
|
||||
// mergeStored 将一个已汇总的细层 rollup 行合入当前更粗桶。这是级联步骤:
|
||||
// tier i+1 的桶通过合并其覆盖的 tier i 行构建。
|
||||
func (b *rollupBucket) mergeStored(o *rollupBucket) {
|
||||
if o.count == 0 {
|
||||
return
|
||||
}
|
||||
if b.count == 0 {
|
||||
b.min, b.max = o.min, o.max
|
||||
b.firstVal, b.firstTS = o.firstVal, o.firstTS
|
||||
b.lastVal, b.lastTS = o.lastVal, o.lastTS
|
||||
} else {
|
||||
if o.min < b.min {
|
||||
b.min = o.min
|
||||
}
|
||||
if o.max > b.max {
|
||||
b.max = o.max
|
||||
}
|
||||
if o.firstTS < b.firstTS {
|
||||
b.firstVal, b.firstTS = o.firstVal, o.firstTS
|
||||
}
|
||||
if o.lastTS > b.lastTS {
|
||||
b.lastVal, b.lastTS = o.lastVal, o.lastTS
|
||||
}
|
||||
}
|
||||
b.count += o.count
|
||||
b.sum += o.sum
|
||||
b.sumSq += o.sumSq
|
||||
if b.digest == nil {
|
||||
b.digest = NewTDigest(defaultTDigestCompression)
|
||||
}
|
||||
b.digest.Merge(o.digest)
|
||||
}
|
||||
|
||||
// value computes the requested aggregation from the bucket summaries. ok=false
|
||||
// means the aggregation is not derivable from a rollup (only AggRate, which
|
||||
// needs the ordered raw series).
|
||||
//
|
||||
// value 从桶摘要中计算请求的聚合值;ok=false 表示该聚合无法由 rollup 推导
|
||||
// (目前主要是需要有序原始序列的 AggRate)。
|
||||
func (b *rollupBucket) value(agg Aggregation) (float64, bool) {
|
||||
switch agg {
|
||||
case AggAvg:
|
||||
if b.count == 0 {
|
||||
return 0, true
|
||||
}
|
||||
return b.sum / float64(b.count), true
|
||||
case AggMin:
|
||||
return b.min, true
|
||||
case AggMax:
|
||||
return b.max, true
|
||||
case AggSum:
|
||||
return b.sum, true
|
||||
case AggCount:
|
||||
return float64(b.count), true
|
||||
case AggFirst:
|
||||
return b.firstVal, true
|
||||
case AggLast:
|
||||
return b.lastVal, true
|
||||
case AggStdDev:
|
||||
if b.count == 0 {
|
||||
return 0, true
|
||||
}
|
||||
mean := b.sum / float64(b.count)
|
||||
variance := b.sumSq/float64(b.count) - mean*mean
|
||||
if variance < 0 {
|
||||
variance = 0 // floating-point guard
|
||||
}
|
||||
return math.Sqrt(variance), true
|
||||
case AggRate:
|
||||
return 0, false // rate needs the ordered raw series; not in a rollup
|
||||
default:
|
||||
if frac, ok := parsePercentile(agg); ok {
|
||||
if b.digest == nil || b.digest.Count() == 0 {
|
||||
return 0, true
|
||||
}
|
||||
return b.digest.Quantile(frac), true
|
||||
}
|
||||
return 0, false
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,135 @@
|
||||
package metric
|
||||
|
||||
import (
|
||||
"context"
|
||||
"math"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// TestSeriesHybridKeepsUncompactedRawNearCutoff verifies the hybrid read does
|
||||
// not lose raw points that fall between the (finest-aligned) raw cutoff and the
|
||||
// next bucket boundary, including a point written AFTER the last compaction.
|
||||
//
|
||||
// Reproduction: now=12:00:30, RawRetention=30m so the cutoff is 11:30:30, which
|
||||
// the policy floors to 11:30:00 (finest = 1m). A point at 11:30:15 written
|
||||
// before compaction is kept (>= aligned cutoff) and NOT rolled up; a point at
|
||||
// 11:30:45 written after compaction is also raw. Both belong to the 11:00 1h
|
||||
// output bucket. The old code aligned the split UP to 11:31:00 and read raw from
|
||||
// there, dropping both boundary points (count=1, the stale rollup half only).
|
||||
// Serving raw from the aligned cutoff fixes it (count=2).
|
||||
//
|
||||
// TestSeriesHybridKeepsUncompactedRawNearCutoff 验证混合读取不会丢失落在
|
||||
// (最细对齐后的)原始 cutoff 与下一个桶边界之间的 raw 点,包括上次 compact 之后
|
||||
// 才写入的点。
|
||||
func TestSeriesHybridKeepsUncompactedRawNearCutoff(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
policy := RollupPolicy{
|
||||
RawRetention: 30 * time.Minute,
|
||||
Tiers: []RollupTier{{Interval: time.Minute, Retention: 24 * time.Hour}},
|
||||
}
|
||||
s := newRollupStore(t, policy)
|
||||
if err := s.CreateMetric(ctx, Definition{Name: "nearcut", Type: TypeGauge}); err != nil {
|
||||
t.Fatalf("create: %v", err)
|
||||
}
|
||||
now := time.Date(2026, 6, 18, 12, 0, 30, 0, time.UTC)
|
||||
// 11:30:15 is >= the finest-aligned cutoff (11:30:00) -> survives compaction
|
||||
// as raw and is not rolled up.
|
||||
before := time.Date(2026, 6, 18, 11, 30, 15, 0, time.UTC)
|
||||
if err := s.Write(ctx, Point{MetricName: "nearcut", EntityID: "n1", Timestamp: before, Value: 10}); err != nil {
|
||||
t.Fatalf("write before: %v", err)
|
||||
}
|
||||
if _, err := s.Compact(ctx, now); err != nil {
|
||||
t.Fatalf("compact: %v", err)
|
||||
}
|
||||
// Written after compaction: only ever lives in raw.
|
||||
after := time.Date(2026, 6, 18, 11, 30, 45, 0, time.UTC)
|
||||
if err := s.Write(ctx, Point{MetricName: "nearcut", EntityID: "n1", Timestamp: after, Value: 100}); err != nil {
|
||||
t.Fatalf("write after: %v", err)
|
||||
}
|
||||
|
||||
got, err := s.Series(ctx, AggregateQuery{
|
||||
Query: Query{MetricName: "nearcut", EntityID: "n1", Start: time.Date(2026, 6, 18, 11, 0, 0, 0, time.UTC), End: now},
|
||||
Aggregation: AggAvg,
|
||||
Interval: time.Hour,
|
||||
}, now)
|
||||
if err != nil {
|
||||
t.Fatalf("series: %v", err)
|
||||
}
|
||||
bucket := alignTime(time.Date(2026, 6, 18, 11, 0, 0, 0, time.UTC), time.Hour)
|
||||
var found bool
|
||||
for _, p := range got {
|
||||
if p.Bucket.Equal(bucket) {
|
||||
found = true
|
||||
if p.Count != 2 {
|
||||
t.Fatalf("near-cutoff bucket count: got %d want 2 (%#v)", p.Count, p)
|
||||
}
|
||||
if math.Abs(p.Value-55) > 1e-9 {
|
||||
t.Fatalf("near-cutoff bucket avg: got %v want 55 (%#v)", p.Value, p)
|
||||
}
|
||||
}
|
||||
}
|
||||
if !found {
|
||||
t.Fatalf("expected 11:00 bucket with both raw points: %#v", got)
|
||||
}
|
||||
}
|
||||
|
||||
// TestSeriesHybridEndEqualsCutoff verifies a query whose End lands exactly on the
|
||||
// raw cutoff still includes the raw point sitting at the cutoff, instead of
|
||||
// returning empty.
|
||||
//
|
||||
// Reproduction: now=12:00:00, RawRetention=30m -> cutoff 11:30:00. A point at
|
||||
// exactly 11:30:00 is kept by compaction (not < cutoff) and lives in raw. An old
|
||||
// point at 11:00:10 is rolled up and its raw deleted. Querying [11:00, 11:30:00]
|
||||
// (End == cutoff) must combine the rolled-up old point and the raw cutoff point.
|
||||
// The old entry gate required End strictly AFTER the cutoff, so this fell through
|
||||
// to the pure-rollup path and returned the cutoff point's bucket empty.
|
||||
//
|
||||
// TestSeriesHybridEndEqualsCutoff 验证 End 恰好等于原始 cutoff 的查询仍会包含
|
||||
// 落在 cutoff 上的 raw 点,而不是返回空。
|
||||
func TestSeriesHybridEndEqualsCutoff(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
policy := RollupPolicy{
|
||||
RawRetention: 30 * time.Minute,
|
||||
Tiers: []RollupTier{{Interval: time.Minute, Retention: 24 * time.Hour}},
|
||||
}
|
||||
s := newRollupStore(t, policy)
|
||||
if err := s.CreateMetric(ctx, Definition{Name: "endcut", Type: TypeGauge}); err != nil {
|
||||
t.Fatalf("create: %v", err)
|
||||
}
|
||||
now := time.Date(2026, 6, 18, 12, 0, 0, 0, time.UTC)
|
||||
cutoff := time.Date(2026, 6, 18, 11, 30, 0, 0, time.UTC)
|
||||
old := time.Date(2026, 6, 18, 11, 0, 10, 0, time.UTC) // rolled up, raw deleted
|
||||
if err := s.Write(ctx, Point{MetricName: "endcut", EntityID: "n1", Timestamp: old, Value: 10}); err != nil {
|
||||
t.Fatalf("write old: %v", err)
|
||||
}
|
||||
// Point exactly at the cutoff: kept as raw (not < cutoff) after compaction.
|
||||
if err := s.Write(ctx, Point{MetricName: "endcut", EntityID: "n1", Timestamp: cutoff, Value: 100}); err != nil {
|
||||
t.Fatalf("write cutoff: %v", err)
|
||||
}
|
||||
if _, err := s.Compact(ctx, now); err != nil {
|
||||
t.Fatalf("compact: %v", err)
|
||||
}
|
||||
|
||||
got, err := s.Series(ctx, AggregateQuery{
|
||||
Query: Query{MetricName: "endcut", EntityID: "n1", Start: time.Date(2026, 6, 18, 11, 0, 0, 0, time.UTC), End: cutoff},
|
||||
Aggregation: AggSum,
|
||||
Interval: time.Hour,
|
||||
}, now)
|
||||
if err != nil {
|
||||
t.Fatalf("series: %v", err)
|
||||
}
|
||||
bucket := alignTime(time.Date(2026, 6, 18, 11, 0, 0, 0, time.UTC), time.Hour)
|
||||
var found bool
|
||||
for _, p := range got {
|
||||
if p.Bucket.Equal(bucket) {
|
||||
found = true
|
||||
if p.Count != 2 || math.Abs(p.Value-110) > 1e-9 {
|
||||
t.Fatalf("End==cutoff bucket: got count=%d value=%v want count=2 value=110", p.Count, p.Value)
|
||||
}
|
||||
}
|
||||
}
|
||||
if !found {
|
||||
t.Fatalf("expected 11:00 bucket including the cutoff raw point: %#v", got)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,347 @@
|
||||
package metric
|
||||
|
||||
import (
|
||||
"context"
|
||||
"math"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// TestAggregateRollupExcludesPartialBucket verifies the rollup query no longer
|
||||
// drags an out-of-window sample in through a partially-overlapping bucket.
|
||||
//
|
||||
// Two samples share one 1m rollup bucket [00:00:00, 00:01:00): 00:00:10=100 and
|
||||
// 00:00:50=10. A query window of [00:00:40, 00:01:30] only overlaps part of that
|
||||
// bucket. Because a rollup bucket is an indivisible summary, the old "any
|
||||
// overlap" rule folded the whole bucket in and counted 00:00:10 (value 100),
|
||||
// which lies OUTSIDE the window. Full containment excludes the straddling
|
||||
// bucket entirely, so the out-of-window sample can no longer leak in.
|
||||
//
|
||||
// TestAggregateRollupExcludesPartialBucket 验证 rollup 查询不再通过部分重叠的桶
|
||||
// 把窗口外的样本带进来。
|
||||
func TestAggregateRollupExcludesPartialBucket(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
policy := RollupPolicy{Tiers: []RollupTier{{Interval: time.Minute, Retention: 24 * time.Hour}}}
|
||||
s := newRollupStore(t, policy)
|
||||
if err := s.CreateMetric(ctx, Definition{Name: "win", Type: TypeGauge}); err != nil {
|
||||
t.Fatalf("create: %v", err)
|
||||
}
|
||||
base := time.Date(2026, 6, 18, 0, 0, 0, 0, time.UTC)
|
||||
batch := []Point{
|
||||
{MetricName: "win", EntityID: "n1", Timestamp: base.Add(10 * time.Second), Value: 100}, // out of window
|
||||
{MetricName: "win", EntityID: "n1", Timestamp: base.Add(50 * time.Second), Value: 10}, // in window
|
||||
}
|
||||
if err := s.WriteBatch(ctx, batch); err != nil {
|
||||
t.Fatalf("write: %v", err)
|
||||
}
|
||||
if _, err := s.Compact(ctx, base.Add(time.Hour)); err != nil {
|
||||
t.Fatalf("compact: %v", err)
|
||||
}
|
||||
|
||||
// Window [00:00:40, 00:01:30] partially overlaps the single 1m bucket.
|
||||
res, err := s.AggregateRollup(ctx, AggregateQuery{
|
||||
Query: Query{MetricName: "win", EntityID: "n1", Start: base.Add(40 * time.Second), End: base.Add(90 * time.Second)},
|
||||
Aggregation: AggSum,
|
||||
Interval: time.Minute,
|
||||
}, time.Minute)
|
||||
if err != nil {
|
||||
t.Fatalf("aggregate rollup: %v", err)
|
||||
}
|
||||
// The partially-overlapping bucket is excluded under full containment, so the
|
||||
// out-of-window 00:00:10 sample (value 100) must not appear. Concretely: no
|
||||
// returned bucket may carry the out-of-window value or a count of 2.
|
||||
for _, p := range res {
|
||||
if p.Count >= 2 || p.Value == 100 {
|
||||
t.Fatalf("partial bucket leaked out-of-window sample: %#v", p)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestAggregateRollupEndBoundaryInclusive verifies the rollup query end boundary
|
||||
// is inclusive (matching the raw query's ts <= end) and that a window aligned to
|
||||
// resolution boundaries returns every whole bucket it covers.
|
||||
//
|
||||
// Buckets at 00:00 and 00:01 are both whole-contained in [00:00:00, 00:02:00]:
|
||||
// the second bucket's window is [00:01:00, 00:02:00), whose last covered nano is
|
||||
// < 00:02:00, so an inclusive end keeps it. The previous half-open (< end) rule
|
||||
// could drop a bucket sitting exactly on the end edge.
|
||||
//
|
||||
// TestAggregateRollupEndBoundaryInclusive 验证 rollup 查询 end 边界为闭区间
|
||||
// (与 raw 的 ts <= end 一致),且对齐到分辨率边界的窗口会返回它覆盖的每个完整桶。
|
||||
func TestAggregateRollupEndBoundaryInclusive(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
policy := RollupPolicy{Tiers: []RollupTier{{Interval: time.Minute, Retention: 24 * time.Hour}}}
|
||||
s := newRollupStore(t, policy)
|
||||
if err := s.CreateMetric(ctx, Definition{Name: "edge", Type: TypeGauge}); err != nil {
|
||||
t.Fatalf("create: %v", err)
|
||||
}
|
||||
base := time.Date(2026, 6, 18, 0, 0, 0, 0, time.UTC)
|
||||
batch := []Point{
|
||||
{MetricName: "edge", EntityID: "n1", Timestamp: base.Add(10 * time.Second), Value: 1}, // bucket 00:00
|
||||
{MetricName: "edge", EntityID: "n1", Timestamp: base.Add(time.Minute + 10*time.Second), Value: 2}, // bucket 00:01
|
||||
}
|
||||
if err := s.WriteBatch(ctx, batch); err != nil {
|
||||
t.Fatalf("write: %v", err)
|
||||
}
|
||||
if _, err := s.Compact(ctx, base.Add(time.Hour)); err != nil {
|
||||
t.Fatalf("compact: %v", err)
|
||||
}
|
||||
|
||||
res, err := s.AggregateRollup(ctx, AggregateQuery{
|
||||
Query: Query{MetricName: "edge", EntityID: "n1", Start: base, End: base.Add(2 * time.Minute)},
|
||||
Aggregation: AggSum,
|
||||
Interval: time.Minute,
|
||||
}, time.Minute)
|
||||
if err != nil {
|
||||
t.Fatalf("aggregate rollup: %v", err)
|
||||
}
|
||||
if len(res) != 2 {
|
||||
t.Fatalf("aligned window should return both whole buckets, got %d: %#v", len(res), res)
|
||||
}
|
||||
if res[0].Value != 1 || res[1].Value != 2 {
|
||||
t.Fatalf("unexpected bucket values: %#v", res)
|
||||
}
|
||||
}
|
||||
|
||||
// TestSeriesHybridMergesStraddlingBucket verifies a query output bucket that
|
||||
// straddles the raw-retention boundary aggregates BOTH its rollup half and its
|
||||
// raw half, instead of letting raw replace the rollup half.
|
||||
//
|
||||
// With a 30m raw retention and now=12:00 the boundary is 11:30. A 1h output
|
||||
// bucket [11:00, 12:00) therefore straddles it. An 11:05 sample (value 10, older
|
||||
// than the boundary -> rolled up, raw deleted) and an 11:35 sample (value 100,
|
||||
// within retention -> still raw) both belong to that 11:00 bucket. The correct
|
||||
// answer folds both: count=2, avg=(10+100)/2=55. The old merge-by-bucket-time
|
||||
// path kept only the raw half and reported count=1, avg=100.
|
||||
//
|
||||
// TestSeriesHybridMergesStraddlingBucket 验证跨越原始保留期边界的输出桶会同时
|
||||
// 聚合 rollup 半边和 raw 半边,而不是让 raw 覆盖 rollup 半边。
|
||||
func TestSeriesHybridMergesStraddlingBucket(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
policy := RollupPolicy{
|
||||
RawRetention: 30 * time.Minute,
|
||||
Tiers: []RollupTier{{Interval: time.Minute, Retention: 24 * time.Hour}},
|
||||
}
|
||||
s := newRollupStore(t, policy)
|
||||
if err := s.CreateMetric(ctx, Definition{Name: "straddle", Type: TypeGauge}); err != nil {
|
||||
t.Fatalf("create: %v", err)
|
||||
}
|
||||
now := time.Date(2026, 6, 18, 12, 0, 0, 0, time.UTC)
|
||||
// 11:05 is older than the 11:30 cutoff (rolled up, raw deleted on compact);
|
||||
// 11:35 is within retention (kept raw).
|
||||
old := now.Add(-55 * time.Minute) // 11:05
|
||||
recent := now.Add(-25 * time.Minute) // 11:35
|
||||
batch := []Point{
|
||||
{MetricName: "straddle", EntityID: "n1", Timestamp: old, Value: 10},
|
||||
{MetricName: "straddle", EntityID: "n1", Timestamp: recent, Value: 100},
|
||||
}
|
||||
if err := s.WriteBatch(ctx, batch); err != nil {
|
||||
t.Fatalf("write: %v", err)
|
||||
}
|
||||
if _, err := s.Compact(ctx, now); err != nil {
|
||||
t.Fatalf("compact: %v", err)
|
||||
}
|
||||
|
||||
// Confirm the old raw point was deleted and the recent one survives, so the
|
||||
// two halves genuinely come from different sources.
|
||||
oldRaw, err := s.Query(ctx, Query{MetricName: "straddle", EntityID: "n1", Start: old.Add(-time.Minute), End: old.Add(time.Minute)})
|
||||
if err != nil {
|
||||
t.Fatalf("query old raw: %v", err)
|
||||
}
|
||||
if len(oldRaw) != 0 {
|
||||
t.Fatalf("expected old raw point deleted by retention, got %d", len(oldRaw))
|
||||
}
|
||||
|
||||
got, err := s.Series(ctx, AggregateQuery{
|
||||
Query: Query{MetricName: "straddle", EntityID: "n1", Start: now.Add(-time.Hour), End: now},
|
||||
Aggregation: AggAvg,
|
||||
Interval: time.Hour,
|
||||
}, now)
|
||||
if err != nil {
|
||||
t.Fatalf("series: %v", err)
|
||||
}
|
||||
|
||||
bucket := alignTime(now.Add(-time.Hour), time.Hour) // 11:00
|
||||
var found bool
|
||||
for _, p := range got {
|
||||
if p.Bucket.Equal(bucket) {
|
||||
found = true
|
||||
if p.Count != 2 {
|
||||
t.Fatalf("straddling bucket count: got %d want 2 (%#v)", p.Count, p)
|
||||
}
|
||||
if math.Abs(p.Value-55) > 1e-9 {
|
||||
t.Fatalf("straddling bucket avg: got %v want 55 (%#v)", p.Value, p)
|
||||
}
|
||||
}
|
||||
}
|
||||
if !found {
|
||||
t.Fatalf("expected a merged 11:00 bucket in result: %#v", got)
|
||||
}
|
||||
}
|
||||
|
||||
// TestSeriesHybridSumAcrossBoundary is the AggSum analogue: the straddling 1h
|
||||
// bucket must sum both halves (10 + 100 = 110), proving the merge is at the
|
||||
// summary level rather than a bucket-time replacement.
|
||||
//
|
||||
// TestSeriesHybridSumAcrossBoundary 是 AggSum 版本:跨边界的 1h 桶必须把两半求和
|
||||
// (10 + 100 = 110),证明合并发生在摘要层面,而不是按桶时间替换。
|
||||
func TestSeriesHybridSumAcrossBoundary(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
policy := RollupPolicy{
|
||||
RawRetention: 30 * time.Minute,
|
||||
Tiers: []RollupTier{{Interval: time.Minute, Retention: 24 * time.Hour}},
|
||||
}
|
||||
s := newRollupStore(t, policy)
|
||||
if err := s.CreateMetric(ctx, Definition{Name: "sumstraddle", Type: TypeGauge}); err != nil {
|
||||
t.Fatalf("create: %v", err)
|
||||
}
|
||||
now := time.Date(2026, 6, 18, 12, 0, 0, 0, time.UTC)
|
||||
batch := []Point{
|
||||
{MetricName: "sumstraddle", EntityID: "n1", Timestamp: now.Add(-55 * time.Minute), Value: 10},
|
||||
{MetricName: "sumstraddle", EntityID: "n1", Timestamp: now.Add(-25 * time.Minute), Value: 100},
|
||||
}
|
||||
if err := s.WriteBatch(ctx, batch); err != nil {
|
||||
t.Fatalf("write: %v", err)
|
||||
}
|
||||
if _, err := s.Compact(ctx, now); err != nil {
|
||||
t.Fatalf("compact: %v", err)
|
||||
}
|
||||
got, err := s.Series(ctx, AggregateQuery{
|
||||
Query: Query{MetricName: "sumstraddle", EntityID: "n1", Start: now.Add(-time.Hour), End: now},
|
||||
Aggregation: AggSum,
|
||||
Interval: time.Hour,
|
||||
}, now)
|
||||
if err != nil {
|
||||
t.Fatalf("series: %v", err)
|
||||
}
|
||||
bucket := alignTime(now.Add(-time.Hour), time.Hour)
|
||||
var found bool
|
||||
for _, p := range got {
|
||||
if p.Bucket.Equal(bucket) {
|
||||
found = true
|
||||
if p.Count != 2 || math.Abs(p.Value-110) > 1e-9 {
|
||||
t.Fatalf("straddling sum bucket: got count=%d value=%v want count=2 value=110", p.Count, p.Value)
|
||||
}
|
||||
}
|
||||
}
|
||||
if !found {
|
||||
t.Fatalf("expected merged sum bucket: %#v", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSeriesHybridIncludesRawBetweenCutoffAndNextBucket(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
policy := RollupPolicy{
|
||||
RawRetention: 30 * time.Minute,
|
||||
Tiers: []RollupTier{{Interval: time.Minute, Retention: 24 * time.Hour}},
|
||||
}
|
||||
s := newRollupStore(t, policy)
|
||||
if err := s.CreateMetric(ctx, Definition{Name: "cutgap", Type: TypeGauge}); err != nil {
|
||||
t.Fatalf("create: %v", err)
|
||||
}
|
||||
|
||||
now := time.Date(2026, 6, 18, 12, 0, 30, 0, time.UTC)
|
||||
old := time.Date(2026, 6, 18, 11, 30, 10, 0, time.UTC)
|
||||
if err := s.Write(ctx, Point{MetricName: "cutgap", EntityID: "n1", Timestamp: old, Value: 10}); err != nil {
|
||||
t.Fatalf("write old: %v", err)
|
||||
}
|
||||
if _, err := s.Compact(ctx, now); err != nil {
|
||||
t.Fatalf("compact: %v", err)
|
||||
}
|
||||
recentUncompacted := time.Date(2026, 6, 18, 11, 30, 45, 0, time.UTC)
|
||||
if err := s.Write(ctx, Point{MetricName: "cutgap", EntityID: "n1", Timestamp: recentUncompacted, Value: 100}); err != nil {
|
||||
t.Fatalf("write recent: %v", err)
|
||||
}
|
||||
|
||||
got, err := s.Series(ctx, AggregateQuery{
|
||||
Query: Query{MetricName: "cutgap", EntityID: "n1", Start: time.Date(2026, 6, 18, 11, 0, 0, 0, time.UTC), End: now},
|
||||
Aggregation: AggCount,
|
||||
Interval: time.Hour,
|
||||
}, now)
|
||||
if err != nil {
|
||||
t.Fatalf("series: %v", err)
|
||||
}
|
||||
if len(got) != 1 || got[0].Count != 2 {
|
||||
t.Fatalf("series should include rollup plus raw at the aligned cutoff, got %#v", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSeriesHybridEndAtCutoffIncludesRawBoundary(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
policy := RollupPolicy{
|
||||
RawRetention: 30 * time.Minute,
|
||||
Tiers: []RollupTier{{Interval: time.Minute, Retention: 24 * time.Hour}},
|
||||
}
|
||||
s := newRollupStore(t, policy)
|
||||
if err := s.CreateMetric(ctx, Definition{Name: "cutedge", Type: TypeGauge}); err != nil {
|
||||
t.Fatalf("create: %v", err)
|
||||
}
|
||||
|
||||
now := time.Date(2026, 6, 18, 12, 0, 30, 0, time.UTC)
|
||||
cutoff := policy.rawCutoff(now)
|
||||
if err := s.Write(ctx, Point{MetricName: "cutedge", EntityID: "n1", Timestamp: cutoff, Value: 7}); err != nil {
|
||||
t.Fatalf("write: %v", err)
|
||||
}
|
||||
if _, err := s.Compact(ctx, now); err != nil {
|
||||
t.Fatalf("compact: %v", err)
|
||||
}
|
||||
|
||||
got, err := s.Series(ctx, AggregateQuery{
|
||||
Query: Query{MetricName: "cutedge", EntityID: "n1", Start: cutoff.Add(-time.Hour), End: cutoff},
|
||||
Aggregation: AggCount,
|
||||
Interval: time.Hour,
|
||||
}, now)
|
||||
if err != nil {
|
||||
t.Fatalf("series: %v", err)
|
||||
}
|
||||
var total int
|
||||
for _, p := range got {
|
||||
total += p.Count
|
||||
}
|
||||
if total != 1 {
|
||||
t.Fatalf("series should include raw point exactly at cutoff/end, got %#v", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCompactDoesNotDoubleCountRawBucketAfterCutoffAdvances(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
policy := RollupPolicy{
|
||||
RawRetention: 30 * time.Minute,
|
||||
Tiers: []RollupTier{{Interval: time.Minute, Retention: 24 * time.Hour}},
|
||||
}
|
||||
s := newRollupStore(t, policy)
|
||||
if err := s.CreateMetric(ctx, Definition{Name: "nodup", Type: TypeGauge}); err != nil {
|
||||
t.Fatalf("create: %v", err)
|
||||
}
|
||||
|
||||
now := time.Date(2026, 6, 18, 12, 0, 30, 0, time.UTC)
|
||||
if err := s.Write(ctx, Point{MetricName: "nodup", EntityID: "n1", Timestamp: time.Date(2026, 6, 18, 11, 30, 10, 0, time.UTC), Value: 10}); err != nil {
|
||||
t.Fatalf("write old: %v", err)
|
||||
}
|
||||
if _, err := s.Compact(ctx, now); err != nil {
|
||||
t.Fatalf("compact 1: %v", err)
|
||||
}
|
||||
if err := s.Write(ctx, Point{MetricName: "nodup", EntityID: "n1", Timestamp: time.Date(2026, 6, 18, 11, 30, 45, 0, time.UTC), Value: 100}); err != nil {
|
||||
t.Fatalf("write recent: %v", err)
|
||||
}
|
||||
now = now.Add(time.Minute)
|
||||
if _, err := s.Compact(ctx, now); err != nil {
|
||||
t.Fatalf("compact 2: %v", err)
|
||||
}
|
||||
|
||||
got, err := s.Series(ctx, AggregateQuery{
|
||||
Query: Query{MetricName: "nodup", EntityID: "n1", Start: time.Date(2026, 6, 18, 11, 0, 0, 0, time.UTC), End: now},
|
||||
Aggregation: AggCount,
|
||||
Interval: time.Hour,
|
||||
}, now)
|
||||
if err != nil {
|
||||
t.Fatalf("series: %v", err)
|
||||
}
|
||||
var total int
|
||||
for _, p := range got {
|
||||
total += p.Count
|
||||
}
|
||||
if total != 2 {
|
||||
t.Fatalf("compaction should not double-count a bucket after cutoff advances, got %#v", got)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,503 @@
|
||||
package metric
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// AggregateRollup answers an AggregateQuery from a stored rollup tier instead of
|
||||
// raw points. resolution names which tier to read (it must match a tier
|
||||
// Interval that Compact has materialized). The query Interval must be a positive
|
||||
// integer multiple of resolution, so each output bucket is composed of whole
|
||||
// rollup buckets.
|
||||
//
|
||||
// query.Tags is honored: only rollup series whose stored tag set matches the
|
||||
// filter are folded in, so a tag filter selects the same data it would over raw
|
||||
// points (each distinct tag combination is its own rollup series). With no tag
|
||||
// filter, all tag series within a bucket merge together, matching the raw path's
|
||||
// cross-tag aggregation.
|
||||
//
|
||||
// Every aggregation works except AggRate, which needs the ordered raw series and
|
||||
// is therefore raw-only. Percentiles (p50, p95, p99, and arbitrary pXX) are
|
||||
// answered by merging the per-bucket t-digests, so they survive downsampling
|
||||
// with bounded error.
|
||||
//
|
||||
// AggregateRollup 从已存储的 rollup 层回答 AggregateQuery,而不是读取原始点。
|
||||
// resolution 指定要读取的层级(必须匹配 Compact 已物化的某个层级 Interval)。
|
||||
// 查询的 Interval 必须是 resolution 的正整数倍,因此每个输出桶都由完整的
|
||||
// rollup 桶组成。
|
||||
//
|
||||
// query.Tags 会被遵守:只有存储标签集合匹配过滤条件的 rollup 序列会被合入,
|
||||
// 因此标签过滤会选中与原始点查询相同的数据(每个不同标签组合都是自己的
|
||||
// rollup 序列)。没有标签过滤时,同一桶内的所有标签序列会合并在一起,
|
||||
// 与原始路径的跨标签聚合一致。
|
||||
//
|
||||
// 除 AggRate 外,每种聚合都可用;AggRate 需要有序原始序列,因此只能基于原始点。
|
||||
// 百分位(p50、p95、p99 和任意 pXX)通过合并每桶 t-digest 回答,因此能在
|
||||
// 降采样后以有界误差保留下来。
|
||||
func (s *Store) AggregateRollup(ctx context.Context, query AggregateQuery, resolution time.Duration) ([]AggregatePoint, error) {
|
||||
if err := s.ensureOpen(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := query.Validate(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if resolution <= 0 {
|
||||
return nil, fmt.Errorf("%w: rollup resolution must be positive", ErrInvalidArgument)
|
||||
}
|
||||
if query.Interval < resolution || query.Interval%resolution != 0 {
|
||||
return nil, fmt.Errorf("%w: query interval must be a positive multiple of the rollup resolution", ErrInvalidArgument)
|
||||
}
|
||||
if query.Aggregation == AggRate {
|
||||
return nil, fmt.Errorf("%w: rate is not derivable from rollups (raw only)", ErrInvalidArgument)
|
||||
}
|
||||
|
||||
q := query.Query.normalized()
|
||||
comp := s.cfg.RollupPolicy.compression()
|
||||
|
||||
// Read rollup buckets at this resolution that are FULLY contained in the
|
||||
// inclusive window [Start, End] (entity and tag filters pushed into SQL),
|
||||
// then fold them into query.Interval-wide output buckets. Full containment
|
||||
// (rather than mere overlap) is what keeps a partially-overlapping bucket's
|
||||
// out-of-window samples from leaking into the result: a rollup bucket is an
|
||||
// indivisible summary, so a bucket straddling a window edge cannot be
|
||||
// trimmed to the window and is therefore excluded. Callers that need every
|
||||
// sample in a sub-bucket window must align the window to resolution
|
||||
// boundaries (or query raw points).
|
||||
rows, err := s.scanRollupRowsContained(ctx, q.MetricName, q.EntityID, q.Tags, resolution, q.Start, q.End)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
groups := foldRollupRows(nil, rows, query.Interval, comp)
|
||||
|
||||
out, err := rollupGroupsToPoints(groups, query)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return pageBuckets(out, query.BucketLimit, query.BucketOffset), nil
|
||||
}
|
||||
|
||||
// rollupGroupsToPoints turns the merged output buckets into ordered
|
||||
// AggregatePoints, computing the requested aggregation from each bucket's
|
||||
// summaries/digest. FillEmpty emits zero-count buckets for gaps, mirroring the
|
||||
// raw AggregatePoints behavior.
|
||||
//
|
||||
// rollupGroupsToPoints 将合并后的输出桶转换为有序 AggregatePoint,并根据
|
||||
// 每个桶的摘要或 digest 计算请求的聚合。FillEmpty 会为空洞输出零计数桶,
|
||||
// 与原始 AggregatePoints 行为一致。
|
||||
func rollupGroupsToPoints(groups map[int64]*rollupBucket, query AggregateQuery) ([]AggregatePoint, error) {
|
||||
var bucketStarts []int64
|
||||
if query.FillEmpty {
|
||||
for t := alignTime(query.Start, query.Interval); !t.After(query.End); t = t.Add(query.Interval) {
|
||||
bucketStarts = append(bucketStarts, t.UnixNano())
|
||||
}
|
||||
} else {
|
||||
for k := range groups {
|
||||
bucketStarts = append(bucketStarts, k)
|
||||
}
|
||||
sort.Slice(bucketStarts, func(i, j int) bool { return bucketStarts[i] < bucketStarts[j] })
|
||||
}
|
||||
|
||||
out := make([]AggregatePoint, 0, len(bucketStarts))
|
||||
for _, start := range bucketStarts {
|
||||
b := groups[start]
|
||||
if b == nil {
|
||||
out = append(out, AggregatePoint{
|
||||
MetricName: query.MetricName,
|
||||
EntityID: query.EntityID,
|
||||
Bucket: time.Unix(0, start).UTC(),
|
||||
})
|
||||
continue
|
||||
}
|
||||
v, ok := b.value(query.Aggregation)
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("%w: aggregation %q not supported over rollups", ErrInvalidArgument, query.Aggregation)
|
||||
}
|
||||
out = append(out, AggregatePoint{
|
||||
MetricName: query.MetricName,
|
||||
EntityID: query.EntityID,
|
||||
Bucket: time.Unix(0, start).UTC(),
|
||||
Value: v,
|
||||
Count: int(b.count),
|
||||
})
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// scanRollupRowsContained loads rollup rows for one resolution whose whole
|
||||
// bucket window [bucket, bucket+resolution) lies inside the inclusive query
|
||||
// window [start, end], with optional entity and tag filters pushed into SQL.
|
||||
//
|
||||
// Full containment (not mere overlap) is the boundary rule that keeps a rollup
|
||||
// query from over-counting: a rollup bucket is an indivisible summary, so a
|
||||
// bucket that only partially overlaps the window would drag its out-of-window
|
||||
// samples in if it were included. The end boundary is inclusive to match the
|
||||
// raw query semantics (raw uses ts <= end); a bucket whose last covered nano is
|
||||
// exactly end is therefore still contained. A degenerate zero-width window
|
||||
// (start == end) contains no whole bucket and yields an empty result — callers
|
||||
// that need sub-bucket precision must query raw points.
|
||||
//
|
||||
// scanRollupRowsContained 读取某分辨率下整桶窗口 [bucket, bucket+resolution)
|
||||
// 完整落在闭区间 [start, end] 内的 rollup 行,并可把实体和标签过滤下推到 SQL。
|
||||
//
|
||||
// 采用“完整包含”(而非仅重叠)作为边界规则,避免 rollup 查询过度计数:rollup
|
||||
// 桶是不可分割的摘要,只与窗口部分重叠的桶若被纳入,会把窗口外的样本一起带入。
|
||||
// end 边界为闭区间,以匹配原始查询语义(raw 使用 ts <= end);因此最后覆盖纳秒
|
||||
// 恰为 end 的桶仍算被包含。零宽窗口(start == end)不包含任何完整桶,返回空结果,
|
||||
// 需要亚桶精度的调用方应查询原始点。
|
||||
func (s *Store) scanRollupRowsContained(ctx context.Context, metricName, entityID string, tags map[string]string, resolution time.Duration, start, end time.Time) ([]storedRollup, error) {
|
||||
resNano := resolution.Nanoseconds()
|
||||
startNano := start.UTC().UnixNano()
|
||||
endNano := end.UTC().UnixNano()
|
||||
// A fully-contained bucket has start >= startNano and end (inclusive,
|
||||
// bucket+resNano-1) <= endNano, i.e. bucket in [startNano, endNano-resNano+1].
|
||||
// Push that exact closed range into SQL; no post-filter is then required.
|
||||
lower := startNano
|
||||
upper := endNano - resNano + 1 // inclusive upper bound for bucket_nano
|
||||
if upper < lower {
|
||||
return nil, nil // window narrower than one bucket: nothing is contained
|
||||
}
|
||||
rows, err := s.scanRollupRowsBetween(ctx, metricName, entityID, tags, resNano, lower, upper)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return rows, nil
|
||||
}
|
||||
|
||||
// scanRollupRowsBetween loads rollup rows for one resolution whose bucket start
|
||||
// falls in the inclusive nano range [lowerBucket, upperBucket], with optional
|
||||
// entity and tag filters pushed into SQL. It is the shared SQL primitive behind
|
||||
// the containment and hybrid scans; the bucket-window semantics are imposed by
|
||||
// the caller through the bounds it passes.
|
||||
//
|
||||
// scanRollupRowsBetween 读取某分辨率下桶起点落在闭区间
|
||||
// [lowerBucket, upperBucket] 内的 rollup 行,并可把实体和标签过滤下推到 SQL。
|
||||
// 它是包含扫描和混合扫描共用的 SQL 原语;桶窗口语义由调用方通过传入的边界决定。
|
||||
func (s *Store) scanRollupRowsBetween(ctx context.Context, metricName, entityID string, tags map[string]string, resNano, lowerBucket, upperBucket int64) ([]storedRollup, error) {
|
||||
args := []any{metricName, resNano, lowerBucket, upperBucket}
|
||||
parts := []string{
|
||||
"metric_name = " + s.dialect.placeholder(1),
|
||||
"resolution_nano = " + s.dialect.placeholder(2),
|
||||
"bucket_nano >= " + s.dialect.placeholder(3),
|
||||
"bucket_nano <= " + s.dialect.placeholder(4),
|
||||
}
|
||||
if strings.TrimSpace(entityID) != "" {
|
||||
args = append(args, entityID)
|
||||
parts = append(parts, "entity_id = "+s.dialect.placeholder(len(args)))
|
||||
}
|
||||
for _, k := range sortedKeys(tags) {
|
||||
args = append(args, tags[k])
|
||||
parts = append(parts, s.dialect.jsonExtractEquals("tags", k, s.dialect.placeholder(len(args))))
|
||||
}
|
||||
sqlText := fmt.Sprintf(
|
||||
`SELECT entity_id, tags_hash, tags, bucket_nano, count, sum, sum_sq, min_val, max_val, first_val, first_ts, last_val, last_ts, digest
|
||||
FROM %s WHERE %s ORDER BY bucket_nano ASC`,
|
||||
s.tables.rollups, strings.Join(parts, " AND "),
|
||||
)
|
||||
rows, err := s.reader().QueryContext(ctx, sqlText, args...)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
return scanStoredRollups(rows)
|
||||
}
|
||||
|
||||
// foldRollupRows folds stored rollup rows into interval-wide output buckets,
|
||||
// merging summaries (including the t-digest) per bucket. groups may be nil, in
|
||||
// which case a fresh map is allocated; passing an existing map lets a caller
|
||||
// accumulate rollup and raw contributions into the same output buckets.
|
||||
//
|
||||
// foldRollupRows 将存储的 rollup 行折叠进 interval 宽的输出桶,按桶合并摘要
|
||||
// (包括 t-digest)。groups 可为 nil,此时会分配新 map;传入已有 map 可让调用方
|
||||
// 把 rollup 和 raw 的贡献累加到同一批输出桶中。
|
||||
func foldRollupRows(groups map[int64]*rollupBucket, rows []storedRollup, interval time.Duration, comp float64) map[int64]*rollupBucket {
|
||||
if groups == nil {
|
||||
groups = make(map[int64]*rollupBucket)
|
||||
}
|
||||
size := interval.Nanoseconds()
|
||||
for _, r := range rows {
|
||||
bkt := floorDivNano(r.bucket, size)
|
||||
b := groups[bkt]
|
||||
if b == nil {
|
||||
b = newRollupBucket(comp)
|
||||
groups[bkt] = b
|
||||
}
|
||||
b.mergeStored(r.bucketData)
|
||||
}
|
||||
return groups
|
||||
}
|
||||
|
||||
// foldRawPoints folds raw points into interval-wide output buckets, adding each
|
||||
// observation into the matching bucket's accumulator. It shares the bucket map
|
||||
// with foldRollupRows so a hybrid query can combine an old rollup half and a
|
||||
// recent raw half into the same output buckets and aggregate them together
|
||||
// (correct count/avg/percentile across the boundary).
|
||||
//
|
||||
// foldRawPoints 将原始点折叠进 interval 宽的输出桶,把每个观测值加入对应桶的
|
||||
// 累加器。它与 foldRollupRows 共用桶 map,因此混合查询可以把旧 rollup 半边和
|
||||
// 近期 raw 半边合并到同一批输出桶中并一起聚合(跨边界的 count/avg/百分位正确)。
|
||||
func foldRawPoints(groups map[int64]*rollupBucket, points []Point, interval time.Duration, comp float64) map[int64]*rollupBucket {
|
||||
if groups == nil {
|
||||
groups = make(map[int64]*rollupBucket)
|
||||
}
|
||||
size := interval.Nanoseconds()
|
||||
for _, p := range points {
|
||||
ts := p.Timestamp.UTC().UnixNano()
|
||||
bkt := floorDivNano(ts, size)
|
||||
b := groups[bkt]
|
||||
if b == nil {
|
||||
b = newRollupBucket(comp)
|
||||
groups[bkt] = b
|
||||
}
|
||||
b.addPoint(p.Value, ts)
|
||||
}
|
||||
return groups
|
||||
}
|
||||
|
||||
// scanStoredRollups reconstructs storedRollup rows (including tag identity and
|
||||
// the decoded t-digest) from a result set whose columns are, in order:
|
||||
// entity_id, tags_hash, tags, bucket_nano, count, sum, sum_sq, min_val, max_val,
|
||||
// first_val, first_ts, last_val, last_ts, digest.
|
||||
//
|
||||
// scanStoredRollups 从结果集中还原 storedRollup 行(包括标签身份和解码后的
|
||||
// t-digest)。结果集的列顺序必须是:entity_id、tags_hash、tags、bucket_nano、
|
||||
// count、sum、sum_sq、min_val、max_val、first_val、first_ts、last_val、
|
||||
// last_ts、digest。
|
||||
func scanStoredRollups(rows *sql.Rows) ([]storedRollup, error) {
|
||||
var out []storedRollup
|
||||
for rows.Next() {
|
||||
var (
|
||||
eid string
|
||||
tagsHash string
|
||||
rawTags any
|
||||
bucketNano int64
|
||||
count int64
|
||||
sum, sumSq, minV, maxV, firstV, lastV float64
|
||||
firstTS, lastTS int64
|
||||
digestBlob []byte
|
||||
)
|
||||
if err := rows.Scan(&eid, &tagsHash, &rawTags, &bucketNano, &count, &sum, &sumSq, &minV, &maxV, &firstV, &firstTS, &lastV, &lastTS, &digestBlob); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
td, err := DecodeTDigest(digestBlob)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
tagsJSON, err := rawTagsToJSON(rawTags)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out = append(out, storedRollup{
|
||||
entityID: eid,
|
||||
bucket: bucketNano,
|
||||
bucketData: &rollupBucket{
|
||||
count: count, sum: sum, sumSq: sumSq,
|
||||
min: minV, max: maxV,
|
||||
firstVal: firstV, firstTS: firstTS,
|
||||
lastVal: lastV, lastTS: lastTS,
|
||||
digest: td,
|
||||
tagsHash: tagsHash,
|
||||
tagsJSON: tagsJSON,
|
||||
},
|
||||
})
|
||||
}
|
||||
return out, rows.Err()
|
||||
}
|
||||
|
||||
// rawTagsToJSON normalizes a scanned tags column (string or []byte) into the
|
||||
// canonical JSON string used when the bucket is re-written by a coarser tier.
|
||||
//
|
||||
// rawTagsToJSON 将扫描出的 tags 列(string 或 []byte)规范化为 JSON 字符串,
|
||||
// 供更粗层级重写桶时复用。
|
||||
func rawTagsToJSON(v any) (string, error) {
|
||||
switch x := v.(type) {
|
||||
case nil:
|
||||
return "{}", nil
|
||||
case string:
|
||||
if x == "" {
|
||||
return "{}", nil
|
||||
}
|
||||
return x, nil
|
||||
case []byte:
|
||||
if len(x) == 0 {
|
||||
return "{}", nil
|
||||
}
|
||||
return string(x), nil
|
||||
default:
|
||||
return "", fmt.Errorf("unsupported tags column type %T", v)
|
||||
}
|
||||
}
|
||||
|
||||
// Series answers an AggregateQuery by transparently choosing the best data
|
||||
// source for the requested window, given `now`:
|
||||
//
|
||||
// - If rollups are disabled, or the whole window still lies within raw
|
||||
// retention, it reads raw points (Aggregate) for full fidelity.
|
||||
// - Otherwise it picks the FINEST rollup tier that both (a) has an Interval
|
||||
// dividing query.Interval and (b) whose retention reaches back to the start
|
||||
// of the window, and serves the query from that tier.
|
||||
// - If the query spans the raw retention boundary, it uses a hybrid approach:
|
||||
// read rollups for the old part and raw for the recent part, then merge
|
||||
// the results to avoid losing uncompacted recent data.
|
||||
// - If no tier qualifies, it falls back to raw (which may be incomplete for
|
||||
// data already aged out) so the call still returns its best answer.
|
||||
//
|
||||
// query.Tags is honored on both branches: the raw path already filters by tag,
|
||||
// and the rollup path filters by the stored tag set, so a tag filter selects the
|
||||
// same series regardless of which source answers. This is the "downsampling
|
||||
// TSDB" read path: recent ranges answer from raw at full resolution, older
|
||||
// ranges answer from progressively coarser rollups.
|
||||
//
|
||||
// Series 会在给定 `now` 的情况下,通过透明选择最佳数据源来回答 AggregateQuery:
|
||||
//
|
||||
// - 如果 rollup 已禁用,或整个窗口仍在原始保留期内,它会读取原始点
|
||||
// (Aggregate)以获得完整保真度。
|
||||
// - 否则它会选择最细的 rollup 层,该层必须同时满足 (a) Interval 能整除
|
||||
// query.Interval,且 (b) 保留时间能覆盖到窗口起点,然后从该层服务查询。
|
||||
// - 如果查询跨越原始数据保留期边界,它使用混合方式:读取旧部分的 rollup
|
||||
// 和最近部分的原始点,然后合并结果,避免丢掉未 compact 的最近数据。
|
||||
// - 如果没有层级符合条件,它会回退到原始点(对于已经过期的数据可能不完整),
|
||||
// 让调用仍返回当前能给出的最佳答案。
|
||||
//
|
||||
// query.Tags 在两条分支上都会被遵守:原始路径已按标签过滤,rollup 路径会按
|
||||
// 存储的标签集合过滤,因此无论由哪个数据源回答,标签过滤都会选择相同序列。
|
||||
// 这是“降采样 TSDB”的读取路径:近期范围以完整分辨率从原始点回答,旧范围从
|
||||
// 逐级更粗的 rollup 回答。
|
||||
func (s *Store) Series(ctx context.Context, query AggregateQuery, now time.Time) ([]AggregatePoint, error) {
|
||||
if err := s.ensureOpen(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := query.Validate(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
policy := s.cfg.RollupPolicy
|
||||
if !policy.Enabled() {
|
||||
return s.Aggregate(ctx, query)
|
||||
}
|
||||
q := query.Query.normalized()
|
||||
now = now.UTC()
|
||||
|
||||
rawCutoff := policy.rawCutoff(now)
|
||||
|
||||
// Whole window inside raw retention (or raw kept forever) -> raw.
|
||||
if rawCutoff.IsZero() || !q.Start.Before(rawCutoff) {
|
||||
return s.Aggregate(ctx, query)
|
||||
}
|
||||
// Rate is raw-only; the caller asked for something rollups can't provide, so
|
||||
// answer from raw regardless of age.
|
||||
if query.Aggregation == AggRate {
|
||||
return s.Aggregate(ctx, query)
|
||||
}
|
||||
|
||||
if q.Start.Before(rawCutoff) && !q.End.Before(rawCutoff) {
|
||||
for _, tier := range policy.Tiers { // finest-first
|
||||
if query.Interval < tier.Interval || query.Interval%tier.Interval != 0 {
|
||||
continue
|
||||
}
|
||||
if now.Add(-tier.Retention).After(q.Start) {
|
||||
continue
|
||||
}
|
||||
return s.seriesHybrid(ctx, query, rawCutoff, &tier)
|
||||
}
|
||||
}
|
||||
|
||||
// Find the finest tier that can serve the entire window from the start.
|
||||
var servicingTier *RollupTier
|
||||
for _, tier := range policy.Tiers { // finest-first
|
||||
if query.Interval < tier.Interval || query.Interval%tier.Interval != 0 {
|
||||
continue
|
||||
}
|
||||
if now.Add(-tier.Retention).After(q.Start) {
|
||||
continue // tier doesn't reach back to the window start
|
||||
}
|
||||
servicingTier = &tier
|
||||
break
|
||||
}
|
||||
|
||||
if servicingTier == nil {
|
||||
return s.Aggregate(ctx, query)
|
||||
}
|
||||
|
||||
return s.AggregateRollup(ctx, query, servicingTier.Interval)
|
||||
}
|
||||
|
||||
// seriesHybrid answers a query that spans the raw-retention boundary by reading
|
||||
// rollups for the old part and raw for the recent part and folding BOTH into the
|
||||
// same query.Interval-wide output buckets before reducing them.
|
||||
//
|
||||
// The old lossy approach reduced each half to AggregatePoints independently and
|
||||
// then deduplicated by bucket time, letting raw fully replace a rollup point in
|
||||
// any output bucket they shared. That dropped the rollup half of a bucket that
|
||||
// straddles the boundary: e.g. with a 1h output bucket and the boundary mid-hour,
|
||||
// an 11:05 rollup sample and an 11:35 raw sample both belong to the 11:00 bucket,
|
||||
// yet the result kept only the raw sample (count/avg wrong). Folding both halves
|
||||
// into one rollupBucket per output bucket fixes that: count, sum/avg, min/max and
|
||||
// percentiles are computed over the union of the two halves.
|
||||
//
|
||||
// To avoid double counting or gaps, the caller passes a boundary already aligned
|
||||
// to a rollup-resolution bucket. The rollup half supplies whole buckets strictly
|
||||
// before the boundary and the raw half supplies points at or after it.
|
||||
//
|
||||
// seriesHybrid 回答跨越原始保留期边界的查询:读取旧部分的 rollup 和近期部分的
|
||||
// 原始点,并在归约前把两者折叠进同一批 query.Interval 宽的输出桶。
|
||||
//
|
||||
// 旧的有损做法会把两半各自归约成 AggregatePoint 再按桶时间去重,让 raw 在二者
|
||||
// 共有的输出桶里完全覆盖 rollup 点,从而丢掉跨边界桶的 rollup 半边(例如 1h 输出
|
||||
// 桶且边界落在小时中间时,11:05 的 rollup 样本和 11:35 的 raw 样本都属于 11:00 桶,
|
||||
// 结果却只剩 raw 样本,count/avg 错误)。把两半折叠进同一个 rollupBucket 即可修复:
|
||||
// count、sum/avg、min/max 和百分位都在两半的并集上计算。
|
||||
//
|
||||
// 为避免重复计数或留空,调用方传入的边界已经对齐到 rollup 分辨率桶。rollup 半边
|
||||
// 提供严格早于该边界的完整桶,raw 半边提供该边界及之后的点。
|
||||
func (s *Store) seriesHybrid(ctx context.Context, query AggregateQuery, boundary time.Time, tier *RollupTier) ([]AggregatePoint, error) {
|
||||
q := query.Query.normalized()
|
||||
comp := s.cfg.RollupPolicy.compression()
|
||||
resNano := tier.Interval.Nanoseconds()
|
||||
startNano := q.Start.UnixNano()
|
||||
endNano := q.End.UnixNano()
|
||||
splitAt := boundary.UTC().UnixNano()
|
||||
|
||||
groups := make(map[int64]*rollupBucket)
|
||||
|
||||
// Old half: rollup buckets fully contained in [Start, min(splitAt, End]].
|
||||
// "Fully before splitAt" is bucket <= splitAt-resNano (splitAt is aligned);
|
||||
// also clamp to the query's own inclusive end so a bucket past End is never
|
||||
// pulled in when splitAt overshoots a short window.
|
||||
upperBucket := splitAt - resNano
|
||||
if clamp := endNano - resNano + 1; clamp < upperBucket {
|
||||
upperBucket = clamp
|
||||
}
|
||||
if upperBucket >= startNano {
|
||||
rows, err := s.scanRollupRowsBetween(ctx, q.MetricName, q.EntityID, q.Tags, resNano, startNano, upperBucket)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
foldRollupRows(groups, rows, query.Interval, comp)
|
||||
}
|
||||
|
||||
// Recent half: raw points in [splitAt, End]. Reuse the raw Query path so the
|
||||
// entity and tag filters match the rollup half exactly, then fold the points
|
||||
// into the SAME output buckets so a bucket straddling splitAt aggregates both
|
||||
// halves. Skip when splitAt is past the window end (no recent raw to add).
|
||||
if splitAt <= endNano {
|
||||
rawQuery := q
|
||||
rawQuery.Start = time.Unix(0, splitAt).UTC()
|
||||
rawQuery.Limit = 0
|
||||
rawQuery.Offset = 0
|
||||
points, err := s.Query(ctx, rawQuery)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
foldRawPoints(groups, points, query.Interval, comp)
|
||||
}
|
||||
|
||||
out, err := rollupGroupsToPoints(groups, query)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return pageBuckets(out, query.BucketLimit, query.BucketOffset), nil
|
||||
}
|
||||
@@ -0,0 +1,740 @@
|
||||
package metric
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Compact runs the store's RollupPolicy over every metric: it rebuilds the
|
||||
// rollup tiers up to `now` and enforces every retention window (raw points and
|
||||
// each tier). It is the explicit maintenance entry point — call it from a cron
|
||||
// job or scheduler. It is safe to call repeatedly and is idempotent for
|
||||
// unchanged windows.
|
||||
//
|
||||
// Returns the number of rollup buckets written across all tiers and metrics.
|
||||
//
|
||||
// Compact 会对所有指标执行 Store 的 RollupPolicy:它会重建截至 `now` 的
|
||||
// rollup 层,并执行每个保留窗口(原始点和各层级)。这是显式维护入口,
|
||||
// 应从 cron 任务或调度器调用。它可以重复安全调用,未变化窗口保持幂等。
|
||||
//
|
||||
// 返回所有指标、所有层级中写入的 rollup 桶数量。
|
||||
func (s *Store) Compact(ctx context.Context, now time.Time) (int, error) {
|
||||
if err := s.ensureOpen(); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
if !s.cfg.RollupPolicy.Enabled() {
|
||||
return 0, nil
|
||||
}
|
||||
defs, err := s.ListMetrics(ctx)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
total := 0
|
||||
for _, def := range defs {
|
||||
n, err := s.CompactMetric(ctx, def.Name, now)
|
||||
if err != nil {
|
||||
return total, fmt.Errorf("compact metric %q: %w", def.Name, err)
|
||||
}
|
||||
total += n
|
||||
}
|
||||
return total, nil
|
||||
}
|
||||
|
||||
// CompactMetric compacts a single metric. It builds the finest tier from raw
|
||||
// points, then each coarser tier from the tier below it, upserts the resulting
|
||||
// buckets, and finally deletes data that has aged out of each retention window
|
||||
// (raw points first, then each tier).
|
||||
//
|
||||
// The entire compaction is performed within a SERIALIZABLE transaction to
|
||||
// guarantee that the raw scan and the raw deletion observe the same snapshot.
|
||||
// Without this, the default isolation level on PostgreSQL/MySQL lets a point
|
||||
// inserted between the scan and the delete be deleted by the cutoff DELETE
|
||||
// without ever being folded into a rollup: the scan never saw it (it ran on an
|
||||
// earlier read), yet the DELETE (a fresh read) does. SERIALIZABLE makes the
|
||||
// scan acquire predicate/range protection so a concurrent write that would
|
||||
// otherwise slip into that gap either is excluded from the delete or forces a
|
||||
// serialization failure that we retry on a fresh snapshot. SQLite serializes
|
||||
// writes on a single connection, so its default isolation already provides this
|
||||
// guarantee and needs no escalation.
|
||||
//
|
||||
// Rollups are keyed by tag set as well as entity, so each distinct tag
|
||||
// combination (e.g. a GPU device_index) is summarized into its own series and
|
||||
// can be queried independently after the raw points are gone.
|
||||
//
|
||||
// CompactMetric 会压缩单个指标。它先由原始点构建最细层,再由下一层之下的
|
||||
// 层级逐层合成更粗层,upsert 生成的桶,最后删除每个保留窗口中过期的数据
|
||||
// (先删除原始点,再删除各层级)。
|
||||
//
|
||||
// 整个 compaction 在一个 SERIALIZABLE 事务内执行,以保证 raw 扫描和 raw 删除
|
||||
// 看到同一个快照。否则在 PostgreSQL/MySQL 的默认隔离级别下,扫描之后、删除
|
||||
// 之前写入的旧时间点仍可能被 cutoff 删除却没有进入 rollup:扫描(较早的读)
|
||||
// 没看到它,而删除(一次新的读)看到了。SERIALIZABLE 让扫描获得谓词/范围
|
||||
// 保护,使这种并发写要么被删除排除在外,要么触发可重试的序列化失败,由我们
|
||||
// 在新快照上重试。SQLite 在单连接上串行化写入,其默认隔离已提供该保证,
|
||||
// 无需提升隔离级别。
|
||||
func (s *Store) CompactMetric(ctx context.Context, metricName string, now time.Time) (int, error) {
|
||||
if err := s.ensureOpen(); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
policy := s.cfg.RollupPolicy
|
||||
if !policy.Enabled() {
|
||||
return 0, nil
|
||||
}
|
||||
now = now.UTC()
|
||||
|
||||
// Retry the whole compaction on a transient serialization/deadlock failure.
|
||||
// Under SERIALIZABLE, a concurrent writer touching the same raw range can
|
||||
// abort our transaction; re-running on a fresh snapshot is the correct and
|
||||
// expected recovery, and Compact is idempotent so a retry is safe.
|
||||
const maxAttempts = 5
|
||||
var lastErr error
|
||||
for attempt := 0; attempt < maxAttempts; attempt++ {
|
||||
written, err := s.compactMetricOnce(ctx, metricName, now)
|
||||
if err == nil {
|
||||
return written, nil
|
||||
}
|
||||
if !isRetryableSerializationError(err) {
|
||||
return written, err
|
||||
}
|
||||
lastErr = err
|
||||
}
|
||||
return 0, fmt.Errorf("compact metric %q: exhausted retries after serialization failures: %w", metricName, lastErr)
|
||||
}
|
||||
|
||||
// compactMetricOnce runs a single compaction attempt inside one transaction.
|
||||
//
|
||||
// compactMetricOnce 在单个事务内执行一次 compaction 尝试。
|
||||
func (s *Store) compactMetricOnce(ctx context.Context, metricName string, now time.Time) (int, error) {
|
||||
// Use a transaction to ensure consistency between raw scan, rollup write, and
|
||||
// raw deletion. The isolation level is backend-specific (SERIALIZABLE on
|
||||
// PostgreSQL/MySQL, default on SQLite) so late-arriving points cannot be
|
||||
// deleted without first being rolled up.
|
||||
tx, err := s.db.BeginTx(ctx, s.dialect.compactTxOptions())
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
defer func() { _ = tx.Rollback() }()
|
||||
|
||||
written, err := s.compactMetricWithinTx(ctx, metricName, now, tx)
|
||||
if err != nil {
|
||||
return written, err
|
||||
}
|
||||
|
||||
if err := tx.Commit(); err != nil {
|
||||
return written, err
|
||||
}
|
||||
return written, nil
|
||||
}
|
||||
|
||||
// isRetryableSerializationError reports whether err is a transient
|
||||
// serialization failure or deadlock that should be retried on a fresh
|
||||
// transaction. It matches on portable SQLSTATE codes and driver error text so
|
||||
// the package stays free of driver-specific error type imports:
|
||||
//
|
||||
// - PostgreSQL: 40001 serialization_failure, 40P01 deadlock_detected.
|
||||
// - MySQL: 1213 deadlock, 1205 lock wait timeout.
|
||||
// - SQLite: SQLITE_BUSY / database is locked (only relevant with >1 conn).
|
||||
//
|
||||
// isRetryableSerializationError 判断 err 是否为应在新事务上重试的瞬时序列化
|
||||
// 失败或死锁。它通过可移植的 SQLSTATE 码和驱动错误文本匹配,使包不需要导入
|
||||
// 驱动专用错误类型。
|
||||
func isRetryableSerializationError(err error) bool {
|
||||
if err == nil {
|
||||
return false
|
||||
}
|
||||
msg := strings.ToLower(err.Error())
|
||||
switch {
|
||||
case strings.Contains(msg, "40001"): // postgres serialization_failure
|
||||
return true
|
||||
case strings.Contains(msg, "40p01"): // postgres deadlock_detected
|
||||
return true
|
||||
case strings.Contains(msg, "serialization failure"):
|
||||
return true
|
||||
case strings.Contains(msg, "could not serialize"):
|
||||
return true
|
||||
case strings.Contains(msg, "deadlock"): // mysql 1213, postgres text
|
||||
return true
|
||||
case strings.Contains(msg, "lock wait timeout"): // mysql 1205
|
||||
return true
|
||||
case strings.Contains(msg, "database is locked"): // sqlite busy
|
||||
return true
|
||||
case strings.Contains(msg, "database table is locked"): // sqlite busy
|
||||
return true
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
// compactMetricWithinTx is the actual compaction logic, meant to be called
|
||||
// within a transaction. The tx parameter is passed explicitly to all operations
|
||||
// that need transactional consistency.
|
||||
func (s *Store) compactMetricWithinTx(ctx context.Context, metricName string, now time.Time, tx *sql.Tx) (int, error) {
|
||||
policy := s.cfg.RollupPolicy
|
||||
if !policy.Enabled() {
|
||||
return 0, nil
|
||||
}
|
||||
comp := policy.compression()
|
||||
|
||||
written := 0
|
||||
var prevInterval time.Duration
|
||||
var prevDelta map[rollupKey]*rollupBucket
|
||||
rawCutoff := policy.rawCutoff(now)
|
||||
|
||||
for tierIdx, tier := range policy.Tiers {
|
||||
var buckets map[rollupKey]*rollupBucket
|
||||
var currentDelta map[rollupKey]*rollupBucket
|
||||
var err error
|
||||
// Reads must run on the transaction itself, not s.reader(): with a
|
||||
// single-connection pool (SQLite MaxOpenConns=1) the tx already holds
|
||||
// the only connection, so a pool read here would deadlock.
|
||||
if tierIdx == 0 {
|
||||
buckets, err = s.buildFinestTier(ctx, tx, metricName, tier.Interval, comp)
|
||||
if err == nil && !rawCutoff.IsZero() {
|
||||
cutoffNano := rawCutoff.UnixNano()
|
||||
for k := range buckets {
|
||||
if k.bucket >= cutoffNano {
|
||||
delete(buckets, k)
|
||||
}
|
||||
}
|
||||
}
|
||||
if !rawCutoff.IsZero() {
|
||||
currentDelta = buckets
|
||||
}
|
||||
} else {
|
||||
buckets, err = s.buildCoarserTier(ctx, tx, metricName, prevInterval, tier.Interval, comp)
|
||||
if prevDelta != nil {
|
||||
currentDelta = buildCoarserBucketsFromDelta(prevDelta, tier.Interval, comp)
|
||||
}
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
return written, err
|
||||
}
|
||||
// For the finest tier only, merge buckets older than the raw cutoff:
|
||||
// those buckets may already have a retained rollup while only late raw
|
||||
// samples remain in the raw table.
|
||||
var n int
|
||||
if tierIdx == 0 && !rawCutoff.IsZero() {
|
||||
n, err = s.writeRollupBucketsWithMergePointTx(ctx, metricName, tier.Interval, buckets, rawCutoff, time.Time{}, nil, tx)
|
||||
} else if tierIdx > 0 {
|
||||
deltaCutoff := alignRollupRetentionCutoff(now.Add(-policy.Tiers[tierIdx-1].Retention), tier.Interval)
|
||||
n, err = s.writeRollupBucketsWithMergePointTx(ctx, metricName, tier.Interval, buckets, time.Time{}, deltaCutoff, currentDelta, tx)
|
||||
} else {
|
||||
n, err = s.writeRollupBucketsTx(ctx, metricName, tier.Interval, buckets, tx)
|
||||
}
|
||||
if err != nil {
|
||||
return written, err
|
||||
}
|
||||
written += n
|
||||
prevInterval = tier.Interval
|
||||
prevDelta = currentDelta
|
||||
}
|
||||
|
||||
// Enforce retention windows after all tiers are materialized, so a coarser
|
||||
// tier is always built before the finer source it depends on is trimmed.
|
||||
if policy.RawRetention > 0 {
|
||||
cutoff := policy.rawCutoff(now)
|
||||
if _, err := s.DeleteBeforeTx(ctx, metricName, cutoff, tx); err != nil {
|
||||
return written, err
|
||||
}
|
||||
}
|
||||
for i, tier := range policy.Tiers {
|
||||
cutoff := now.Add(-tier.Retention)
|
||||
if i+1 < len(policy.Tiers) {
|
||||
cutoff = alignRollupRetentionCutoff(cutoff, policy.Tiers[i+1].Interval)
|
||||
}
|
||||
if err := s.deleteRollupsBeforeTx(ctx, metricName, tier.Interval, cutoff, tx); err != nil {
|
||||
return written, err
|
||||
}
|
||||
}
|
||||
return written, nil
|
||||
}
|
||||
|
||||
func alignRollupRetentionCutoff(cutoff time.Time, nextInterval time.Duration) time.Time {
|
||||
if nextInterval <= 0 {
|
||||
return cutoff.UTC()
|
||||
}
|
||||
return time.Unix(0, floorDivNano(cutoff.UTC().UnixNano(), nextInterval.Nanoseconds())).UTC()
|
||||
}
|
||||
|
||||
// rollupKey identifies one rollup cell. The tag dimension (tagsHash) is part of
|
||||
// the key so points carrying different tags never collapse into the same bucket.
|
||||
//
|
||||
// rollupKey 标识一个 rollup 单元;tagsHash 是 key 的一部分,确保不同标签
|
||||
// 的点不会落入同一个桶。
|
||||
type rollupKey struct {
|
||||
// entityID is the entity dimension of the rollup cell.
|
||||
//
|
||||
// entityID 是 rollup 单元的实体维度。
|
||||
entityID string
|
||||
// tagsHash is the stable fingerprint of the tag set.
|
||||
//
|
||||
// tagsHash 是标签集合的稳定指纹。
|
||||
tagsHash string
|
||||
// bucket is the bucket start timestamp in nanoseconds.
|
||||
//
|
||||
// bucket 是桶起始时间的纳秒时间戳。
|
||||
bucket int64
|
||||
}
|
||||
|
||||
// buildFinestTier scans raw points for the metric and groups them into buckets
|
||||
// of the given interval, keyed by (entity, tag set, bucket-start). Each point's
|
||||
// tag map determines which series it belongs to.
|
||||
//
|
||||
// buildFinestTier 扫描某指标的原始点,并按给定 interval 分桶,key 为
|
||||
// (实体、标签集合、桶起点)。每个点的标签 map 决定它属于哪条序列。
|
||||
func (s *Store) buildFinestTier(ctx context.Context, q querier, metricName string, interval time.Duration, comp float64) (map[rollupKey]*rollupBucket, error) {
|
||||
size := interval.Nanoseconds()
|
||||
out := make(map[rollupKey]*rollupBucket)
|
||||
sqlText := fmt.Sprintf(
|
||||
`SELECT entity_id, ts_nano, value, tags FROM %s WHERE metric_name = %s ORDER BY ts_nano ASC`,
|
||||
s.tables.points, s.dialect.placeholder(1),
|
||||
)
|
||||
rows, err := q.QueryContext(ctx, sqlText, metricName)
|
||||
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
for rows.Next() {
|
||||
var entityID string
|
||||
var ts int64
|
||||
var value float64
|
||||
var rawTags any
|
||||
if err := rows.Scan(&entityID, &ts, &value, &rawTags); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
tags, err := decodeMap(rawTags)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
hash, canonical, err := tagsFingerprint(tags)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
bucket := floorDivNano(ts, size)
|
||||
k := rollupKey{entityID: entityID, tagsHash: hash, bucket: bucket}
|
||||
b := out[k]
|
||||
if b == nil {
|
||||
b = newRollupBucket(comp)
|
||||
b.tagsHash = hash
|
||||
b.tagsJSON = canonical
|
||||
out[k] = b
|
||||
}
|
||||
b.addPoint(value, ts)
|
||||
}
|
||||
return out, rows.Err()
|
||||
}
|
||||
|
||||
// buildCoarserTier composes a coarser tier from the already-stored finer tier:
|
||||
// it reads the finer rollup rows and merges every finer bucket into the coarse
|
||||
// bucket that shares its (entity, tag set). Tag identity is preserved end to
|
||||
// end, so a coarse series only ever merges finer buckets of the same tag set.
|
||||
//
|
||||
// buildCoarserTier 基于已存储的细层 rollup 合成更粗层:它读取细层 rollup 行,
|
||||
// 并把每个细桶合并进共享相同(实体、标签集合)的粗桶。标签身份会端到端保留,
|
||||
// 因此一条粗粒度序列只会合并相同标签集合的细桶。
|
||||
func (s *Store) buildCoarserTier(ctx context.Context, q querier, metricName string, fineInterval, coarseInterval time.Duration, comp float64) (map[rollupKey]*rollupBucket, error) {
|
||||
coarseSize := coarseInterval.Nanoseconds()
|
||||
out := make(map[rollupKey]*rollupBucket)
|
||||
fineRows, err := s.scanRollupRows(ctx, q, metricName, fineInterval)
|
||||
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for _, fr := range fineRows {
|
||||
bucket := floorDivNano(fr.bucket, coarseSize)
|
||||
k := rollupKey{entityID: fr.entityID, tagsHash: fr.bucketData.tagsHash, bucket: bucket}
|
||||
b := out[k]
|
||||
if b == nil {
|
||||
b = newRollupBucket(comp)
|
||||
b.tagsHash = fr.bucketData.tagsHash
|
||||
b.tagsJSON = fr.bucketData.tagsJSON
|
||||
out[k] = b
|
||||
}
|
||||
b.mergeStored(fr.bucketData)
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func buildCoarserBucketsFromDelta(delta map[rollupKey]*rollupBucket, coarseInterval time.Duration, comp float64) map[rollupKey]*rollupBucket {
|
||||
out := make(map[rollupKey]*rollupBucket)
|
||||
if len(delta) == 0 {
|
||||
return out
|
||||
}
|
||||
coarseSize := coarseInterval.Nanoseconds()
|
||||
for k, src := range delta {
|
||||
bucket := floorDivNano(k.bucket, coarseSize)
|
||||
ck := rollupKey{entityID: k.entityID, tagsHash: k.tagsHash, bucket: bucket}
|
||||
b := out[ck]
|
||||
if b == nil {
|
||||
b = newRollupBucket(comp)
|
||||
b.tagsHash = src.tagsHash
|
||||
b.tagsJSON = src.tagsJSON
|
||||
out[ck] = b
|
||||
}
|
||||
b.mergeStored(src)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// storedRollup represents a rollup row reconstructed from storage.
|
||||
//
|
||||
// storedRollup 表示从存储中还原的一行 rollup 数据。
|
||||
type storedRollup struct {
|
||||
// entityID is the entity stored on the rollup row.
|
||||
//
|
||||
// entityID 是 rollup 行中保存的实体。
|
||||
entityID string
|
||||
// bucket is the stored bucket start timestamp in nanoseconds.
|
||||
//
|
||||
// bucket 是存储的桶起始纳秒时间戳。
|
||||
bucket int64
|
||||
// bucketData is the reconstructed in-memory accumulator for the row.
|
||||
//
|
||||
// bucketData 是该行还原出的内存累加器。
|
||||
bucketData *rollupBucket
|
||||
}
|
||||
|
||||
// scanRollupRows loads all rollup rows for a metric at a given resolution and
|
||||
// reconstructs their in-memory accumulators (including tag identity and the
|
||||
// decoded t-digest).
|
||||
//
|
||||
// scanRollupRows 读取某指标在给定分辨率下的所有 rollup 行,并还原它们的
|
||||
// 内存累加器(包括标签身份和解码后的 t-digest)。
|
||||
func (s *Store) scanRollupRows(ctx context.Context, q querier, metricName string, interval time.Duration) ([]storedRollup, error) {
|
||||
sqlText := fmt.Sprintf(
|
||||
`SELECT entity_id, tags_hash, tags, bucket_nano, count, sum, sum_sq, min_val, max_val, first_val, first_ts, last_val, last_ts, digest
|
||||
FROM %s WHERE metric_name = %s AND resolution_nano = %s ORDER BY bucket_nano ASC`,
|
||||
s.tables.rollups, s.dialect.placeholder(1), s.dialect.placeholder(2),
|
||||
)
|
||||
rows, err := q.QueryContext(ctx, sqlText, metricName, interval.Nanoseconds())
|
||||
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
return scanStoredRollups(rows)
|
||||
}
|
||||
|
||||
// writeRollupBuckets upserts a set of computed buckets for one resolution.
|
||||
//
|
||||
// writeRollupBuckets 将某个分辨率下计算出的 rollup 桶批量 upsert 到数据库。
|
||||
func (s *Store) writeRollupBuckets(ctx context.Context, metricName string, interval time.Duration, buckets map[rollupKey]*rollupBucket) (int, error) {
|
||||
return s.writeRollupBucketsWithMergePoint(ctx, metricName, interval, buckets, time.Time{})
|
||||
}
|
||||
|
||||
// writeRollupBucketsTx upserts buckets within an existing transaction.
|
||||
func (s *Store) writeRollupBucketsTx(ctx context.Context, metricName string, interval time.Duration, buckets map[rollupKey]*rollupBucket, tx *sql.Tx) (int, error) {
|
||||
return s.writeRollupBucketsWithMergePointTx(ctx, metricName, interval, buckets, time.Time{}, time.Time{}, nil, tx)
|
||||
}
|
||||
|
||||
// writeRollupBucketsWithMergePoint is the internal implementation that optionally
|
||||
// merges buckets older than a cutoff point instead of replacing them.
|
||||
func (s *Store) writeRollupBucketsWithMergePoint(ctx context.Context, metricName string, interval time.Duration, buckets map[rollupKey]*rollupBucket, mergeCutoff time.Time) (int, error) {
|
||||
if len(buckets) == 0 {
|
||||
return 0, nil
|
||||
}
|
||||
keys := make([]rollupKey, 0, len(buckets))
|
||||
for k := range buckets {
|
||||
keys = append(keys, k)
|
||||
}
|
||||
sort.Slice(keys, func(i, j int) bool {
|
||||
if keys[i].entityID != keys[j].entityID {
|
||||
return keys[i].entityID < keys[j].entityID
|
||||
}
|
||||
if keys[i].tagsHash != keys[j].tagsHash {
|
||||
return keys[i].tagsHash < keys[j].tagsHash
|
||||
}
|
||||
return keys[i].bucket < keys[j].bucket
|
||||
})
|
||||
|
||||
stmt := s.dialect.upsertRollupSQL(s.tables)
|
||||
resNano := interval.Nanoseconds()
|
||||
now := time.Now().UTC().UnixNano()
|
||||
mergeCutoffNano := mergeCutoff.UnixNano()
|
||||
|
||||
run := func(ex execer) error {
|
||||
for _, k := range keys {
|
||||
b := buckets[k]
|
||||
tagsJSON := b.tagsJSON
|
||||
if tagsJSON == "" {
|
||||
tagsJSON = "{}"
|
||||
}
|
||||
|
||||
// If mergeCutoff is set and this bucket is older than the cutoff,
|
||||
// merge with existing data instead of replacing it.
|
||||
if !mergeCutoff.IsZero() && k.bucket < mergeCutoffNano {
|
||||
// Read existing row if it exists
|
||||
existing, err := s.readRollupBucket(ctx, metricName, k.entityID, k.tagsHash, interval, k.bucket)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if existing != nil {
|
||||
// Merge the new bucket into the existing one
|
||||
existing.mergeStored(b)
|
||||
b = existing
|
||||
}
|
||||
}
|
||||
|
||||
// Column order must match rollupColumns in dialect_rollup.go:
|
||||
// metric_name, entity_id, tags_hash, tags, resolution_nano, bucket_nano,
|
||||
// count, sum, sum_sq, min_val, max_val, first_val, first_ts, last_val,
|
||||
// last_ts, digest, created_at.
|
||||
_, err := ex.ExecContext(ctx, stmt,
|
||||
metricName, k.entityID, k.tagsHash, tagsJSON, resNano, k.bucket,
|
||||
b.count, b.sum, b.sumSq, b.min, b.max,
|
||||
b.firstVal, b.firstTS, b.lastVal, b.lastTS,
|
||||
b.digest.Encode(), now,
|
||||
)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
if len(keys) == 1 {
|
||||
if err := run(s.db); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return len(keys), nil
|
||||
}
|
||||
tx, err := s.db.BeginTx(ctx, nil)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
defer func() { _ = tx.Rollback() }()
|
||||
if err := run(tx); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
if err := tx.Commit(); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return len(keys), nil
|
||||
}
|
||||
|
||||
// writeRollupBucketsWithMergePointTx is the transactional version that executes
|
||||
// all operations within an existing transaction. deltaCutoff marks coarser
|
||||
// buckets whose source tier may already have aged out; a smaller recomputed
|
||||
// bucket in that region is treated as late-arriving delta data and merged into
|
||||
// the retained coarse bucket instead of replacing it.
|
||||
func (s *Store) writeRollupBucketsWithMergePointTx(ctx context.Context, metricName string, interval time.Duration, buckets map[rollupKey]*rollupBucket, mergeCutoff, protectCutoff time.Time, deltaBuckets map[rollupKey]*rollupBucket, tx *sql.Tx) (int, error) {
|
||||
if len(buckets) == 0 {
|
||||
return 0, nil
|
||||
}
|
||||
keys := make([]rollupKey, 0, len(buckets))
|
||||
for k := range buckets {
|
||||
keys = append(keys, k)
|
||||
}
|
||||
sort.Slice(keys, func(i, j int) bool {
|
||||
if keys[i].entityID != keys[j].entityID {
|
||||
return keys[i].entityID < keys[j].entityID
|
||||
}
|
||||
if keys[i].tagsHash != keys[j].tagsHash {
|
||||
return keys[i].tagsHash < keys[j].tagsHash
|
||||
}
|
||||
return keys[i].bucket < keys[j].bucket
|
||||
})
|
||||
|
||||
stmt := s.dialect.upsertRollupSQL(s.tables)
|
||||
resNano := interval.Nanoseconds()
|
||||
now := time.Now().UTC().UnixNano()
|
||||
mergeCutoffNano := mergeCutoff.UnixNano()
|
||||
deltaCutoffNano := protectCutoff.UnixNano()
|
||||
written := 0
|
||||
|
||||
for _, k := range keys {
|
||||
b := buckets[k]
|
||||
tagsJSON := b.tagsJSON
|
||||
if tagsJSON == "" {
|
||||
tagsJSON = "{}"
|
||||
}
|
||||
|
||||
// If mergeCutoff is set and this bucket is older than the cutoff,
|
||||
// merge with existing data instead of replacing it.
|
||||
if !mergeCutoff.IsZero() && k.bucket < mergeCutoffNano {
|
||||
// Read existing row if it exists
|
||||
existing, err := s.readRollupBucketTx(ctx, metricName, k.entityID, k.tagsHash, interval, k.bucket, tx)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
if existing != nil {
|
||||
// Merge the new bucket into the existing one
|
||||
existing.mergeStored(b)
|
||||
b = existing
|
||||
}
|
||||
}
|
||||
if !protectCutoff.IsZero() && k.bucket < deltaCutoffNano {
|
||||
existing, err := s.readRollupBucketTx(ctx, metricName, k.entityID, k.tagsHash, interval, k.bucket, tx)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
if existing != nil {
|
||||
if deltaBuckets == nil {
|
||||
// Without delta tracking (raw is retained forever), the
|
||||
// source tier is rebuilt from full raw data and can replace.
|
||||
} else if delta := deltaBuckets[k]; delta != nil && delta.count > 0 {
|
||||
existing.mergeStored(delta)
|
||||
b = existing
|
||||
} else {
|
||||
continue
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Column order must match rollupColumns in dialect_rollup.go
|
||||
_, err := tx.ExecContext(ctx, stmt,
|
||||
metricName, k.entityID, k.tagsHash, tagsJSON, resNano, k.bucket,
|
||||
b.count, b.sum, b.sumSq, b.min, b.max,
|
||||
b.firstVal, b.firstTS, b.lastVal, b.lastTS,
|
||||
b.digest.Encode(), now,
|
||||
)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
written++
|
||||
}
|
||||
return written, nil
|
||||
}
|
||||
|
||||
// readRollupBucket reads a single rollup bucket from storage, returning nil if
|
||||
// the bucket doesn't exist. Used to merge late-arriving data with existing rollups.
|
||||
func (s *Store) readRollupBucket(ctx context.Context, metricName, entityID, tagsHash string, interval time.Duration, bucketNano int64) (*rollupBucket, error) {
|
||||
sqlText := fmt.Sprintf(
|
||||
`SELECT count, sum, sum_sq, min_val, max_val, first_val, first_ts, last_val, last_ts, digest, tags
|
||||
FROM %s WHERE metric_name = %s AND resolution_nano = %s AND entity_id = %s
|
||||
AND tags_hash = %s AND bucket_nano = %s`,
|
||||
s.tables.rollups,
|
||||
s.dialect.placeholder(1), s.dialect.placeholder(2), s.dialect.placeholder(3),
|
||||
s.dialect.placeholder(4), s.dialect.placeholder(5),
|
||||
)
|
||||
row := s.reader().QueryRowContext(ctx, sqlText, metricName, interval.Nanoseconds(), entityID, tagsHash, bucketNano)
|
||||
|
||||
var count int64
|
||||
var sum, sumSq, minV, maxV, firstV, lastV float64
|
||||
var firstTS, lastTS int64
|
||||
var digestBlob []byte
|
||||
var rawTags any
|
||||
|
||||
err := row.Scan(&count, &sum, &sumSq, &minV, &maxV, &firstV, &firstTS, &lastV, &lastTS, &digestBlob, &rawTags)
|
||||
if err == sql.ErrNoRows {
|
||||
return nil, nil
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
td, err := DecodeTDigest(digestBlob)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
tagsJSON, err := rawTagsToJSON(rawTags)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return &rollupBucket{
|
||||
count: count, sum: sum, sumSq: sumSq,
|
||||
min: minV, max: maxV,
|
||||
firstVal: firstV, firstTS: firstTS,
|
||||
lastVal: lastV, lastTS: lastTS,
|
||||
digest: td,
|
||||
tagsHash: tagsHash,
|
||||
tagsJSON: tagsJSON,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// deleteRollupsBefore deletes stored rollup rows older than a cutoff.
|
||||
func (s *Store) deleteRollupsBefore(ctx context.Context, metricName string, interval time.Duration, before time.Time) error {
|
||||
sqlText := fmt.Sprintf(
|
||||
`DELETE FROM %s WHERE metric_name = %s AND resolution_nano = %s AND bucket_nano < %s`,
|
||||
s.tables.rollups, s.dialect.placeholder(1), s.dialect.placeholder(2), s.dialect.placeholder(3),
|
||||
)
|
||||
_, err := s.db.ExecContext(ctx, sqlText, metricName, interval.Nanoseconds(), before.UTC().UnixNano())
|
||||
return err
|
||||
}
|
||||
|
||||
// deleteRollupsBeforeTx deletes stored rollup rows within a transaction.
|
||||
func (s *Store) deleteRollupsBeforeTx(ctx context.Context, metricName string, interval time.Duration, before time.Time, tx *sql.Tx) error {
|
||||
sqlText := fmt.Sprintf(
|
||||
`DELETE FROM %s WHERE metric_name = %s AND resolution_nano = %s AND bucket_nano < %s`,
|
||||
s.tables.rollups, s.dialect.placeholder(1), s.dialect.placeholder(2), s.dialect.placeholder(3),
|
||||
)
|
||||
_, err := tx.ExecContext(ctx, sqlText, metricName, interval.Nanoseconds(), before.UTC().UnixNano())
|
||||
return err
|
||||
}
|
||||
|
||||
// DeleteBeforeTx deletes raw points before a cutoff within a transaction.
|
||||
func (s *Store) DeleteBeforeTx(ctx context.Context, metricName string, before time.Time, tx *sql.Tx) (int64, error) {
|
||||
sqlText := fmt.Sprintf(
|
||||
`DELETE FROM %s WHERE metric_name = %s AND ts_nano < %s`,
|
||||
s.tables.points, s.dialect.placeholder(1), s.dialect.placeholder(2),
|
||||
)
|
||||
result, err := tx.ExecContext(ctx, sqlText, metricName, before.UTC().UnixNano())
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return result.RowsAffected()
|
||||
}
|
||||
|
||||
// floorDivNano floors ts to the start of its size-wide bucket, handling
|
||||
// negative timestamps (pre-epoch) toward negative infinity so buckets align
|
||||
// consistently. Mirrors alignTime but operates on raw nanos.
|
||||
//
|
||||
// floorDivNano 将 ts 向下对齐到 size 宽桶的起点,并把负时间戳(Unix epoch 前)
|
||||
// 朝负无穷取整,让桶保持一致对齐。它与 alignTime 逻辑一致,但直接操作纳秒值。
|
||||
func floorDivNano(ts, size int64) int64 {
|
||||
rem := ((ts % size) + size) % size
|
||||
return ts - rem
|
||||
}
|
||||
|
||||
// readRollupBucketTx reads a single rollup bucket from a transaction.
|
||||
func (s *Store) readRollupBucketTx(ctx context.Context, metricName, entityID, tagsHash string, interval time.Duration, bucketNano int64, tx *sql.Tx) (*rollupBucket, error) {
|
||||
sqlText := fmt.Sprintf(
|
||||
`SELECT count, sum, sum_sq, min_val, max_val, first_val, first_ts, last_val, last_ts, digest, tags
|
||||
FROM %s WHERE metric_name = %s AND resolution_nano = %s AND entity_id = %s
|
||||
AND tags_hash = %s AND bucket_nano = %s`,
|
||||
s.tables.rollups,
|
||||
s.dialect.placeholder(1), s.dialect.placeholder(2), s.dialect.placeholder(3),
|
||||
s.dialect.placeholder(4), s.dialect.placeholder(5),
|
||||
)
|
||||
row := tx.QueryRowContext(ctx, sqlText, metricName, interval.Nanoseconds(), entityID, tagsHash, bucketNano)
|
||||
|
||||
var count int64
|
||||
var sum, sumSq, minV, maxV, firstV, lastV float64
|
||||
var firstTS, lastTS int64
|
||||
var digestBlob []byte
|
||||
var rawTags any
|
||||
|
||||
err := row.Scan(&count, &sum, &sumSq, &minV, &maxV, &firstV, &firstTS, &lastV, &lastTS, &digestBlob, &rawTags)
|
||||
if err == sql.ErrNoRows {
|
||||
return nil, nil
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
td, err := DecodeTDigest(digestBlob)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
tagsJSON, err := rawTagsToJSON(rawTags)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return &rollupBucket{
|
||||
count: count, sum: sum, sumSq: sumSq,
|
||||
min: minV, max: maxV,
|
||||
firstVal: firstV, firstTS: firstTS,
|
||||
lastVal: lastV, lastTS: lastTS,
|
||||
digest: td,
|
||||
tagsHash: tagsHash,
|
||||
tagsJSON: tagsJSON,
|
||||
}, nil
|
||||
}
|
||||
@@ -0,0 +1,349 @@
|
||||
package metric
|
||||
|
||||
import (
|
||||
"context"
|
||||
"math"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// TestRollupKeepsTagSeriesSeparate is the regression test for the bug where the
|
||||
// rollup dropped the tag dimension: two points in the SAME metric/entity/bucket
|
||||
// but with different tags must become two independent rollup series, and a tag
|
||||
// filter must return that tag's own value rather than a cross-tag merge.
|
||||
//
|
||||
// TestRollupKeepsTagSeriesSeparate 是针对 rollup 曾丢失标签维度这一问题的
|
||||
// 回归测试:同一个指标、实体和桶内带有不同标签的两个点,必须成为两条独立的
|
||||
// rollup 序列;标签过滤必须返回该标签自己的值,而不是跨标签合并结果。
|
||||
func TestRollupKeepsTagSeriesSeparate(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
policy := RollupPolicy{Tiers: []RollupTier{{Interval: time.Minute, Retention: 24 * time.Hour}}}
|
||||
s := newRollupStore(t, policy)
|
||||
if err := s.CreateMetric(ctx, Definition{Name: "util", Type: TypeGauge}); err != nil {
|
||||
t.Fatalf("create: %v", err)
|
||||
}
|
||||
base := time.Date(2026, 6, 18, 0, 0, 0, 0, time.UTC)
|
||||
// Same entity, same 1-minute bucket; two GPUs distinguished only by tag.
|
||||
// device 0 carries values 10..19, device 1 carries 80..89.
|
||||
var batch []Point
|
||||
for i := 0; i < 10; i++ {
|
||||
batch = append(batch,
|
||||
Point{MetricName: "util", EntityID: "host-1", Timestamp: base.Add(time.Duration(i) * time.Second), Value: float64(10 + i), Tags: map[string]string{"device_index": "0"}},
|
||||
Point{MetricName: "util", EntityID: "host-1", Timestamp: base.Add(time.Duration(i) * time.Second), Value: float64(80 + i), Tags: map[string]string{"device_index": "1"}},
|
||||
)
|
||||
}
|
||||
if err := s.WriteBatch(ctx, batch); err != nil {
|
||||
t.Fatalf("write: %v", err)
|
||||
}
|
||||
written, err := s.Compact(ctx, base.Add(10*time.Minute))
|
||||
if err != nil {
|
||||
t.Fatalf("compact: %v", err)
|
||||
}
|
||||
// Two distinct tag series in the one bucket => two rollup rows.
|
||||
if written != 2 {
|
||||
t.Fatalf("expected 2 rollup series (one per device_index), got %d", written)
|
||||
}
|
||||
|
||||
q := func(dev string) AggregateQuery {
|
||||
return AggregateQuery{
|
||||
Query: Query{
|
||||
MetricName: "util", EntityID: "host-1",
|
||||
Start: base, End: base.Add(10 * time.Minute),
|
||||
Tags: map[string]string{"device_index": dev},
|
||||
},
|
||||
Aggregation: AggAvg,
|
||||
Interval: time.Minute,
|
||||
}
|
||||
}
|
||||
// device 0: avg of 10..19 = 14.5
|
||||
res0, err := s.AggregateRollup(ctx, q("0"), time.Minute)
|
||||
if err != nil {
|
||||
t.Fatalf("rollup dev0: %v", err)
|
||||
}
|
||||
if len(res0) != 1 || res0[0].Count != 10 || math.Abs(res0[0].Value-14.5) > 1e-9 {
|
||||
t.Fatalf("device 0 should be its own series avg=14.5 count=10, got %#v", res0)
|
||||
}
|
||||
// device 1: avg of 80..89 = 84.5
|
||||
res1, err := s.AggregateRollup(ctx, q("1"), time.Minute)
|
||||
if err != nil {
|
||||
t.Fatalf("rollup dev1: %v", err)
|
||||
}
|
||||
if len(res1) != 1 || res1[0].Count != 10 || math.Abs(res1[0].Value-84.5) > 1e-9 {
|
||||
t.Fatalf("device 1 should be its own series avg=84.5 count=10, got %#v", res1)
|
||||
}
|
||||
|
||||
// No tag filter => both series merge (count 20, avg of all 20 values).
|
||||
all, err := s.AggregateRollup(ctx, AggregateQuery{
|
||||
Query: Query{MetricName: "util", EntityID: "host-1", Start: base, End: base.Add(10 * time.Minute)},
|
||||
Aggregation: AggAvg,
|
||||
Interval: time.Minute,
|
||||
}, time.Minute)
|
||||
if err != nil {
|
||||
t.Fatalf("rollup all: %v", err)
|
||||
}
|
||||
wantAvg := (145.0 + 845.0) / 20.0 // sum(10..19)=145, sum(80..89)=845
|
||||
if len(all) != 1 || all[0].Count != 20 || math.Abs(all[0].Value-wantAvg) > 1e-9 {
|
||||
t.Fatalf("no-filter should merge both tags count=20 avg=%.2f, got %#v", wantAvg, all)
|
||||
}
|
||||
}
|
||||
|
||||
// TestRollupTagPercentileSurvivesRetention proves the headline TSDB property
|
||||
// still holds per tag: after raw is deleted, a tag-filtered percentile is still
|
||||
// answerable from that tag's surviving rollup series (and not contaminated by
|
||||
// the other tag).
|
||||
//
|
||||
// TestRollupTagPercentileSurvivesRetention 验证核心 TSDB 特性在每个标签上
|
||||
// 仍然成立:原始点删除后,按标签过滤的百分位仍能从该标签保留下来的 rollup 序列
|
||||
// 回答,并且不会被另一个标签污染。
|
||||
func TestRollupTagPercentileSurvivesRetention(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
policy := RollupPolicy{
|
||||
RawRetention: 2 * time.Minute,
|
||||
Tiers: []RollupTier{{Interval: time.Minute, Retention: 365 * 24 * time.Hour}},
|
||||
}
|
||||
s := newRollupStore(t, policy)
|
||||
if err := s.CreateMetric(ctx, Definition{Name: "lat", Type: TypeGauge}); err != nil {
|
||||
t.Fatalf("create: %v", err)
|
||||
}
|
||||
base := time.Date(2026, 6, 18, 0, 0, 0, 0, time.UTC)
|
||||
// One 1m bucket. region=ap: values 0..99; region=eu: values 1000..1099.
|
||||
var batch []Point
|
||||
for i := 0; i < 100; i++ {
|
||||
ts := base.Add(time.Duration(i) * time.Millisecond * 500)
|
||||
batch = append(batch,
|
||||
Point{MetricName: "lat", EntityID: "n1", Timestamp: ts, Value: float64(i), Tags: map[string]string{"region": "ap"}},
|
||||
Point{MetricName: "lat", EntityID: "n1", Timestamp: ts, Value: float64(1000 + i), Tags: map[string]string{"region": "eu"}},
|
||||
)
|
||||
}
|
||||
if err := s.WriteBatch(ctx, batch); err != nil {
|
||||
t.Fatalf("write: %v", err)
|
||||
}
|
||||
if _, err := s.Compact(ctx, base.Add(48*time.Hour)); err != nil {
|
||||
t.Fatalf("compact: %v", err)
|
||||
}
|
||||
// Raw gone.
|
||||
raw, err := s.Query(ctx, Query{MetricName: "lat", EntityID: "n1", Start: base, End: base.Add(time.Hour)})
|
||||
if err != nil {
|
||||
t.Fatalf("query raw: %v", err)
|
||||
}
|
||||
if len(raw) != 0 {
|
||||
t.Fatalf("expected raw deleted, got %d", len(raw))
|
||||
}
|
||||
// p90 of region=ap must be near the ap-only exact (≈89), NOT pulled up toward
|
||||
// the eu values (which would happen if tags were merged).
|
||||
res, err := s.AggregateRollup(ctx, AggregateQuery{
|
||||
Query: Query{
|
||||
MetricName: "lat", EntityID: "n1",
|
||||
Start: base, End: base.Add(time.Hour),
|
||||
Tags: map[string]string{"region": "ap"},
|
||||
},
|
||||
Aggregation: Pxx(90),
|
||||
Interval: time.Minute,
|
||||
}, time.Minute)
|
||||
if err != nil {
|
||||
t.Fatalf("rollup ap p90: %v", err)
|
||||
}
|
||||
if len(res) != 1 {
|
||||
t.Fatalf("expected 1 ap bucket, got %d", len(res))
|
||||
}
|
||||
exactAP := percentileSortedRange(0, 99, 0.90)
|
||||
if math.Abs(res[0].Value-exactAP)/exactAP > 0.05 {
|
||||
t.Fatalf("ap p90 %v contaminated or wrong (exact ap ≈ %v)", res[0].Value, exactAP)
|
||||
}
|
||||
if res[0].Value > 500 {
|
||||
t.Fatalf("ap p90 %v is contaminated by eu series (>500)", res[0].Value)
|
||||
}
|
||||
}
|
||||
|
||||
// TestSeriesTagFilterRoutesPerTag checks the auto-routing read path honors tags
|
||||
// on both the raw branch (recent) and the rollup branch (old).
|
||||
//
|
||||
// TestSeriesTagFilterRoutesPerTag 检查自动路由读取路径在原始分支(近期数据)和
|
||||
// rollup 分支(旧数据)上都遵守标签。
|
||||
func TestSeriesTagFilterRoutesPerTag(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
policy := RollupPolicy{
|
||||
RawRetention: time.Hour,
|
||||
Tiers: []RollupTier{{Interval: time.Minute, Retention: 30 * 24 * time.Hour}},
|
||||
}
|
||||
s := newRollupStore(t, policy)
|
||||
if err := s.CreateMetric(ctx, Definition{Name: "m", Type: TypeGauge}); err != nil {
|
||||
t.Fatalf("create: %v", err)
|
||||
}
|
||||
now := time.Date(2026, 6, 18, 12, 0, 0, 0, time.UTC)
|
||||
old := now.Add(-48 * time.Hour)
|
||||
// Old bucket: env=prod values 0..9, env=stage values 500..509.
|
||||
var batch []Point
|
||||
for i := 0; i < 10; i++ {
|
||||
ts := old.Add(time.Duration(i) * time.Second)
|
||||
batch = append(batch,
|
||||
Point{MetricName: "m", EntityID: "n1", Timestamp: ts, Value: float64(i), Tags: map[string]string{"env": "prod"}},
|
||||
Point{MetricName: "m", EntityID: "n1", Timestamp: ts, Value: float64(500 + i), Tags: map[string]string{"env": "stage"}},
|
||||
)
|
||||
}
|
||||
if err := s.WriteBatch(ctx, batch); err != nil {
|
||||
t.Fatalf("write: %v", err)
|
||||
}
|
||||
if _, err := s.Compact(ctx, now); err != nil {
|
||||
t.Fatalf("compact: %v", err)
|
||||
}
|
||||
// Old window, raw aged out -> Series must use rollup, filtered to env=prod
|
||||
// (avg of 0..9 = 4.5), not the merged 252.
|
||||
got, err := s.Series(ctx, AggregateQuery{
|
||||
Query: Query{
|
||||
MetricName: "m", EntityID: "n1",
|
||||
Start: old.Add(-time.Minute), End: old.Add(2 * time.Minute),
|
||||
Tags: map[string]string{"env": "prod"},
|
||||
},
|
||||
Aggregation: AggAvg,
|
||||
Interval: time.Minute,
|
||||
}, now)
|
||||
if err != nil {
|
||||
t.Fatalf("series old prod: %v", err)
|
||||
}
|
||||
var found bool
|
||||
for _, p := range got {
|
||||
if p.Count == 10 && math.Abs(p.Value-4.5) < 1e-9 {
|
||||
found = true
|
||||
}
|
||||
if p.Count > 10 {
|
||||
t.Fatalf("env=prod bucket contaminated by stage: %#v", p)
|
||||
}
|
||||
}
|
||||
if !found {
|
||||
t.Fatalf("env=prod old series not served correctly from rollup: %#v", got)
|
||||
}
|
||||
}
|
||||
|
||||
// TestDeleteSeriesRemovesTaggedTaskAcrossAgents verifies a task_id tag can be
|
||||
// deleted across every agent, including stored rollups.
|
||||
//
|
||||
// TestDeleteSeriesRemovesTaggedTaskAcrossAgents 验证可以跨所有 agent 删除某个
|
||||
// task_id 标签,包括已存储的 rollup。
|
||||
func TestDeleteSeriesRemovesTaggedTaskAcrossAgents(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
policy := RollupPolicy{
|
||||
RawRetention: 2 * time.Minute,
|
||||
Tiers: []RollupTier{{Interval: time.Minute, Retention: 24 * time.Hour}},
|
||||
}
|
||||
s := newRollupStore(t, policy)
|
||||
if err := s.CreateMetric(ctx, Definition{Name: "ping.latency_ms", Type: TypeGauge}); err != nil {
|
||||
t.Fatalf("create: %v", err)
|
||||
}
|
||||
base := time.Date(2026, 6, 18, 0, 0, 0, 0, time.UTC)
|
||||
var batch []Point
|
||||
for _, agent := range []string{"agent-001", "agent-002"} {
|
||||
for i := 0; i < 60; i++ {
|
||||
ts := base.Add(time.Duration(i) * time.Second)
|
||||
batch = append(batch,
|
||||
Point{MetricName: "ping.latency_ms", EntityID: agent, Timestamp: ts, Value: float64(20 + i), Tags: map[string]string{"task_id": "pingtask1"}},
|
||||
Point{MetricName: "ping.latency_ms", EntityID: agent, Timestamp: ts, Value: float64(60 + i), Tags: map[string]string{"task_id": "pingtask2"}},
|
||||
)
|
||||
}
|
||||
}
|
||||
if err := s.WriteBatch(ctx, batch); err != nil {
|
||||
t.Fatalf("write: %v", err)
|
||||
}
|
||||
if _, err := s.Compact(ctx, base.Add(time.Hour)); err != nil {
|
||||
t.Fatalf("compact: %v", err)
|
||||
}
|
||||
if _, err := s.DeleteSeries(ctx, Query{
|
||||
MetricName: "ping.latency_ms",
|
||||
Tags: map[string]string{"task_id": "pingtask2"},
|
||||
}); err != nil {
|
||||
t.Fatalf("delete series: %v", err)
|
||||
}
|
||||
|
||||
task2, err := s.AggregateRollup(ctx, AggregateQuery{
|
||||
Query: Query{
|
||||
MetricName: "ping.latency_ms",
|
||||
Start: base, End: base.Add(time.Minute),
|
||||
Tags: map[string]string{"task_id": "pingtask2"},
|
||||
},
|
||||
Aggregation: AggAvg,
|
||||
Interval: time.Minute,
|
||||
}, time.Minute)
|
||||
if err != nil {
|
||||
t.Fatalf("rollup task2: %v", err)
|
||||
}
|
||||
if len(task2) != 0 {
|
||||
t.Fatalf("deleted task still has rollup buckets: %#v", task2)
|
||||
}
|
||||
task1, err := s.AggregateRollup(ctx, AggregateQuery{
|
||||
Query: Query{
|
||||
MetricName: "ping.latency_ms",
|
||||
Start: base, End: base.Add(time.Minute),
|
||||
Tags: map[string]string{"task_id": "pingtask1"},
|
||||
},
|
||||
Aggregation: AggAvg,
|
||||
Interval: time.Minute,
|
||||
}, time.Minute)
|
||||
if err != nil {
|
||||
t.Fatalf("rollup task1: %v", err)
|
||||
}
|
||||
if len(task1) != 1 || task1[0].Count != 120 {
|
||||
t.Fatalf("remaining task should keep both agents, got %#v", task1)
|
||||
}
|
||||
}
|
||||
|
||||
// TestDeleteMetricRemovesRollups verifies metric deletion clears rollup rows.
|
||||
//
|
||||
// TestDeleteMetricRemovesRollups 验证删除指标会清理 rollup 行。
|
||||
func TestDeleteMetricRemovesRollups(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
s := newRollupStore(t, RollupPolicy{
|
||||
Tiers: []RollupTier{{Interval: time.Minute, Retention: 24 * time.Hour}},
|
||||
})
|
||||
if err := s.CreateMetric(ctx, Definition{Name: "gone", Type: TypeGauge}); err != nil {
|
||||
t.Fatalf("create: %v", err)
|
||||
}
|
||||
base := time.Date(2026, 6, 18, 0, 0, 0, 0, time.UTC)
|
||||
if err := s.Write(ctx, Point{MetricName: "gone", EntityID: "agent-001", Timestamp: base, Value: 1}); err != nil {
|
||||
t.Fatalf("write: %v", err)
|
||||
}
|
||||
if _, err := s.Compact(ctx, base.Add(time.Hour)); err != nil {
|
||||
t.Fatalf("compact: %v", err)
|
||||
}
|
||||
if err := s.DeleteMetric(ctx, "gone"); err != nil {
|
||||
t.Fatalf("delete metric: %v", err)
|
||||
}
|
||||
rows, err := s.scanRollupRows(ctx, s.reader(), "gone", time.Minute)
|
||||
|
||||
if err != nil {
|
||||
t.Fatalf("scan rollups: %v", err)
|
||||
}
|
||||
if len(rows) != 0 {
|
||||
t.Fatalf("deleted metric still has rollups: %#v", rows)
|
||||
}
|
||||
}
|
||||
|
||||
// TestTagsFingerprintStable verifies equal tag maps fingerprint identically
|
||||
// regardless of construction order, and different maps differ.
|
||||
//
|
||||
// TestTagsFingerprintStable 验证相同标签 map 无论构造顺序如何都会生成相同指纹,
|
||||
// 而不同 map 会得到不同指纹。
|
||||
func TestTagsFingerprintStable(t *testing.T) {
|
||||
a := map[string]string{"region": "ap", "device_index": "3"}
|
||||
b := map[string]string{"device_index": "3", "region": "ap"}
|
||||
ha, _, err := tagsFingerprint(a)
|
||||
if err != nil {
|
||||
t.Fatalf("fp a: %v", err)
|
||||
}
|
||||
hb, _, err := tagsFingerprint(b)
|
||||
if err != nil {
|
||||
t.Fatalf("fp b: %v", err)
|
||||
}
|
||||
if ha != hb {
|
||||
t.Fatalf("equal tag maps must fingerprint identically: %s vs %s", ha, hb)
|
||||
}
|
||||
hc, _, _ := tagsFingerprint(map[string]string{"region": "eu", "device_index": "3"})
|
||||
if ha == hc {
|
||||
t.Fatalf("different tag maps must fingerprint differently")
|
||||
}
|
||||
// nil and empty map agree.
|
||||
hn, _, _ := tagsFingerprint(nil)
|
||||
he, _, _ := tagsFingerprint(map[string]string{})
|
||||
if hn != he {
|
||||
t.Fatalf("nil and empty tag maps must agree: %s vs %s", hn, he)
|
||||
}
|
||||
}
|
||||
Loaded 100 of 238 files, more files were not shown because too many files have changed in this diff.
Show more
Reference in new issue
Block a user