feat: remove charts to improve performance (backend only)

This commit is contained in:
naskya 2023-09-05 00:18:28 +09:00
parent 3d81d35b7e
commit b8ac374da4
Signed by: naskya
GPG key ID: 164DFF24E2D40139
63 changed files with 202 additions and 1527 deletions

View file

@ -10,6 +10,8 @@
## 主要な変更点 ## 主要な変更点
- パフォーマンス向上のためアクティブユーザー以外のチャート生成を無効化
- サードパーティー製クライアントが動かなくなるのを阻止するため API のエンドポイントは残していますが、叩いても `0` が並んだ配列しか返しません。
- モデレーターでない一般ユーザーにもカスタム絵文字の管理権を与えられるように - モデレーターでない一般ユーザーにもカスタム絵文字の管理権を与えられるように
- カスタム絵文字の管理が大変なサーバー管理者さんがたくさんいらっしゃったのでこの機能を追加するべきではないか他の開発者に訊いたところロール機能の実装を待つべきだと言われてしまったが、Firefish のロール機能は現状では仕様がまだ固まっておらず実装までに時間が掛かると考えられるため - カスタム絵文字の管理が大変なサーバー管理者さんがたくさんいらっしゃったのでこの機能を追加するべきではないか他の開発者に訊いたところロール機能の実装を待つべきだと言われてしまったが、Firefish のロール機能は現状では仕様がまだ固まっておらず実装までに時間が掛かると考えられるため
- 以下の権限を与えられます - 以下の権限を与えられます

View file

@ -537,15 +537,6 @@ export default function () {
}, },
); );
systemQueue.add(
"resyncCharts",
{},
{
repeat: { cron: "0 0 * * *" },
removeOnComplete: true,
},
);
systemQueue.add( systemQueue.add(
"cleanCharts", "cleanCharts",
{}, {},

View file

@ -3,11 +3,6 @@ import request from "@/remote/activitypub/request.js";
import { registerOrFetchInstanceDoc } from "@/services/register-or-fetch-instance-doc.js"; import { registerOrFetchInstanceDoc } from "@/services/register-or-fetch-instance-doc.js";
import Logger from "@/services/logger.js"; import Logger from "@/services/logger.js";
import { Instances } from "@/models/index.js"; import { Instances } from "@/models/index.js";
import {
apRequestChart,
federationChart,
instanceChart,
} from "@/services/chart/index.js";
import { fetchInstanceMetadata } from "@/services/fetch-instance-metadata.js"; import { fetchInstanceMetadata } from "@/services/fetch-instance-metadata.js";
import { toPuny } from "@/misc/convert-host.js"; import { toPuny } from "@/misc/convert-host.js";
import { StatusError } from "@/misc/fetch.js"; import { StatusError } from "@/misc/fetch.js";
@ -42,10 +37,6 @@ export default async (job: Bull.Job<DeliverJobData>) => {
}); });
fetchInstanceMetadata(i); fetchInstanceMetadata(i);
instanceChart.requestSent(i.host, true);
apRequestChart.deliverSucc();
federationChart.deliverd(i.host, true);
}); });
return "Success"; return "Success";
@ -57,10 +48,6 @@ export default async (job: Bull.Job<DeliverJobData>) => {
latestStatus: res instanceof StatusError ? res.statusCode : null, latestStatus: res instanceof StatusError ? res.statusCode : null,
isNotResponding: true, isNotResponding: true,
}); });
instanceChart.requestSent(i.host, false);
apRequestChart.deliverFail();
federationChart.deliverd(i.host, false);
}); });
if (res instanceof StatusError) { if (res instanceof StatusError) {

View file

@ -5,11 +5,6 @@ import perform from "@/remote/activitypub/perform.js";
import Logger from "@/services/logger.js"; import Logger from "@/services/logger.js";
import { registerOrFetchInstanceDoc } from "@/services/register-or-fetch-instance-doc.js"; import { registerOrFetchInstanceDoc } from "@/services/register-or-fetch-instance-doc.js";
import { Instances } from "@/models/index.js"; import { Instances } from "@/models/index.js";
import {
apRequestChart,
federationChart,
instanceChart,
} from "@/services/chart/index.js";
import { fetchMeta } from "@/misc/fetch-meta.js"; import { fetchMeta } from "@/misc/fetch-meta.js";
import { toPuny, extractDbHost } from "@/misc/convert-host.js"; import { toPuny, extractDbHost } from "@/misc/convert-host.js";
import { getApId } from "@/remote/activitypub/type.js"; import { getApId } from "@/remote/activitypub/type.js";
@ -169,10 +164,6 @@ export default async (job: Bull.Job<InboxJobData>): Promise<string> => {
}); });
fetchInstanceMetadata(i); fetchInstanceMetadata(i);
instanceChart.requestReceived(i.host);
apRequestChart.inbox();
federationChart.inbox(i.host);
}); });
// アクティビティを処理 // アクティビティを処理

View file

@ -1,20 +1,7 @@
import type Bull from "bull"; import type Bull from "bull";
import { queueLogger } from "../../logger.js"; import { queueLogger } from "../../logger.js";
import { import { activeUsersChart } from "@/services/chart/index.js";
activeUsersChart,
driveChart,
federationChart,
hashtagChart,
instanceChart,
notesChart,
perUserDriveChart,
perUserFollowingChart,
perUserNotesChart,
perUserReactionsChart,
usersChart,
apRequestChart,
} from "@/services/chart/index.js";
const logger = queueLogger.createSubLogger("clean-charts"); const logger = queueLogger.createSubLogger("clean-charts");
@ -23,22 +10,7 @@ export async function cleanCharts(
done: any, done: any,
): Promise<void> { ): Promise<void> {
logger.info("Clean charts..."); logger.info("Clean charts...");
await activeUsersChart.clean();
await Promise.all([
federationChart.clean(),
notesChart.clean(),
usersChart.clean(),
activeUsersChart.clean(),
instanceChart.clean(),
perUserNotesChart.clean(),
driveChart.clean(),
perUserReactionsChart.clean(),
hashtagChart.clean(),
perUserFollowingChart.clean(),
perUserDriveChart.clean(),
apRequestChart.clean(),
]);
logger.succ("All charts successfully cleaned."); logger.succ("All charts successfully cleaned.");
done(); done();
} }

View file

@ -1,6 +1,5 @@
import type Bull from "bull"; import type Bull from "bull";
import { tickCharts } from "./tick-charts.js"; import { tickCharts } from "./tick-charts.js";
import { resyncCharts } from "./resync-charts.js";
import { cleanCharts } from "./clean-charts.js"; import { cleanCharts } from "./clean-charts.js";
import { checkExpiredMutings } from "./check-expired-mutings.js"; import { checkExpiredMutings } from "./check-expired-mutings.js";
import { clean } from "./clean.js"; import { clean } from "./clean.js";
@ -9,7 +8,6 @@ import { verifyLinks } from "./verify-links.js";
const jobs = { const jobs = {
tickCharts, tickCharts,
resyncCharts,
cleanCharts, cleanCharts,
checkExpiredMutings, checkExpiredMutings,
clean, clean,

View file

@ -1,24 +0,0 @@
import type Bull from "bull";
import { queueLogger } from "../../logger.js";
import { driveChart, notesChart, usersChart } from "@/services/chart/index.js";
const logger = queueLogger.createSubLogger("resync-charts");
export async function resyncCharts(
job: Bull.Job<Record<string, unknown>>,
done: any,
): Promise<void> {
logger.info("Resync charts...");
// TODO: ユーザーごとのチャートも更新する
// TODO: インスタンスごとのチャートも更新する
await Promise.all([
driveChart.resync(),
notesChart.resync(),
usersChart.resync(),
]);
logger.succ("All charts successfully resynced.");
done();
}

View file

@ -1,20 +1,7 @@
import type Bull from "bull"; import type Bull from "bull";
import { queueLogger } from "../../logger.js"; import { queueLogger } from "../../logger.js";
import { import { activeUsersChart } from "@/services/chart/index.js";
activeUsersChart,
driveChart,
federationChart,
hashtagChart,
instanceChart,
notesChart,
perUserDriveChart,
perUserFollowingChart,
perUserNotesChart,
perUserReactionsChart,
usersChart,
apRequestChart,
} from "@/services/chart/index.js";
const logger = queueLogger.createSubLogger("tick-charts"); const logger = queueLogger.createSubLogger("tick-charts");
@ -23,22 +10,7 @@ export async function tickCharts(
done: any, done: any,
): Promise<void> { ): Promise<void> {
logger.info("Tick charts..."); logger.info("Tick charts...");
await activeUsersChart.tick(false);
await Promise.all([
federationChart.tick(false),
notesChart.tick(false),
usersChart.tick(false),
activeUsersChart.tick(false),
instanceChart.tick(false),
perUserNotesChart.tick(false),
driveChart.tick(false),
perUserReactionsChart.tick(false),
hashtagChart.tick(false),
perUserFollowingChart.tick(false),
perUserDriveChart.tick(false),
apRequestChart.tick(false),
]);
logger.succ("All charts successfully ticked."); logger.succ("All charts successfully ticked.");
done(); done();
} }

View file

@ -18,7 +18,6 @@ import { User } from "@/models/entities/user.js";
import type { Emoji } from "@/models/entities/emoji.js"; import type { Emoji } from "@/models/entities/emoji.js";
import { UserNotePining } from "@/models/entities/user-note-pining.js"; import { UserNotePining } from "@/models/entities/user-note-pining.js";
import { genId } from "@/misc/gen-id.js"; import { genId } from "@/misc/gen-id.js";
import { instanceChart, usersChart } from "@/services/chart/index.js";
import { UserPublickey } from "@/models/entities/user-publickey.js"; import { UserPublickey } from "@/models/entities/user-publickey.js";
import { isDuplicateKeyValueError } from "@/misc/is-duplicate-key-value-error.js"; import { isDuplicateKeyValueError } from "@/misc/is-duplicate-key-value-error.js";
import { toPuny } from "@/misc/convert-host.js"; import { toPuny } from "@/misc/convert-host.js";
@ -352,12 +351,9 @@ export async function createPerson(
// Register host // Register host
registerOrFetchInstanceDoc(host).then((i) => { registerOrFetchInstanceDoc(host).then((i) => {
Instances.increment({ id: i.id }, "usersCount", 1); Instances.increment({ id: i.id }, "usersCount", 1);
instanceChart.newUser(i.host);
fetchInstanceMetadata(i); fetchInstanceMetadata(i);
}); });
usersChart.update(user!, true);
// Hashtag update // Hashtag update
updateUsertags(user!, tags); updateUsertags(user!, tags);

View file

@ -7,7 +7,6 @@ import { IsNull } from "typeorm";
import { genId } from "@/misc/gen-id.js"; import { genId } from "@/misc/gen-id.js";
import { toPunyNullable } from "@/misc/convert-host.js"; import { toPunyNullable } from "@/misc/convert-host.js";
import { UserKeypair } from "@/models/entities/user-keypair.js"; import { UserKeypair } from "@/models/entities/user-keypair.js";
import { usersChart } from "@/services/chart/index.js";
import { UsedUsername } from "@/models/entities/used-username.js"; import { UsedUsername } from "@/models/entities/used-username.js";
import { db } from "@/db/postgre.js"; import { db } from "@/db/postgre.js";
import config from "@/config/index.js"; import config from "@/config/index.js";
@ -135,7 +134,5 @@ export async function signup(opts: {
); );
}); });
usersChart.update(account, true);
return { account, secret }; return { account, secret };
} }

View file

@ -1,15 +1,9 @@
import { getJsonSchema } from "@/services/chart/core.js";
import { apRequestChart } from "@/services/chart/index.js";
import define from "../../define.js"; import define from "../../define.js";
export const meta = { export const meta = {
tags: ["charts"], tags: ["charts"],
requireCredentialPrivateMode: true, requireCredentialPrivateMode: true,
res: getJsonSchema(apRequestChart.schema),
allowGet: true, allowGet: true,
cacheSec: 60 * 60,
} as const; } as const;
export const paramDef = { export const paramDef = {
@ -23,9 +17,10 @@ export const paramDef = {
} as const; } as const;
export default define(meta, paramDef, async (ps) => { export default define(meta, paramDef, async (ps) => {
return await apRequestChart.getChart( const zeros = new Array<number>(ps.limit ?? 30).fill(0);
ps.span, return {
ps.limit, deliverFailed: zeros,
ps.offset ? new Date(ps.offset) : null, deliverSucceeded: zeros,
); inboxReceived: zeros,
};
}); });

View file

@ -1,15 +1,9 @@
import { getJsonSchema } from "@/services/chart/core.js";
import { driveChart } from "@/services/chart/index.js";
import define from "../../define.js"; import define from "../../define.js";
export const meta = { export const meta = {
tags: ["charts", "drive"], tags: ["charts", "drive"],
requireCredentialPrivateMode: true, requireCredentialPrivateMode: true,
res: getJsonSchema(driveChart.schema),
allowGet: true, allowGet: true,
cacheSec: 60 * 60,
} as const; } as const;
export const paramDef = { export const paramDef = {
@ -23,9 +17,23 @@ export const paramDef = {
} as const; } as const;
export default define(meta, paramDef, async (ps) => { export default define(meta, paramDef, async (ps) => {
return await driveChart.getChart( const zeros = new Array<number>(ps.limit ?? 30).fill(0);
ps.span, return {
ps.limit, local: {
ps.offset ? new Date(ps.offset) : null, decCount: zeros,
); decSize: zeros,
incCount: zeros,
incSize: zeros,
totalCount: zeros,
totalSize: zeros,
},
remote: {
decCount: zeros,
decSize: zeros,
incCount: zeros,
incSize: zeros,
totalCount: zeros,
totalSize: zeros,
},
};
}); });

View file

@ -1,15 +1,9 @@
import { getJsonSchema } from "@/services/chart/core.js";
import { federationChart } from "@/services/chart/index.js";
import define from "../../define.js"; import define from "../../define.js";
export const meta = { export const meta = {
tags: ["charts"], tags: ["charts"],
requireCredentialPrivateMode: true, requireCredentialPrivateMode: true,
res: getJsonSchema(federationChart.schema),
allowGet: true, allowGet: true,
cacheSec: 60 * 60,
} as const; } as const;
export const paramDef = { export const paramDef = {
@ -23,9 +17,15 @@ export const paramDef = {
} as const; } as const;
export default define(meta, paramDef, async (ps) => { export default define(meta, paramDef, async (ps) => {
return await federationChart.getChart( const zeros = Array<number>(ps.limit ?? 30).fill(0);
ps.span, return {
ps.limit, deliveredInstances: zeros,
ps.offset ? new Date(ps.offset) : null, inboxInstances: zeros,
); stalled: zeros,
sub: zeros,
pub: zeros,
pubsub: zeros,
subActive: zeros,
pubActive: zeros,
};
}); });

View file

@ -1,15 +1,9 @@
import { getJsonSchema } from "@/services/chart/core.js";
import { hashtagChart } from "@/services/chart/index.js";
import define from "../../define.js"; import define from "../../define.js";
export const meta = { export const meta = {
tags: ["charts", "hashtags"], tags: ["charts", "hashtags"],
requireCredentialPrivateMode: true, requireCredentialPrivateMode: true,
res: getJsonSchema(hashtagChart.schema),
allowGet: true, allowGet: true,
cacheSec: 60 * 60,
} as const; } as const;
export const paramDef = { export const paramDef = {
@ -24,10 +18,13 @@ export const paramDef = {
} as const; } as const;
export default define(meta, paramDef, async (ps) => { export default define(meta, paramDef, async (ps) => {
return await hashtagChart.getChart( const zeros = new Array<number>(ps.limit ?? 30).fill(0);
ps.span, return {
ps.limit, local: {
ps.offset ? new Date(ps.offset) : null, users: zeros,
ps.tag, },
); remote: {
users: zeros,
},
};
}); });

View file

@ -1,15 +1,9 @@
import { getJsonSchema } from "@/services/chart/core.js";
import { instanceChart } from "@/services/chart/index.js";
import define from "../../define.js"; import define from "../../define.js";
export const meta = { export const meta = {
tags: ["charts"], tags: ["charts"],
requireCredentialPrivateMode: true, requireCredentialPrivateMode: true,
res: getJsonSchema(instanceChart.schema),
allowGet: true, allowGet: true,
cacheSec: 60 * 60,
} as const; } as const;
export const paramDef = { export const paramDef = {
@ -24,10 +18,24 @@ export const paramDef = {
} as const; } as const;
export default define(meta, paramDef, async (ps) => { export default define(meta, paramDef, async (ps) => {
return await instanceChart.getChart( const zeros = new Array<number>(ps.limit ?? 30).fill(0);
ps.span, return {
ps.limit, requests: { failed: zeros, succeeded: zeros, received: zeros },
ps.offset ? new Date(ps.offset) : null, notes: {
ps.host, total: zeros,
); inc: zeros,
dec: zeros,
diffs: { normal: zeros, reply: zeros, renote: zeros, withFile: zeros },
},
users: { total: zeros, inc: zeros, dec: zeros },
following: { total: zeros, inc: zeros, dec: zeros },
followers: { total: zeros, inc: zeros, dec: zeros },
drive: {
totalFiles: zeros,
incFiles: zeros,
decFiles: zeros,
incUsage: zeros,
decUsage: zeros,
},
};
}); });

View file

@ -1,15 +1,9 @@
import { getJsonSchema } from "@/services/chart/core.js";
import { notesChart } from "@/services/chart/index.js";
import define from "../../define.js"; import define from "../../define.js";
export const meta = { export const meta = {
tags: ["charts", "notes"], tags: ["charts", "notes"],
requireCredentialPrivateMode: true, requireCredentialPrivateMode: true,
res: getJsonSchema(notesChart.schema),
allowGet: true, allowGet: true,
cacheSec: 60 * 60,
} as const; } as const;
export const paramDef = { export const paramDef = {
@ -23,9 +17,29 @@ export const paramDef = {
} as const; } as const;
export default define(meta, paramDef, async (ps) => { export default define(meta, paramDef, async (ps) => {
return await notesChart.getChart( const zeros = new Array<number>(ps.limit ?? 30).fill(0);
ps.span, return {
ps.limit, local: {
ps.offset ? new Date(ps.offset) : null, total: zeros,
); inc: zeros,
dec: zeros,
diffs: {
normal: zeros,
reply: zeros,
renote: zeros,
withFile: zeros,
},
},
remote: {
total: zeros,
inc: zeros,
dec: zeros,
diffs: {
normal: zeros,
reply: zeros,
renote: zeros,
withFile: zeros,
},
},
};
}); });

View file

@ -1,15 +1,9 @@
import { getJsonSchema } from "@/services/chart/core.js";
import { perUserDriveChart } from "@/services/chart/index.js";
import define from "../../../define.js"; import define from "../../../define.js";
export const meta = { export const meta = {
tags: ["charts", "drive", "users"], tags: ["charts", "drive", "users"],
requireCredentialPrivateMode: true, requireCredentialPrivateMode: true,
res: getJsonSchema(perUserDriveChart.schema),
allowGet: true, allowGet: true,
cacheSec: 60 * 60,
} as const; } as const;
export const paramDef = { export const paramDef = {
@ -24,10 +18,13 @@ export const paramDef = {
} as const; } as const;
export default define(meta, paramDef, async (ps) => { export default define(meta, paramDef, async (ps) => {
return await perUserDriveChart.getChart( const zeros = new Array<number>(ps.limit ?? 30).fill(0);
ps.span, return {
ps.limit, totalCount: zeros,
ps.offset ? new Date(ps.offset) : null, totalSize: zeros,
ps.userId, incCount: zeros,
); incSize: zeros,
decCount: zeros,
decSize: zeros,
};
}); });

View file

@ -1,15 +1,9 @@
import define from "../../../define.js"; import define from "../../../define.js";
import { getJsonSchema } from "@/services/chart/core.js";
import { perUserFollowingChart } from "@/services/chart/index.js";
export const meta = { export const meta = {
tags: ["charts", "users", "following"], tags: ["charts", "users", "following"],
requireCredentialPrivateMode: true, requireCredentialPrivateMode: true,
res: getJsonSchema(perUserFollowingChart.schema),
allowGet: true, allowGet: true,
cacheSec: 60 * 60,
} as const; } as const;
export const paramDef = { export const paramDef = {
@ -24,10 +18,31 @@ export const paramDef = {
} as const; } as const;
export default define(meta, paramDef, async (ps) => { export default define(meta, paramDef, async (ps) => {
return await perUserFollowingChart.getChart( const zeros = new Array<number>(ps.limit ?? 30).fill(0);
ps.span, return {
ps.limit, local: {
ps.offset ? new Date(ps.offset) : null, followings: {
ps.userId, total: zeros,
); inc: zeros,
dec: zeros,
},
followers: {
total: zeros,
inc: zeros,
dec: zeros,
},
},
remote: {
followings: {
total: zeros,
inc: zeros,
dec: zeros,
},
followers: {
total: zeros,
inc: zeros,
dec: zeros,
},
},
};
}); });

View file

@ -1,15 +1,9 @@
import { getJsonSchema } from "@/services/chart/core.js";
import { perUserNotesChart } from "@/services/chart/index.js";
import define from "../../../define.js"; import define from "../../../define.js";
export const meta = { export const meta = {
tags: ["charts", "users", "notes"], tags: ["charts", "users", "notes"],
requireCredentialPrivateMode: true, requireCredentialPrivateMode: true,
res: getJsonSchema(perUserNotesChart.schema),
allowGet: true, allowGet: true,
cacheSec: 60 * 60,
} as const; } as const;
export const paramDef = { export const paramDef = {
@ -24,10 +18,16 @@ export const paramDef = {
} as const; } as const;
export default define(meta, paramDef, async (ps) => { export default define(meta, paramDef, async (ps) => {
return await perUserNotesChart.getChart( const zeros = new Array<number>(ps.limit ?? 30).fill(0);
ps.span, return {
ps.limit, total: zeros,
ps.offset ? new Date(ps.offset) : null, inc: zeros,
ps.userId, dec: zeros,
); diffs: {
normal: zeros,
reply: zeros,
renote: zeros,
withFile: zeros,
},
};
}); });

View file

@ -1,15 +1,9 @@
import { getJsonSchema } from "@/services/chart/core.js";
import { perUserReactionsChart } from "@/services/chart/index.js";
import define from "../../../define.js"; import define from "../../../define.js";
export const meta = { export const meta = {
tags: ["charts", "users", "reactions"], tags: ["charts", "users", "reactions"],
requireCredentialPrivateMode: true, requireCredentialPrivateMode: true,
res: getJsonSchema(perUserReactionsChart.schema),
allowGet: true, allowGet: true,
cacheSec: 60 * 60,
} as const; } as const;
export const paramDef = { export const paramDef = {
@ -24,10 +18,13 @@ export const paramDef = {
} as const; } as const;
export default define(meta, paramDef, async (ps) => { export default define(meta, paramDef, async (ps) => {
return await perUserReactionsChart.getChart( const zeros = new Array<number>(ps.limit ?? 30).fill(0);
ps.span, return {
ps.limit, local: {
ps.offset ? new Date(ps.offset) : null, count: zeros,
ps.userId, },
); remote: {
count: zeros,
},
};
}); });

View file

@ -1,15 +1,9 @@
import { getJsonSchema } from "@/services/chart/core.js";
import { usersChart } from "@/services/chart/index.js";
import define from "../../define.js"; import define from "../../define.js";
export const meta = { export const meta = {
tags: ["charts", "users"], tags: ["charts", "users"],
requireCredentialPrivateMode: true, requireCredentialPrivateMode: true,
res: getJsonSchema(usersChart.schema),
allowGet: true, allowGet: true,
cacheSec: 60 * 60,
} as const; } as const;
export const paramDef = { export const paramDef = {
@ -23,9 +17,17 @@ export const paramDef = {
} as const; } as const;
export default define(meta, paramDef, async (ps) => { export default define(meta, paramDef, async (ps) => {
return await usersChart.getChart( const zeros = new Array<number>(ps.limit ?? 30).fill(0);
ps.span, return {
ps.limit, local: {
ps.offset ? new Date(ps.offset) : null, total: zeros,
); inc: zeros,
dec: zeros,
},
remote: {
total: zeros,
inc: zeros,
dec: zeros,
},
};
}); });

View file

@ -1,6 +1,5 @@
import { Instances, NoteReactions, Notes, Users } from "@/models/index.js"; import { Instances, Notes, Users } from "@/models/index.js";
import define from "../define.js"; import define from "../define.js";
import { driveChart, notesChart, usersChart } from "@/services/chart/index.js";
import { IsNull } from "typeorm"; import { IsNull } from "typeorm";
export const meta = { export const meta = {
@ -9,6 +8,8 @@ export const meta = {
tags: ["meta"], tags: ["meta"],
cacheSec: 300,
res: { res: {
type: "object", type: "object",
optional: false, optional: false,
@ -60,28 +61,40 @@ export const paramDef = {
} as const; } as const;
export default define(meta, paramDef, async () => { export default define(meta, paramDef, async () => {
const notesChartData = await notesChart.getChart("hour", 1, null);
const notesCount =
notesChartData.local.total[0] + notesChartData.remote.total[0];
const originalNotesCount = notesChartData.local.total[0];
const usersChartData = await usersChart.getChart("hour", 1, null);
const usersCount =
usersChartData.local.total[0] + usersChartData.remote.total[0];
const originalUsersCount = usersChartData.local.total[0];
const driveChartData = await driveChart.getChart("hour", 1, null);
//TODO: fixme currently returns 0
const driveUsageLocal = driveChartData.local.incSize[0];
const driveUsageRemote = driveChartData.remote.incSize[0];
const [ const [
notesCount,
originalNotesCount,
usersCount,
originalUsersCount,
reactionsCount, reactionsCount,
//originalReactionsCount,
instances, instances,
driveUsageLocal,
driveUsageRemote,
] = await Promise.all([ ] = await Promise.all([
NoteReactions.count({ cache: 3600000 }), // 1 hour // notesCount
//NoteReactions.count({ where: { userHost: IsNull() }, cache: 3600000 }), Notes.count(),
Instances.count({ cache: 3600000 }), // originalNotesCount
Notes.count({
where: {
userHost: IsNull(),
},
}),
// usersCount
Users.count(),
// originalUsersCount
Users.count({
where: {
host: IsNull(),
},
}),
// reactionsCount
0,
// instances
Instances.count(),
// driveUsageLocal
0,
// driveUsageRemote
0,
]); ]);
return { return {
@ -90,7 +103,6 @@ export default define(meta, paramDef, async () => {
usersCount, usersCount,
originalUsersCount, originalUsersCount,
reactionsCount, reactionsCount,
//originalReactionsCount,
instances, instances,
driveUsageLocal, driveUsageLocal,
driveUsageRemote, driveUsageRemote,

View file

@ -15,7 +15,6 @@ import {
UserListJoinings, UserListJoinings,
UserLists, UserLists,
} from "@/models/index.js"; } from "@/models/index.js";
import { perUserFollowingChart } from "@/services/chart/index.js";
import { genId } from "@/misc/gen-id.js"; import { genId } from "@/misc/gen-id.js";
import { IdentifiableError } from "@/misc/identifiable-error.js"; import { IdentifiableError } from "@/misc/identifiable-error.js";
import { getActiveWebhooks } from "@/misc/webhook-cache.js"; import { getActiveWebhooks } from "@/misc/webhook-cache.js";
@ -120,7 +119,6 @@ async function unFollow(follower: User, followee: User) {
Followings.delete(following.id), Followings.delete(following.id),
Users.decrement({ id: follower.id }, "followingCount", 1), Users.decrement({ id: follower.id }, "followingCount", 1),
Users.decrement({ id: followee.id }, "followersCount", 1), Users.decrement({ id: followee.id }, "followersCount", 1),
perUserFollowingChart.update(follower, followee, false),
]); ]);
// Publish unfollow event // Publish unfollow event

View file

@ -1,39 +0,0 @@
import type { KVs } from "../core.js";
import Chart from "../core.js";
import { name, schema } from "./entities/ap-request.js";
/**
* Chart about ActivityPub requests
*/
export default class ApRequestChart extends Chart<typeof schema> {
constructor() {
super(name, schema);
}
protected async tickMajor(): Promise<Partial<KVs<typeof schema>>> {
return {};
}
protected async tickMinor(): Promise<Partial<KVs<typeof schema>>> {
return {};
}
public async deliverSucc(): Promise<void> {
await this.commit({
deliverSucceeded: 1,
});
}
public async deliverFail(): Promise<void> {
await this.commit({
deliverFailed: 1,
});
}
public async inbox(): Promise<void> {
await this.commit({
inboxReceived: 1,
});
}
}

View file

@ -1,43 +0,0 @@
import type { KVs } from "../core.js";
import Chart from "../core.js";
import { DriveFiles } from "@/models/index.js";
import { Not, IsNull } from "typeorm";
import type { DriveFile } from "@/models/entities/drive-file.js";
import { name, schema } from "./entities/drive.js";
/**
*
*/
export default class DriveChart extends Chart<typeof schema> {
constructor() {
super(name, schema);
}
protected async tickMajor(): Promise<Partial<KVs<typeof schema>>> {
return {};
}
protected async tickMinor(): Promise<Partial<KVs<typeof schema>>> {
return {};
}
public async update(file: DriveFile, isAdditional: boolean): Promise<void> {
const fileSizeKb = file.size / 1000;
await this.commit(
file.userHost === null
? {
"local.incCount": isAdditional ? 1 : 0,
"local.incSize": isAdditional ? fileSizeKb : 0,
"local.decCount": isAdditional ? 0 : 1,
"local.decSize": isAdditional ? 0 : fileSizeKb,
}
: {
"remote.incCount": isAdditional ? 1 : 0,
"remote.incSize": isAdditional ? fileSizeKb : 0,
"remote.decCount": isAdditional ? 0 : 1,
"remote.decSize": isAdditional ? 0 : fileSizeKb,
},
);
}
}

View file

@ -1,11 +0,0 @@
import Chart from "../../core.js";
export const name = "apRequest";
export const schema = {
deliverFailed: {},
deliverSucceeded: {},
inboxReceived: {},
} as const;
export const entity = Chart.schemaToEntity(name, schema);

View file

@ -1,16 +0,0 @@
import Chart from "../../core.js";
export const name = "drive";
export const schema = {
"local.incCount": {},
"local.incSize": {}, // in kilobyte
"local.decCount": {},
"local.decSize": {}, // in kilobyte
"remote.incCount": {},
"remote.incSize": {}, // in kilobyte
"remote.decCount": {},
"remote.decSize": {}, // in kilobyte
} as const;
export const entity = Chart.schemaToEntity(name, schema);

View file

@ -1,16 +0,0 @@
import Chart from "../../core.js";
export const name = "federation";
export const schema = {
deliveredInstances: { uniqueIncrement: true, range: "small" },
inboxInstances: { uniqueIncrement: true, range: "small" },
stalled: { uniqueIncrement: true, range: "small" },
sub: { accumulate: true, range: "small" },
pub: { accumulate: true, range: "small" },
pubsub: { accumulate: true, range: "small" },
subActive: { accumulate: true, range: "small" },
pubActive: { accumulate: true, range: "small" },
} as const;
export const entity = Chart.schemaToEntity(name, schema);

View file

@ -1,10 +0,0 @@
import Chart from "../../core.js";
export const name = "hashtag";
export const schema = {
"local.users": { uniqueIncrement: true },
"remote.users": { uniqueIncrement: true },
} as const;
export const entity = Chart.schemaToEntity(name, schema, true);

View file

@ -1,32 +0,0 @@
import Chart from "../../core.js";
export const name = "instance";
export const schema = {
"requests.failed": { range: "small" },
"requests.succeeded": { range: "small" },
"requests.received": { range: "small" },
"notes.total": { accumulate: true },
"notes.inc": {},
"notes.dec": {},
"notes.diffs.normal": {},
"notes.diffs.reply": {},
"notes.diffs.renote": {},
"notes.diffs.withFile": {},
"users.total": { accumulate: true },
"users.inc": { range: "small" },
"users.dec": { range: "small" },
"following.total": { accumulate: true },
"following.inc": { range: "small" },
"following.dec": { range: "small" },
"followers.total": { accumulate: true },
"followers.inc": { range: "small" },
"followers.dec": { range: "small" },
"drive.totalFiles": { accumulate: true },
"drive.incFiles": {},
"drive.decFiles": {},
"drive.incUsage": {}, // in kilobyte
"drive.decUsage": {}, // in kilobyte
} as const;
export const entity = Chart.schemaToEntity(name, schema, true);

View file

@ -1,22 +0,0 @@
import Chart from "../../core.js";
export const name = "notes";
export const schema = {
"local.total": { accumulate: true },
"local.inc": {},
"local.dec": {},
"local.diffs.normal": {},
"local.diffs.reply": {},
"local.diffs.renote": {},
"local.diffs.withFile": {},
"remote.total": { accumulate: true },
"remote.inc": {},
"remote.dec": {},
"remote.diffs.normal": {},
"remote.diffs.reply": {},
"remote.diffs.renote": {},
"remote.diffs.withFile": {},
} as const;
export const entity = Chart.schemaToEntity(name, schema);

View file

@ -1,14 +0,0 @@
import Chart from "../../core.js";
export const name = "perUserDrive";
export const schema = {
totalCount: { accumulate: true },
totalSize: { accumulate: true }, // in kilobyte
incCount: { range: "small" },
incSize: {}, // in kilobyte
decCount: { range: "small" },
decSize: {}, // in kilobyte
} as const;
export const entity = Chart.schemaToEntity(name, schema, true);

View file

@ -1,20 +0,0 @@
import Chart from "../../core.js";
export const name = "perUserFollowing";
export const schema = {
"local.followings.total": { accumulate: true },
"local.followings.inc": { range: "small" },
"local.followings.dec": { range: "small" },
"local.followers.total": { accumulate: true },
"local.followers.inc": { range: "small" },
"local.followers.dec": { range: "small" },
"remote.followings.total": { accumulate: true },
"remote.followings.inc": { range: "small" },
"remote.followings.dec": { range: "small" },
"remote.followers.total": { accumulate: true },
"remote.followers.inc": { range: "small" },
"remote.followers.dec": { range: "small" },
} as const;
export const entity = Chart.schemaToEntity(name, schema, true);

View file

@ -1,15 +0,0 @@
import Chart from "../../core.js";
export const name = "perUserNotes";
export const schema = {
total: { accumulate: true },
inc: { range: "small" },
dec: { range: "small" },
"diffs.normal": { range: "small" },
"diffs.reply": { range: "small" },
"diffs.renote": { range: "small" },
"diffs.withFile": { range: "small" },
} as const;
export const entity = Chart.schemaToEntity(name, schema, true);

View file

@ -1,10 +0,0 @@
import Chart from "../../core.js";
export const name = "perUserReaction";
export const schema = {
"local.count": { range: "small" },
"remote.count": { range: "small" },
} as const;
export const entity = Chart.schemaToEntity(name, schema, true);

View file

@ -1,11 +0,0 @@
import Chart from "../../core.js";
export const name = "testGrouped";
export const schema = {
"foo.total": { accumulate: true },
"foo.inc": {},
"foo.dec": {},
} as const;
export const entity = Chart.schemaToEntity(name, schema, true);

View file

@ -1,11 +0,0 @@
import Chart from "../../core.js";
export const name = "testIntersection";
export const schema = {
a: { uniqueIncrement: true },
b: { uniqueIncrement: true },
aAndB: { intersection: ["a", "b"] },
} as const;
export const entity = Chart.schemaToEntity(name, schema);

View file

@ -1,9 +0,0 @@
import Chart from "../../core.js";
export const name = "testUnique";
export const schema = {
foo: { uniqueIncrement: true },
} as const;
export const entity = Chart.schemaToEntity(name, schema);

View file

@ -1,11 +0,0 @@
import Chart from "../../core.js";
export const name = "test";
export const schema = {
"foo.total": { accumulate: true },
"foo.inc": {},
"foo.dec": {},
} as const;
export const entity = Chart.schemaToEntity(name, schema);

View file

@ -1,14 +0,0 @@
import Chart from "../../core.js";
export const name = "users";
export const schema = {
"local.total": { accumulate: true },
"local.inc": { range: "small" },
"local.dec": { range: "small" },
"remote.total": { accumulate: true },
"remote.inc": { range: "small" },
"remote.dec": { range: "small" },
} as const;
export const entity = Chart.schemaToEntity(name, schema);

View file

@ -1,142 +0,0 @@
import type { KVs } from "../core.js";
import Chart from "../core.js";
import { Followings, Instances } from "@/models/index.js";
import { name, schema } from "./entities/federation.js";
import { fetchMeta } from "@/misc/fetch-meta.js";
/**
*
*/
export default class FederationChart extends Chart<typeof schema> {
constructor() {
super(name, schema);
}
protected async tickMajor(): Promise<Partial<KVs<typeof schema>>> {
return {};
}
protected async tickMinor(): Promise<Partial<KVs<typeof schema>>> {
const meta = await fetchMeta();
const suspendedInstancesQuery = Instances.createQueryBuilder("instance")
.select("instance.host")
.where("instance.isSuspended = true");
const pubsubSubQuery = Followings.createQueryBuilder("f")
.select("f.followerHost")
.where("f.followerHost IS NOT NULL");
const subInstancesQuery = Followings.createQueryBuilder("f")
.select("f.followeeHost")
.where("f.followeeHost IS NOT NULL");
const pubInstancesQuery = Followings.createQueryBuilder("f")
.select("f.followerHost")
.where("f.followerHost IS NOT NULL");
const [sub, pub, pubsub, subActive, pubActive] = await Promise.all([
Followings.createQueryBuilder("following")
.select("COUNT(DISTINCT following.followeeHost)")
.where("following.followeeHost IS NOT NULL")
.andWhere(
meta.blockedHosts.length === 0
? "1=1"
: "following.followeeHost NOT IN (:...blocked)",
{ blocked: meta.blockedHosts },
)
.andWhere(
`following.followeeHost NOT IN (${suspendedInstancesQuery.getQuery()})`,
)
.getRawOne()
.then((x) => parseInt(x.count, 10)),
Followings.createQueryBuilder("following")
.select("COUNT(DISTINCT following.followerHost)")
.where("following.followerHost IS NOT NULL")
.andWhere(
meta.blockedHosts.length === 0
? "1=1"
: "following.followerHost NOT IN (:...blocked)",
{ blocked: meta.blockedHosts },
)
.andWhere(
`following.followerHost NOT IN (${suspendedInstancesQuery.getQuery()})`,
)
.getRawOne()
.then((x) => parseInt(x.count, 10)),
Followings.createQueryBuilder("following")
.select("COUNT(DISTINCT following.followeeHost)")
.where("following.followeeHost IS NOT NULL")
.andWhere(
meta.blockedHosts.length === 0
? "1=1"
: "following.followeeHost NOT IN (:...blocked)",
{ blocked: meta.blockedHosts },
)
.andWhere(
`following.followeeHost NOT IN (${suspendedInstancesQuery.getQuery()})`,
)
.andWhere(`following.followeeHost IN (${pubsubSubQuery.getQuery()})`)
.setParameters(pubsubSubQuery.getParameters())
.getRawOne()
.then((x) => parseInt(x.count, 10)),
Instances.createQueryBuilder("instance")
.select("COUNT(instance.id)")
.where(`instance.host IN (${subInstancesQuery.getQuery()})`)
.andWhere(
meta.blockedHosts.length === 0
? "1=1"
: "instance.host NOT IN (:...blocked)",
{ blocked: meta.blockedHosts },
)
.andWhere("instance.isSuspended = false")
.andWhere("instance.lastCommunicatedAt > :gt", {
gt: new Date(Date.now() - 1000 * 60 * 60 * 24 * 30),
})
.getRawOne()
.then((x) => parseInt(x.count, 10)),
Instances.createQueryBuilder("instance")
.select("COUNT(instance.id)")
.where(`instance.host IN (${pubInstancesQuery.getQuery()})`)
.andWhere(
meta.blockedHosts.length === 0
? "1=1"
: "instance.host NOT IN (:...blocked)",
{ blocked: meta.blockedHosts },
)
.andWhere("instance.isSuspended = false")
.andWhere("instance.lastCommunicatedAt > :gt", {
gt: new Date(Date.now() - 1000 * 60 * 60 * 24 * 30),
})
.getRawOne()
.then((x) => parseInt(x.count, 10)),
]);
return {
sub: sub,
pub: pub,
pubsub: pubsub,
subActive: subActive,
pubActive: pubActive,
};
}
public async deliverd(host: string, succeeded: boolean): Promise<void> {
await this.commit(
succeeded
? {
deliveredInstances: [host],
}
: {
stalled: [host],
},
);
}
public async inbox(host: string): Promise<void> {
await this.commit({
inboxInstances: [host],
});
}
}

View file

@ -1,36 +0,0 @@
import type { KVs } from "../core.js";
import Chart from "../core.js";
import type { User } from "@/models/entities/user.js";
import { Users } from "@/models/index.js";
import { name, schema } from "./entities/hashtag.js";
/**
*
*/
export default class HashtagChart extends Chart<typeof schema> {
constructor() {
super(name, schema, true);
}
protected async tickMajor(): Promise<Partial<KVs<typeof schema>>> {
return {};
}
protected async tickMinor(): Promise<Partial<KVs<typeof schema>>> {
return {};
}
public async update(
hashtag: string,
user: { id: User["id"]; host: User["host"] },
): Promise<void> {
await this.commit(
{
"local.users": Users.isLocalUser(user) ? [user.id] : [],
"remote.users": Users.isLocalUser(user) ? [] : [user.id],
},
hashtag,
);
}
}

View file

@ -1,142 +0,0 @@
import type { KVs } from "../core.js";
import Chart from "../core.js";
import { DriveFiles, Followings, Users, Notes } from "@/models/index.js";
import type { DriveFile } from "@/models/entities/drive-file.js";
import type { Note } from "@/models/entities/note.js";
import { toPuny } from "@/misc/convert-host.js";
import { name, schema } from "./entities/instance.js";
/**
*
*/
export default class InstanceChart extends Chart<typeof schema> {
constructor() {
super(name, schema, true);
}
protected async tickMajor(
group: string,
): Promise<Partial<KVs<typeof schema>>> {
const [notesCount, usersCount, followingCount, followersCount, driveFiles] =
await Promise.all([
Notes.countBy({ userHost: group }),
Users.countBy({ host: group }),
Followings.countBy({ followerHost: group }),
Followings.countBy({ followeeHost: group }),
DriveFiles.countBy({ userHost: group }),
]);
return {
"notes.total": notesCount,
"users.total": usersCount,
"following.total": followingCount,
"followers.total": followersCount,
"drive.totalFiles": driveFiles,
};
}
protected async tickMinor(): Promise<Partial<KVs<typeof schema>>> {
return {};
}
public async requestReceived(host: string): Promise<void> {
await this.commit(
{
"requests.received": 1,
},
toPuny(host),
);
}
public async requestSent(host: string, isSucceeded: boolean): Promise<void> {
await this.commit(
{
"requests.succeeded": isSucceeded ? 1 : 0,
"requests.failed": isSucceeded ? 0 : 1,
},
toPuny(host),
);
}
public async newUser(host: string): Promise<void> {
await this.commit(
{
"users.total": 1,
"users.inc": 1,
},
toPuny(host),
);
}
public async updateNote(
host: string,
note: Note,
isAdditional: boolean,
): Promise<void> {
await this.commit(
{
"notes.total": isAdditional ? 1 : -1,
"notes.inc": isAdditional ? 1 : 0,
"notes.dec": isAdditional ? 0 : 1,
"notes.diffs.normal":
note.replyId == null && note.renoteId == null
? isAdditional
? 1
: -1
: 0,
"notes.diffs.renote":
note.renoteId != null ? (isAdditional ? 1 : -1) : 0,
"notes.diffs.reply": note.replyId != null ? (isAdditional ? 1 : -1) : 0,
"notes.diffs.withFile":
note.fileIds.length > 0 ? (isAdditional ? 1 : -1) : 0,
},
toPuny(host),
);
}
public async updateFollowing(
host: string,
isAdditional: boolean,
): Promise<void> {
await this.commit(
{
"following.total": isAdditional ? 1 : -1,
"following.inc": isAdditional ? 1 : 0,
"following.dec": isAdditional ? 0 : 1,
},
toPuny(host),
);
}
public async updateFollowers(
host: string,
isAdditional: boolean,
): Promise<void> {
await this.commit(
{
"followers.total": isAdditional ? 1 : -1,
"followers.inc": isAdditional ? 1 : 0,
"followers.dec": isAdditional ? 0 : 1,
},
toPuny(host),
);
}
public async updateDrive(
file: DriveFile,
isAdditional: boolean,
): Promise<void> {
const fileSizeKb = file.size / 1000;
await this.commit(
{
"drive.totalFiles": isAdditional ? 1 : -1,
"drive.incFiles": isAdditional ? 1 : 0,
"drive.incUsage": isAdditional ? fileSizeKb : 0,
"drive.decFiles": isAdditional ? 1 : 0,
"drive.decUsage": isAdditional ? fileSizeKb : 0,
},
file.userHost,
);
}
}

View file

@ -1,58 +0,0 @@
import type { KVs } from "../core.js";
import Chart from "../core.js";
import { Notes } from "@/models/index.js";
import { Not, IsNull } from "typeorm";
import type { Note } from "@/models/entities/note.js";
import { name, schema } from "./entities/notes.js";
/**
*
*/
export default class NotesChart extends Chart<typeof schema> {
constructor() {
super(name, schema);
}
protected async tickMajor(): Promise<Partial<KVs<typeof schema>>> {
const [localCount, remoteCount] = await Promise.all([
Notes.countBy({ userHost: IsNull() }),
Notes.countBy({ userHost: Not(IsNull()) }),
]);
return {
"local.total": localCount,
"remote.total": remoteCount,
};
}
protected async tickMinor(): Promise<Partial<KVs<typeof schema>>> {
return {};
}
public async update(
note: Note,
isAdditional: boolean,
byBot = false,
): Promise<void> {
const prefix = note.userHost === null ? "local" : "remote";
await this.commit({
[`${prefix}.total`]: isAdditional ? 1 : -1,
[`${prefix}.inc`]: isAdditional ? 1 : 0,
[`${prefix}.dec`]: isAdditional ? 0 : 1,
[`${prefix}.diffs.normal`]:
note.replyId == null && note.renoteId == null
? isAdditional
? 1
: -1
: 0,
[`${prefix}.diffs.renote`]:
note.renoteId != null && !byBot ? (isAdditional ? 1 : -1) : 0,
[`${prefix}.diffs.reply`]:
note.replyId != null ? (isAdditional ? 1 : -1) : 0,
[`${prefix}.diffs.withFile`]:
note.fileIds.length > 0 ? (isAdditional ? 1 : -1) : 0,
});
}
}

View file

@ -1,48 +0,0 @@
import type { KVs } from "../core.js";
import Chart from "../core.js";
import { DriveFiles } from "@/models/index.js";
import type { DriveFile } from "@/models/entities/drive-file.js";
import { name, schema } from "./entities/per-user-drive.js";
/**
*
*/
export default class PerUserDriveChart extends Chart<typeof schema> {
constructor() {
super(name, schema, true);
}
protected async tickMajor(
group: string,
): Promise<Partial<KVs<typeof schema>>> {
const [count, size] = await Promise.all([
DriveFiles.countBy({ userId: group }),
DriveFiles.calcDriveUsageOf(group),
]);
return {
totalCount: count,
totalSize: size,
};
}
protected async tickMinor(): Promise<Partial<KVs<typeof schema>>> {
return {};
}
public async update(file: DriveFile, isAdditional: boolean): Promise<void> {
const fileSizeKb = file.size / 1000;
await this.commit(
{
totalCount: isAdditional ? 1 : -1,
totalSize: isAdditional ? fileSizeKb : -fileSizeKb,
incCount: isAdditional ? 1 : 0,
incSize: isAdditional ? fileSizeKb : 0,
decCount: isAdditional ? 0 : 1,
decSize: isAdditional ? 0 : fileSizeKb,
},
file.userId,
);
}
}

View file

@ -1,69 +0,0 @@
import type { KVs } from "../core.js";
import Chart from "../core.js";
import { Followings, Users } from "@/models/index.js";
import { Not, IsNull } from "typeorm";
import type { User } from "@/models/entities/user.js";
import { name, schema } from "./entities/per-user-following.js";
/**
*
*/
export default class PerUserFollowingChart extends Chart<typeof schema> {
constructor() {
super(name, schema, true);
}
protected async tickMajor(
group: string,
): Promise<Partial<KVs<typeof schema>>> {
const [
localFollowingsCount,
localFollowersCount,
remoteFollowingsCount,
remoteFollowersCount,
] = await Promise.all([
Followings.countBy({ followerId: group, followeeHost: IsNull() }),
Followings.countBy({ followeeId: group, followerHost: IsNull() }),
Followings.countBy({ followerId: group, followeeHost: Not(IsNull()) }),
Followings.countBy({ followeeId: group, followerHost: Not(IsNull()) }),
]);
return {
"local.followings.total": localFollowingsCount,
"local.followers.total": localFollowersCount,
"remote.followings.total": remoteFollowingsCount,
"remote.followers.total": remoteFollowersCount,
};
}
protected async tickMinor(): Promise<Partial<KVs<typeof schema>>> {
return {};
}
public async update(
follower: { id: User["id"]; host: User["host"] },
followee: { id: User["id"]; host: User["host"] },
isFollow: boolean,
): Promise<void> {
const prefixFollower = Users.isLocalUser(follower) ? "local" : "remote";
const prefixFollowee = Users.isLocalUser(followee) ? "local" : "remote";
this.commit(
{
[`${prefixFollower}.followings.total`]: isFollow ? 1 : -1,
[`${prefixFollower}.followings.inc`]: isFollow ? 1 : 0,
[`${prefixFollower}.followings.dec`]: isFollow ? 0 : 1,
},
follower.id,
);
this.commit(
{
[`${prefixFollowee}.followers.total`]: isFollow ? 1 : -1,
[`${prefixFollowee}.followers.inc`]: isFollow ? 1 : 0,
[`${prefixFollowee}.followers.dec`]: isFollow ? 0 : 1,
},
followee.id,
);
}
}

View file

@ -1,56 +0,0 @@
import type { KVs } from "../core.js";
import Chart from "../core.js";
import type { User } from "@/models/entities/user.js";
import { Notes } from "@/models/index.js";
import type { Note } from "@/models/entities/note.js";
import { name, schema } from "./entities/per-user-notes.js";
/**
*
*/
export default class PerUserNotesChart extends Chart<typeof schema> {
constructor() {
super(name, schema, true);
}
protected async tickMajor(
group: string,
): Promise<Partial<KVs<typeof schema>>> {
const [count] = await Promise.all([Notes.countBy({ userId: group })]);
return {
total: count,
};
}
protected async tickMinor(): Promise<Partial<KVs<typeof schema>>> {
return {};
}
public async update(
user: { id: User["id"] },
note: Note,
isAdditional: boolean,
byBot = false,
): Promise<void> {
await this.commit(
{
total: isAdditional ? 1 : -1,
inc: isAdditional ? 1 : 0,
dec: isAdditional ? 0 : 1,
"diffs.normal":
note.replyId == null && note.renoteId == null
? isAdditional
? 1
: -1
: 0,
"diffs.renote":
note.renoteId != null && !byBot ? (isAdditional ? 1 : -1) : 0,
"diffs.reply": note.replyId != null ? (isAdditional ? 1 : -1) : 0,
"diffs.withFile": note.fileIds.length > 0 ? (isAdditional ? 1 : -1) : 0,
},
user.id,
);
}
}

View file

@ -1,39 +0,0 @@
import type { KVs } from "../core.js";
import Chart from "../core.js";
import type { User } from "@/models/entities/user.js";
import type { Note } from "@/models/entities/note.js";
import { Users } from "@/models/index.js";
import { name, schema } from "./entities/per-user-reactions.js";
/**
*
*/
export default class PerUserReactionsChart extends Chart<typeof schema> {
constructor() {
super(name, schema, true);
}
protected async tickMajor(
group: string,
): Promise<Partial<KVs<typeof schema>>> {
return {};
}
protected async tickMinor(): Promise<Partial<KVs<typeof schema>>> {
return {};
}
public async update(
user: { id: User["id"]; host: User["host"] },
note: Note,
): Promise<void> {
const prefix = Users.isLocalUser(user) ? "local" : "remote";
this.commit(
{
[`${prefix}.count`]: 1,
},
note.userId,
);
}
}

View file

@ -1,41 +0,0 @@
import type { KVs } from "../core.js";
import Chart from "../core.js";
import { name, schema } from "./entities/test-grouped.js";
/**
* For testing
*/
export default class TestGroupedChart extends Chart<typeof schema> {
private total = {} as Record<string, number>;
constructor() {
super(name, schema, true);
}
protected async tickMajor(
group: string,
): Promise<Partial<KVs<typeof schema>>> {
return {
"foo.total": this.total[group],
};
}
protected async tickMinor(): Promise<Partial<KVs<typeof schema>>> {
return {};
}
public async increment(group: string): Promise<void> {
if (this.total[group] == null) this.total[group] = 0;
this.total[group]++;
await this.commit(
{
"foo.total": 1,
"foo.inc": 1,
},
group,
);
}
}

View file

@ -1,33 +0,0 @@
import type { KVs } from "../core.js";
import Chart from "../core.js";
import { name, schema } from "./entities/test-intersection.js";
/**
* For testing
*/
export default class TestIntersectionChart extends Chart<typeof schema> {
constructor() {
super(name, schema);
}
protected async tickMajor(): Promise<Partial<KVs<typeof schema>>> {
return {};
}
protected async tickMinor(): Promise<Partial<KVs<typeof schema>>> {
return {};
}
public async addA(key: string): Promise<void> {
await this.commit({
a: [key],
});
}
public async addB(key: string): Promise<void> {
await this.commit({
b: [key],
});
}
}

View file

@ -1,27 +0,0 @@
import type { KVs } from "../core.js";
import Chart from "../core.js";
import { name, schema } from "./entities/test-unique.js";
/**
* For testing
*/
export default class TestUniqueChart extends Chart<typeof schema> {
constructor() {
super(name, schema);
}
protected async tickMajor(): Promise<Partial<KVs<typeof schema>>> {
return {};
}
protected async tickMinor(): Promise<Partial<KVs<typeof schema>>> {
return {};
}
public async uniqueIncrement(key: string): Promise<void> {
await this.commit({
foo: [key],
});
}
}

View file

@ -1,43 +0,0 @@
import type { KVs } from "../core.js";
import Chart from "../core.js";
import { name, schema } from "./entities/test.js";
/**
* For testing
*/
export default class TestChart extends Chart<typeof schema> {
public total = 0; // publicにするのはテストのため
constructor() {
super(name, schema);
}
protected async tickMajor(): Promise<Partial<KVs<typeof schema>>> {
return {
"foo.total": this.total,
};
}
protected async tickMinor(): Promise<Partial<KVs<typeof schema>>> {
return {};
}
public async increment(): Promise<void> {
this.total++;
await this.commit({
"foo.total": 1,
"foo.inc": 1,
});
}
public async decrement(): Promise<void> {
this.total--;
await this.commit({
"foo.total": -1,
"foo.dec": 1,
});
}
}

View file

@ -1,45 +0,0 @@
import type { KVs } from "../core.js";
import Chart from "../core.js";
import { Users } from "@/models/index.js";
import { Not, IsNull } from "typeorm";
import type { User } from "@/models/entities/user.js";
import { name, schema } from "./entities/users.js";
/**
*
*/
export default class UsersChart extends Chart<typeof schema> {
constructor() {
super(name, schema);
}
protected async tickMajor(): Promise<Partial<KVs<typeof schema>>> {
const [localCount, remoteCount] = await Promise.all([
Users.countBy({ host: IsNull() }),
Users.countBy({ host: Not(IsNull()) }),
]);
return {
"local.total": localCount,
"remote.total": remoteCount,
};
}
protected async tickMinor(): Promise<Partial<KVs<typeof schema>>> {
return {};
}
public async update(
user: { id: User["id"]; host: User["host"] },
isAdditional: boolean,
): Promise<void> {
const prefix = Users.isLocalUser(user) ? "local" : "remote";
await this.commit({
[`${prefix}.total`]: isAdditional ? 1 : -1,
[`${prefix}.inc`]: isAdditional ? 1 : 0,
[`${prefix}.dec`]: isAdditional ? 0 : 1,
});
}
}

View file

@ -1,57 +1,3 @@
import { entity as FederationChart } from "./charts/entities/federation.js";
import { entity as NotesChart } from "./charts/entities/notes.js";
import { entity as UsersChart } from "./charts/entities/users.js";
import { entity as ActiveUsersChart } from "./charts/entities/active-users.js"; import { entity as ActiveUsersChart } from "./charts/entities/active-users.js";
import { entity as InstanceChart } from "./charts/entities/instance.js";
import { entity as PerUserNotesChart } from "./charts/entities/per-user-notes.js";
import { entity as DriveChart } from "./charts/entities/drive.js";
import { entity as PerUserReactionsChart } from "./charts/entities/per-user-reactions.js";
import { entity as HashtagChart } from "./charts/entities/hashtag.js";
import { entity as PerUserFollowingChart } from "./charts/entities/per-user-following.js";
import { entity as PerUserDriveChart } from "./charts/entities/per-user-drive.js";
import { entity as ApRequestChart } from "./charts/entities/ap-request.js";
import { entity as TestChart } from "./charts/entities/test.js"; export const entities = [ActiveUsersChart.hour, ActiveUsersChart.day];
import { entity as TestGroupedChart } from "./charts/entities/test-grouped.js";
import { entity as TestUniqueChart } from "./charts/entities/test-unique.js";
import { entity as TestIntersectionChart } from "./charts/entities/test-intersection.js";
export const entities = [
FederationChart.hour,
FederationChart.day,
NotesChart.hour,
NotesChart.day,
UsersChart.hour,
UsersChart.day,
ActiveUsersChart.hour,
ActiveUsersChart.day,
InstanceChart.hour,
InstanceChart.day,
PerUserNotesChart.hour,
PerUserNotesChart.day,
DriveChart.hour,
DriveChart.day,
PerUserReactionsChart.hour,
PerUserReactionsChart.day,
HashtagChart.hour,
HashtagChart.day,
PerUserFollowingChart.hour,
PerUserFollowingChart.day,
PerUserDriveChart.hour,
PerUserDriveChart.day,
ApRequestChart.hour,
ApRequestChart.day,
...(process.env.NODE_ENV === "test"
? [
TestChart.hour,
TestChart.day,
TestGroupedChart.hour,
TestGroupedChart.day,
TestUniqueChart.hour,
TestUniqueChart.day,
TestIntersectionChart.hour,
TestIntersectionChart.day,
]
: []),
];

View file

@ -1,51 +1,9 @@
import { beforeShutdown } from "@/misc/before-shutdown.js"; import { beforeShutdown } from "@/misc/before-shutdown.js";
import FederationChart from "./charts/federation.js";
import NotesChart from "./charts/notes.js";
import UsersChart from "./charts/users.js";
import ActiveUsersChart from "./charts/active-users.js"; import ActiveUsersChart from "./charts/active-users.js";
import InstanceChart from "./charts/instance.js";
import PerUserNotesChart from "./charts/per-user-notes.js";
import DriveChart from "./charts/drive.js";
import PerUserReactionsChart from "./charts/per-user-reactions.js";
import HashtagChart from "./charts/hashtag.js";
import PerUserFollowingChart from "./charts/per-user-following.js";
import PerUserDriveChart from "./charts/per-user-drive.js";
import ApRequestChart from "./charts/ap-request.js";
export const federationChart = new FederationChart();
export const notesChart = new NotesChart();
export const usersChart = new UsersChart();
export const activeUsersChart = new ActiveUsersChart(); export const activeUsersChart = new ActiveUsersChart();
export const instanceChart = new InstanceChart();
export const perUserNotesChart = new PerUserNotesChart();
export const driveChart = new DriveChart();
export const perUserReactionsChart = new PerUserReactionsChart();
export const hashtagChart = new HashtagChart();
export const perUserFollowingChart = new PerUserFollowingChart();
export const perUserDriveChart = new PerUserDriveChart();
export const apRequestChart = new ApRequestChart();
const charts = [
federationChart,
notesChart,
usersChart,
activeUsersChart,
instanceChart,
perUserNotesChart,
driveChart,
perUserReactionsChart,
hashtagChart,
perUserFollowingChart,
perUserDriveChart,
apRequestChart,
];
// 20分おきにメモリ情報をDBに書き込み // 20分おきにメモリ情報をDBに書き込み
setInterval(() => { setInterval(() => activeUsersChart.save(), 1000 * 60 * 20);
for (const chart of charts) {
chart.save();
}
}, 1000 * 60 * 20);
beforeShutdown(() => Promise.all(charts.map((chart) => chart.save()))); beforeShutdown(async () => await activeUsersChars.save());

View file

@ -18,11 +18,6 @@ import {
} from "@/models/index.js"; } from "@/models/index.js";
import { DriveFile } from "@/models/entities/drive-file.js"; import { DriveFile } from "@/models/entities/drive-file.js";
import type { IRemoteUser, User } from "@/models/entities/user.js"; import type { IRemoteUser, User } from "@/models/entities/user.js";
import {
driveChart,
perUserDriveChart,
instanceChart,
} from "@/services/chart/index.js";
import { genId } from "@/misc/gen-id.js"; import { genId } from "@/misc/gen-id.js";
import { isDuplicateKeyValueError } from "@/misc/is-duplicate-key-value-error.js"; import { isDuplicateKeyValueError } from "@/misc/is-duplicate-key-value-error.js";
import { FILE_TYPE_BROWSERSAFE } from "@/const.js"; import { FILE_TYPE_BROWSERSAFE } from "@/const.js";
@ -698,12 +693,5 @@ export async function addFile({
}); });
} }
// 統計を更新
driveChart.update(file, true);
perUserDriveChart.update(file, true);
if (file.userHost !== null) {
instanceChart.updateDrive(file, true);
}
return file; return file;
} }

View file

@ -1,11 +1,6 @@
import type { DriveFile } from "@/models/entities/drive-file.js"; import type { DriveFile } from "@/models/entities/drive-file.js";
import { InternalStorage } from "./internal-storage.js"; import { InternalStorage } from "./internal-storage.js";
import { DriveFiles, Instances } from "@/models/index.js"; import { DriveFiles, Instances } from "@/models/index.js";
import {
driveChart,
perUserDriveChart,
instanceChart,
} from "@/services/chart/index.js";
import { createDeleteObjectStorageFileJob } from "@/queue/index.js"; import { createDeleteObjectStorageFileJob } from "@/queue/index.js";
import { fetchMeta } from "@/misc/fetch-meta.js"; import { fetchMeta } from "@/misc/fetch-meta.js";
import { getS3 } from "./s3.js"; import { getS3 } from "./s3.js";
@ -84,13 +79,6 @@ async function postProcess(file: DriveFile, isExpired = false) {
} else { } else {
DriveFiles.delete(file.id); DriveFiles.delete(file.id);
} }
// 統計を更新
driveChart.update(file, false);
perUserDriveChart.update(file, false);
if (file.userHost !== null) {
instanceChart.updateDrive(file, false);
}
} }
export async function deleteObjectStorageFile(key: string) { export async function deleteObjectStorageFile(key: string) {

View file

@ -17,10 +17,6 @@ import {
Instances, Instances,
UserProfiles, UserProfiles,
} from "@/models/index.js"; } from "@/models/index.js";
import {
instanceChart,
perUserFollowingChart,
} from "@/services/chart/index.js";
import { genId } from "@/misc/gen-id.js"; import { genId } from "@/misc/gen-id.js";
import { createNotification } from "../create-notification.js"; import { createNotification } from "../create-notification.js";
import { isDuplicateKeyValueError } from "@/misc/is-duplicate-key-value-error.js"; import { isDuplicateKeyValueError } from "@/misc/is-duplicate-key-value-error.js";
@ -111,18 +107,14 @@ export async function insertFollowingDoc(
if (Users.isRemoteUser(follower) && Users.isLocalUser(followee)) { if (Users.isRemoteUser(follower) && Users.isLocalUser(followee)) {
registerOrFetchInstanceDoc(follower.host).then((i) => { registerOrFetchInstanceDoc(follower.host).then((i) => {
Instances.increment({ id: i.id }, "followingCount", 1); Instances.increment({ id: i.id }, "followingCount", 1);
instanceChart.updateFollowing(i.host, true);
}); });
} else if (Users.isLocalUser(follower) && Users.isRemoteUser(followee)) { } else if (Users.isLocalUser(follower) && Users.isRemoteUser(followee)) {
registerOrFetchInstanceDoc(followee.host).then((i) => { registerOrFetchInstanceDoc(followee.host).then((i) => {
Instances.increment({ id: i.id }, "followersCount", 1); Instances.increment({ id: i.id }, "followersCount", 1);
instanceChart.updateFollowers(i.host, true);
}); });
} }
//#endregion //#endregion
perUserFollowingChart.update(follower, followee, true);
// Publish follow event // Publish follow event
if (Users.isLocalUser(follower)) { if (Users.isLocalUser(follower)) {
Users.pack(followee.id, follower, { Users.pack(followee.id, follower, {

View file

@ -8,10 +8,6 @@ import Logger from "../logger.js";
import { registerOrFetchInstanceDoc } from "../register-or-fetch-instance-doc.js"; import { registerOrFetchInstanceDoc } from "../register-or-fetch-instance-doc.js";
import type { User } from "@/models/entities/user.js"; import type { User } from "@/models/entities/user.js";
import { Followings, Users, Instances } from "@/models/index.js"; import { Followings, Users, Instances } from "@/models/index.js";
import {
instanceChart,
perUserFollowingChart,
} from "@/services/chart/index.js";
import { getActiveWebhooks } from "@/misc/webhook-cache.js"; import { getActiveWebhooks } from "@/misc/webhook-cache.js";
const logger = new Logger("following/delete"); const logger = new Logger("following/delete");
@ -99,15 +95,11 @@ export async function decrementFollowing(
if (Users.isRemoteUser(follower) && Users.isLocalUser(followee)) { if (Users.isRemoteUser(follower) && Users.isLocalUser(followee)) {
registerOrFetchInstanceDoc(follower.host).then((i) => { registerOrFetchInstanceDoc(follower.host).then((i) => {
Instances.decrement({ id: i.id }, "followingCount", 1); Instances.decrement({ id: i.id }, "followingCount", 1);
instanceChart.updateFollowing(i.host, false);
}); });
} else if (Users.isLocalUser(follower) && Users.isRemoteUser(followee)) { } else if (Users.isLocalUser(follower) && Users.isRemoteUser(followee)) {
registerOrFetchInstanceDoc(followee.host).then((i) => { registerOrFetchInstanceDoc(followee.host).then((i) => {
Instances.decrement({ id: i.id }, "followersCount", 1); Instances.decrement({ id: i.id }, "followersCount", 1);
instanceChart.updateFollowers(i.host, false);
}); });
} }
//#endregion //#endregion
perUserFollowingChart.update(follower, followee, false);
} }

View file

@ -39,12 +39,6 @@ import type { App } from "@/models/entities/app.js";
import { Not, In } from "typeorm"; import { Not, In } from "typeorm";
import type { User, ILocalUser, IRemoteUser } from "@/models/entities/user.js"; import type { User, ILocalUser, IRemoteUser } from "@/models/entities/user.js";
import { genId } from "@/misc/gen-id.js"; import { genId } from "@/misc/gen-id.js";
import {
notesChart,
perUserNotesChart,
activeUsersChart,
instanceChart,
} from "@/services/chart/index.js";
import type { IPoll } from "@/models/entities/poll.js"; import type { IPoll } from "@/models/entities/poll.js";
import { Poll } from "@/models/entities/poll.js"; import { Poll } from "@/models/entities/poll.js";
import { createNotification } from "../create-notification.js"; import { createNotification } from "../create-notification.js";
@ -335,15 +329,10 @@ export default async (
res(note); res(note);
// 統計を更新
notesChart.update(note, true, user.isBot);
perUserNotesChart.update(user, note, true, user.isBot);
// Register host // Register host
if (Users.isRemoteUser(user)) { if (Users.isRemoteUser(user)) {
registerOrFetchInstanceDoc(user.host).then((i) => { registerOrFetchInstanceDoc(user.host).then((i) => {
Instances.increment({ id: i.id }, "notesCount", 1); Instances.increment({ id: i.id }, "notesCount", 1);
instanceChart.updateNote(i.host, note, true);
}); });
} }
@ -432,8 +421,6 @@ export default async (
} }
if (!silent) { if (!silent) {
if (Users.isLocalUser(user)) activeUsersChart.write(user);
// 未読通知を作成 // 未読通知を作成
if (data.visibility === "specified") { if (data.visibility === "specified") {
if (data.visibleUsers == null) throw new Error("invalid param"); if (data.visibleUsers == null) throw new Error("invalid param");

View file

@ -9,11 +9,6 @@ import config from "@/config/index.js";
import type { User, ILocalUser, IRemoteUser } from "@/models/entities/user.js"; import type { User, ILocalUser, IRemoteUser } from "@/models/entities/user.js";
import type { Note, IMentionedRemoteUsers } from "@/models/entities/note.js"; import type { Note, IMentionedRemoteUsers } from "@/models/entities/note.js";
import { Notes, Users, Instances } from "@/models/index.js"; import { Notes, Users, Instances } from "@/models/index.js";
import {
notesChart,
perUserNotesChart,
instanceChart,
} from "@/services/chart/index.js";
import { import {
deliverToFollowers, deliverToFollowers,
deliverToUser, deliverToUser,
@ -104,14 +99,9 @@ export default async function (
} }
//#endregion //#endregion
// 統計を更新
notesChart.update(note, false);
perUserNotesChart.update(user, note, false);
if (Users.isRemoteUser(user)) { if (Users.isRemoteUser(user)) {
registerOrFetchInstanceDoc(user.host).then((i) => { registerOrFetchInstanceDoc(user.host).then((i) => {
Instances.decrement({ id: i.id }, "notesCount", 1); Instances.decrement({ id: i.id }, "notesCount", 1);
instanceChart.updateNote(i.host, note, false);
}); });
} }
} }

View file

@ -14,7 +14,6 @@ import {
Blockings, Blockings,
} from "@/models/index.js"; } from "@/models/index.js";
import { IsNull, Not } from "typeorm"; import { IsNull, Not } from "typeorm";
import { perUserReactionsChart } from "@/services/chart/index.js";
import { genId } from "@/misc/gen-id.js"; import { genId } from "@/misc/gen-id.js";
import { createNotification } from "../../create-notification.js"; import { createNotification } from "../../create-notification.js";
import deleteReaction from "./delete.js"; import deleteReaction from "./delete.js";
@ -91,8 +90,6 @@ export default async (
.where("id = :id", { id: note.id }) .where("id = :id", { id: note.id })
.execute(); .execute();
perUserReactionsChart.update(user, note);
// カスタム絵文字リアクションだったら絵文字情報も送る // カスタム絵文字リアクションだったら絵文字情報も送る
const decodedReaction = decodeReaction(reaction); const decodedReaction = decodeReaction(reaction);

View file

@ -1,6 +1,5 @@
import type { User } from "@/models/entities/user.js"; import type { User } from "@/models/entities/user.js";
import { Hashtags, Users } from "@/models/index.js"; import { Hashtags, Users } from "@/models/index.js";
import { hashtagChart } from "@/services/chart/index.js";
import { genId } from "@/misc/gen-id.js"; import { genId } from "@/misc/gen-id.js";
import type { Hashtag } from "@/models/entities/hashtag.js"; import type { Hashtag } from "@/models/entities/hashtag.js";
import { normalizeForSearch } from "@/misc/normalize-for-search.js"; import { normalizeForSearch } from "@/misc/normalize-for-search.js";
@ -151,8 +150,4 @@ export async function updateHashtag(
} as Hashtag); } as Hashtag);
} }
} }
if (!isUserAttached) {
hashtagChart.update(tag, user);
}
} }