refactor: move misc/count-same-renotes.ts to backend-rs

holding @swc/cli and megalodon

docs/changes.md

@@ -42,6 +42,9 @@
 
 ### 細かい変更点
 
+- コードブロックの構文ハイライトの対応言語を強化
+  - TODO: CDN を利用しない設定でもこれを行えるようにする
+  - TODO: 本家版にもマージリクエストを出す
 - モバイル表示の下部のウィジェットボタンを再読み込みボタンに変更可能に
   - スマートフォンでウィジェットは使わないけど再読み込みはたくさんする人はいそう
 - 設定のバックアップファイルに `misskeyVersion` の値が含まれていなくても警告しないように変更

locales/en-US.yml

@@ -1174,6 +1174,8 @@ pullDownToReload: "Pull down to reload"
 enableTimelineStreaming: "Update timelines automatically"
 useEmojiCdn: "Get Twemoji from CDN"
 useEmojiCdnDescription: "Use Twemoji from the JSDelivr CDN instead of the server's assets."
+useCdn: "Get assets from CDN"
+useCdnDescription: "Load some static assets like Twemoji from the JSDelivr CDN instead of this Firefish server."
 suggested: "Suggested"
 noLanguage: "No language"
 

locales/ja-JP.yml

@@ -1008,6 +1008,8 @@ emphasizeFollowed: "フォロワーのアカウントに表示される「フォ
 iconSet: "アイコンのスタイル"
 useEmojiCdn: "CDNのTwemojiを利用する"
 useEmojiCdnDescription: "サーバー上に保存されているTwemojiのアセットの代わりに、JSDelivr CDNから配信されたものを用います。"
+useCdn: "CDNのアセットを利用する"
+useCdnDescription: "このFirefishサーバーからではなくJSDelivr CDNからTwiemojiなどのアセットを読み込みます。"
 
 _sensitiveMediaDetection:
   description: "機械学習を使って自動でセンシティブなメディアを検出し、モデレーションに役立てられます。サーバーの負荷が少し増えます。"

neko/index.js

@@ -295,13 +295,13 @@ if (!nativeBinding) {
   throw new Error(`Failed to load native binding`)
 }
 
-const { EnvConfig, readEnvironmentConfig, readServerConfig, JsDbConn, initDatabase, AntennaSrcEnum, MutedNoteReasonEnum, NoteVisibilityEnum, NotificationTypeEnum, PageVisibilityEnum, PollNotevisibilityEnum, RelayStatusEnum, UserEmojimodpermEnum, UserProfileFfvisibilityEnum, UserProfileMutingnotificationtypesEnum, stringToAcct, acctToString, getFullApAccount, isSelfHost, extractHost, toPuny, toPunyOptional, convertToHiddenPost, countSameRenotes, sqlLikeEscape, safeForSql, formatMilliseconds, nativeInitIdGenerator, nativeCreateId, nativeGetTimestamp, fetchMeta, metaToPugArgs, genString, IdConvertType, convertId } = nativeBinding
+const { EnvConfig, readEnvironmentConfig, readServerConfig, JsDbConn, connectToDatabase, AntennaSrcEnum, MutedNoteReasonEnum, NoteVisibilityEnum, NotificationTypeEnum, PageVisibilityEnum, PollNotevisibilityEnum, RelayStatusEnum, UserEmojimodpermEnum, UserProfileFfvisibilityEnum, UserProfileMutingnotificationtypesEnum, stringToAcct, acctToString, getFullApAccount, isSelfHost, extractHost, toPuny, toPunyOptional, convertToHiddenPost, sqlLikeEscape, safeForSql, formatMilliseconds, nativeInitIdGenerator, nativeCreateId, nativeGetTimestamp, fetchMeta, metaToPugArgs, otherRenoteByUserExists, genString, IdConvertType, convertId } = nativeBinding
 
 module.exports.EnvConfig = EnvConfig
 module.exports.readEnvironmentConfig = readEnvironmentConfig
 module.exports.readServerConfig = readServerConfig
 module.exports.JsDbConn = JsDbConn
-module.exports.initDatabase = initDatabase
+module.exports.connectToDatabase = connectToDatabase
 module.exports.AntennaSrcEnum = AntennaSrcEnum
 module.exports.MutedNoteReasonEnum = MutedNoteReasonEnum
 module.exports.NoteVisibilityEnum = NoteVisibilityEnum
@@ -320,7 +320,6 @@ module.exports.extractHost = extractHost
 module.exports.toPuny = toPuny
 module.exports.toPunyOptional = toPunyOptional
 module.exports.convertToHiddenPost = convertToHiddenPost
-module.exports.countSameRenotes = countSameRenotes
 module.exports.sqlLikeEscape = sqlLikeEscape
 module.exports.safeForSql = safeForSql
 module.exports.formatMilliseconds = formatMilliseconds
@@ -329,6 +328,7 @@ module.exports.nativeCreateId = nativeCreateId
 module.exports.nativeGetTimestamp = nativeGetTimestamp
 module.exports.fetchMeta = fetchMeta
 module.exports.metaToPugArgs = metaToPugArgs
+module.exports.otherRenoteByUserExists = otherRenoteByUserExists
 module.exports.genString = genString
 module.exports.IdConvertType = IdConvertType
 module.exports.convertId = convertId

neko/update/patch.sh

@@ -1,7 +1,14 @@
 #!/bin/sh
 set -eu
 
+# prefix message filename with timestamp
 [ ! -f neko/flags/install_pgroonga ]             || mv neko/flags/install_pgroonga             neko/flags/20231128_install_pgroonga
 [ ! -f neko/flags/docker_compose_rename ]        || mv neko/flags/docker_compose_rename        neko/flags/20231129_docker_compose_rename
 [ ! -f neko/flags/temp_upgrade_node_to_v21 ]     || mv neko/flags/temp_upgrade_node_to_v21     neko/flags/20231229_upgrade_node_to_v21
 [ ! -f neko/flags/add_volume_to_docker_compose ] || mv neko/flags/add_volume_to_docker_compose neko/flags/20240102_add_volume_to_docker_compose
+
+# remove native-utils
+rm -rf packages/backend/native-utils/built
+rm -rf packages/backend/native-utils/node_modules
+rm -rf packages/backend/native-utils/target
+rmdir packages/backend/native-utils || :

package.json

@@ -9,7 +9,7 @@
 	"engines": {
 		"node": ">= 18.17.0"
 	},
-	"packageManager": "pnpm@8.14.2",
+	"packageManager": "pnpm@8.14.3",
 	"private": true,
 	"scripts": {
 		"rebuild": "pnpm run clean && pnpm run build",
@@ -44,14 +44,14 @@
 		"js-yaml": "4.1.0"
 	},
 	"devDependencies": {
-		"@biomejs/biome": "1.5.2",
-		"@biomejs/cli-darwin-arm64": "1.5.2",
-		"@biomejs/cli-darwin-x64": "1.5.2",
-		"@biomejs/cli-linux-arm64": "1.5.2",
-		"@biomejs/cli-linux-x64": "1.5.2",
-		"@types/node": "20.11.5",
+		"@biomejs/biome": "1.5.3",
+		"@biomejs/cli-darwin-arm64": "1.5.3",
+		"@biomejs/cli-darwin-x64": "1.5.3",
+		"@biomejs/cli-linux-arm64": "1.5.3",
+		"@biomejs/cli-linux-x64": "1.5.3",
+		"@types/node": "20.11.6",
 		"execa": "8.0.1",
-		"pnpm": "8.14.2",
+		"pnpm": "8.14.3",
 		"typescript": "5.3.3"
 	}
 }

packages/backend-rs/Cargo.lock

@@ -1272,9 +1272,9 @@ dependencies = [
 
 [[package]]
 name = "napi"
-version = "2.14.2"
+version = "2.14.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2fc1cb00cde484640da9f01a124edbb013576a6ae9843b23857c940936b76d91"
+checksum = "902495f6b80f53f8435aefbbd2241c9c675fa239cd7e5f8e28fb57f3b69ecd09"
 dependencies = [
  "bitflags 2.4.2",
  "chrono",

packages/backend-rs/Cargo.toml

@@ -13,7 +13,7 @@ crate-type = ["cdylib", "lib"]
 [dependencies]
 async-trait = "0.1.77"
 cfg-if = "1.0.0"
-chrono = "0.4.31"
+chrono = "0.4.32"
 cuid2 = "0.1.2"
 idna = "0.5.0"
 jsonschema = "0.17.1"
@@ -21,7 +21,7 @@ once_cell = "1.19.0"
 parse-display = "0.8.2"
 rand = "0.8.5"
 schemars = { version = "0.8.16", features = ["chrono"] }
-sea-orm = { version = "0.12.11", features = ["sqlx-postgres", "runtime-tokio-rustls"] }
+sea-orm = { version = "0.12.12", features = ["sqlx-postgres", "runtime-tokio-rustls"] }
 serde = { version = "1.0.195", features = ["derive"] }
 serde_json = "1.0.111"
 serde_yaml = "0.9.30"
@@ -30,7 +30,7 @@ tokio = { version = "1.35.1", features = ["full"] }
 url = "2.5.0"
 
 # Default enable napi4 feature, see https://nodejs.org/api/n-api.html#node-api-version-matrix
-napi = { version = "2.14.2", default-features = false, features = ["napi9", "tokio_rt", "chrono_date", "serde-json"] }
+napi = { version = "2.14.4", default-features = false, features = ["napi9", "tokio_rt", "chrono_date", "serde-json"] }
 napi-derive = { version = "2.14.6", optional = true }
 basen = "0.1.0"
 urlencoding = "2.1.3"

packages/backend-rs/package.json

@@ -24,7 +24,7 @@
 	"license": "MIT",
 	"devDependencies": {
 		"@napi-rs/cli": "2.18.0",
-		"ava": "6.0.1"
+		"ava": "6.1.0"
 	},
 	"ava": {
 		"timeout": "3m"

packages/backend-rs/src/database/mod.rs

@@ -16,7 +16,7 @@ impl JsDbConn {
 }
 
 #[napi_derive::napi]
-pub async fn init_database(config: DbConfig) -> napi::Result<JsDbConn> {
+pub async fn connect_to_database(config: DbConfig) -> napi::Result<JsDbConn> {
     let conn_uri = format!(
         "postgres://{}:{}@{}:{}/{}",
         config.user,
@@ -33,18 +33,18 @@ pub async fn init_database(config: DbConfig) -> napi::Result<JsDbConn> {
 
 #[cfg(test)]
 mod unit_test {
-    use super::init_database;
+    use super::connect_to_database;
     use crate::config::server::read_server_config;
 
     #[tokio::test]
-    async fn connect_to_database() {
-        assert!(init_database(read_server_config().db).await.is_ok());
+    async fn connect_with_server_config() {
+        assert!(connect_to_database(read_server_config().db).await.is_ok());
     }
 
     #[tokio::test]
-    async fn invalid_db_password() {
+    async fn connect_with_incorrect_password() {
         let mut config = read_server_config().db;
-        config.pass.push('.'); // make password wrong
-        assert!(init_database(config).await.is_err());
+        config.pass.push('.'); // password should be incorrect now
+        assert!(connect_to_database(config).await.is_err());
     }
 }

packages/backend-rs/src/util/count_same_renotes.rs

@@ -1,19 +0,0 @@
-use crate::database::{JsDbConn, NapiDbErrExt};
-use crate::model::entity::note;
-use sea_orm::prelude::*;
-
-#[napi_derive::napi]
-pub async fn count_same_renotes(
-    conn: &JsDbConn,
-    user_id: String,
-    renote_id: String,
-    exclude_note_id: Option<String>,
-) -> napi::Result<u64> {
-    let mut query = note::Entity::find()
-        .filter(note::Column::UserId.eq(user_id))
-        .filter(note::Column::RenoteId.eq(renote_id));
-    if let Some(exclude_note_id) = exclude_note_id {
-        query = query.filter(note::Column::Id.ne(exclude_note_id));
-    }
-    query.count(conn.inner()).await.map_err(NapiDbErrExt::into)
-}

packages/backend-rs/src/util/meta.rs

@@ -12,9 +12,9 @@ fn update_cache(meta: &Meta) {
 }
 
 #[napi_derive::napi]
-pub async fn fetch_meta(conn: &JsDbConn, invalidate_cache: bool) -> napi::Result<Meta> {
+pub async fn fetch_meta(conn: &JsDbConn, invalidate_cache: Option<bool>) -> napi::Result<Meta> {
     // try using cache
-    if !invalidate_cache {
+    if invalidate_cache == Some(true) {
         if let Some(cache) = CACHE.lock().ok().and_then(|cache| cache.clone()) {
             return Ok(cache);
         }

packages/backend-rs/src/util/mod.rs

@@ -1,9 +1,9 @@
 pub mod acct;
 pub mod convert_host;
 pub mod convert_to_hidden_post;
-pub mod count_same_renotes;
 pub mod escape_sql;
 pub mod format_milliseconds;
 pub mod id;
 pub mod meta;
+pub mod note;
 pub mod random;

packages/backend-rs/src/util/note.rs

@@ -0,0 +1,25 @@
+use crate::database::{JsDbConn, NapiDbErrExt};
+use crate::model::entity::note;
+use sea_orm::prelude::*;
+
+/**
+ * Returns true if user {user_id}
+ * has any renote of note {renote_id}
+ * except the specified renote {excluded_note_id}
+ */
+#[napi_derive::napi]
+pub async fn other_renote_by_user_exists(
+    conn: &JsDbConn,
+    user_id: String,
+    renote_id: String,
+    excluded_note_id: String,
+) -> napi::Result<bool> {
+    note::Entity::find()
+        .filter(note::Column::UserId.eq(user_id))
+        .filter(note::Column::RenoteId.eq(renote_id))
+        .filter(note::Column::Id.ne(excluded_note_id))
+        .one(conn.inner())
+        .await
+        .map(|row| row.is_some())
+        .map_err(NapiDbErrExt::into)
+}

packages/backend/package.json

@@ -28,14 +28,14 @@
 		"@koa/router": "12.0.1",
 		"@ladjs/koa-views": "9.0.0",
 		"@peertube/http-signature": "1.7.0",
-		"@redocly/openapi-core": "1.6.0",
+		"@redocly/openapi-core": "1.7.0",
 		"@sinonjs/fake-timers": "11.2.2",
 		"@twemoji/parser": "^15.0.0",
 		"adm-zip": "^0.5.10",
 		"ajv": "8.12.0",
 		"archiver": "6.0.1",
 		"argon2": "^0.31.2",
-		"aws-sdk": "2.1540.0",
+		"aws-sdk": "2.1542.0",
 		"axios": "1.6.5",
 		"backend-rs": "workspace:*",
 		"bcryptjs": "2.4.3",
@@ -48,7 +48,7 @@
 		"cli-highlight": "2.1.11",
 		"color-convert": "2.0.1",
 		"content-disposition": "0.5.4",
-		"date-fns": "3.3.0",
+		"date-fns": "3.3.1",
 		"decompress": "4.2.1",
 		"deep-email-validator": "0.1.21",
 		"deepl-node": "1.11.0",
@@ -59,7 +59,7 @@
 		"firefish-js": "workspace:*",
 		"got": "14.0.0",
 		"gunzip-maybe": "1.4.2",
-		"happy-dom": "13.2.0",
+		"happy-dom": "13.3.1",
 		"hpagent": "1.2.0",
 		"ioredis": "5.3.2",
 		"ip-cidr": "4.0.0",
@@ -125,7 +125,7 @@
 		"xev": "3.0.2"
 	},
 	"devDependencies": {
-		"@swc/cli": "0.1.64",
+		"@swc/cli": "0.1.65",
 		"@swc/core": "1.3.105",
 		"@types/adm-zip": "0.5.5",
 		"@types/bcryptjs": "2.4.6",
@@ -144,7 +144,7 @@
 		"@types/koa__cors": "5.0.0",
 		"@types/koa__multer": "2.0.7",
 		"@types/koa__router": "12.0.4",
-		"@types/node": "20.11.5",
+		"@types/node": "20.11.6",
 		"@types/node-fetch": "2.6.11",
 		"@types/nodemailer": "6.4.14",
 		"@types/oauth": "0.9.4",
@@ -177,7 +177,7 @@
 		"ts-node": "10.9.2",
 		"tsconfig-paths": "4.2.0",
 		"typescript": "5.3.3",
-		"webpack": "5.89.0",
+		"webpack": "5.90.0",
 		"ws": "8.16.0"
 	}
 }

packages/backend/src/misc/backend-rs.ts

@@ -1,29 +1,27 @@
 import {
 	readServerConfig,
 	readEnvironmentConfig,
-	initDatabase,
-	countSameRenotes as countSameRenotesImpl,
+	connectToDatabase,
+	otherRenoteByUserExists as otherRenoteByUserExistsImpl,
 	fetchMeta as fetchMetaImpl,
 	getFullApAccount as getFullApAccountImpl,
 	isSelfHost as isSelfHostImpl,
+	JsDbConn,
 } from "backend-rs";
 
 export const serverConfig = readServerConfig();
 export const envConfig = readEnvironmentConfig();
-const dbPromise = initDatabase(serverConfig.db);
+const dbPromise = connectToDatabase(serverConfig.db);
 
 type Option<T> = T | null | undefined;
 
-type WithoutState<F> = F extends (state: any, ...args: infer P) => infer R
-	? (...args: P) => R
-	: never;
+const curryDb =
+	<P extends any[], R>(f: (db: JsDbConn, ...args: P) => Promise<R>) =>
+	(...args: P) =>
+		dbPromise.then((db) => f(db, ...args));
 
-export const countSameRenotes: WithoutState<typeof countSameRenotesImpl> = (
-	...args
-) => dbPromise.then((db) => countSameRenotesImpl(db, ...args));
-
-export const fetchMeta = (invalidateCache = false) =>
-	dbPromise.then((db) => fetchMetaImpl(db, invalidateCache));
+export const fetchMeta = curryDb(fetchMetaImpl);
+export const otherRenoteByUserExists = curryDb(otherRenoteByUserExistsImpl);
 
 export function getFullApAccount(
 	username: string,

packages/backend/src/services/note/create.ts

@@ -47,7 +47,7 @@ import { isDuplicateKeyValueError } from "@/misc/is-duplicate-key-value-error.js
 import { checkHitAntenna } from "@/misc/check-hit-antenna.js";
 import { getWordHardMute } from "@/misc/check-word-mute.js";
 import { addNoteToAntenna } from "@/services/add-note-to-antenna.js";
-import { countSameRenotes } from "@/misc/backend-rs.js";
+import { otherRenoteByUserExists } from "@/misc/backend-rs.js";
 import { deliverToRelays, getCachedRelays } from "../relay.js";
 import type { Channel } from "@/models/entities/channel.js";
 import { normalizeForSearch } from "@/misc/normalize-for-search.js";
@@ -412,7 +412,7 @@ export default async (
 		if (
 			data.renote &&
 			!user.isBot &&
-			(await countSameRenotes(user.id, data.renote.id, note.id)) === 0n
+			!(await otherRenoteByUserExists(user.id, data.renote.id, note.id))
 		) {
 			incRenoteCount(data.renote);
 		}

packages/backend/src/services/note/delete.ts

@@ -13,7 +13,7 @@ import {
 	deliverToFollowers,
 	deliverToUser,
 } from "@/remote/activitypub/deliver-manager.js";
-import { countSameRenotes } from "@/misc/backend-rs.js";
+import { otherRenoteByUserExists } from "@/misc/backend-rs.js";
 import { registerOrFetchInstanceDoc } from "@/services/register-or-fetch-instance-doc.js";
 import { deliverToRelays } from "@/services/relay.js";
 // import meilisearch from "@/db/meilisearch.js";
@@ -34,7 +34,7 @@ export default async function (
 	// この投稿を除く指定したユーザーによる指定したノートのリノートが存在しないとき
 	if (
 		note.renoteId &&
-		(await countSameRenotes(user.id, note.renoteId, note.id)) === 0n &&
+		!(await otherRenoteByUserExists(user.id, note.renoteId, note.id)) &&
 		deleteFromDb
 	) {
 		Notes.decrement({ id: note.renoteId }, "renoteCount", 1);

packages/client/package.json

@@ -46,7 +46,7 @@
 		"city-timezones": "1.2.1",
 		"compare-versions": "6.1.0",
 		"cropperjs": "2.0.0-beta.4",
-		"date-fns": "3.3.0",
+		"date-fns": "3.3.1",
 		"emojilib": "^3.0.11",
 		"eslint-config-prettier": "9.1.0",
 		"eslint-plugin-file-progress": "1.3.0",
@@ -76,7 +76,7 @@
 		"swiper": "11.0.5",
 		"syuilo-password-strength": "0.0.1",
 		"textarea-caret": "3.1.0",
-		"three": "0.160.0",
+		"three": "0.160.1",
 		"throttle-debounce": "5.0.0",
 		"tinycolor2": "1.6.0",
 		"tinyld": "^1.3.4",

packages/client/src/components/MkCode.core.vue

@@ -7,8 +7,8 @@
 </template>
 
 <script lang="ts" setup>
-import { computed } from "vue";
-import Prism from "prismjs";
+import { computed, ref } from "vue";
+import Prism, { loadLanguage } from "@/scripts/prism";
 import "prismjs/themes/prism-okaidia.css";
 
 const props = defineProps<{
@@ -17,9 +17,24 @@ const props = defineProps<{
 	inline?: boolean;
 }>();
 
-const prismLang = computed(() =>
-	Prism.languages[props.lang] ? props.lang : "plaintext",
+// fallback to "plaintext" if language not loaded
+const prismLang = ref(
+	props.lang != null && props.lang in Prism.languages
+		? props.lang
+		: "plaintext",
 );
+
+// try to load language asynchronously
+if (props.lang != null && !(props.lang in Prism.languages)) {
+	const { lang } = props;
+	loadLanguage(props.lang).then(
+		// onLoaded
+		() => (prismLang.value = lang),
+		// onError
+		() => {},
+	);
+}
+
 const html = computed(() =>
 	Prism.highlight(
 		props.code,

packages/client/src/pages/settings/general.vue

@@ -309,9 +309,9 @@
 				}}</span></FormSwitch
 			>
 			<FormSwitch v-model="useEmojiCdn" class="_formBlock"
-				>{{ i18n.ts.useEmojiCdn
+				>{{ i18n.ts.useCdn
 				}}<template #caption>{{
-					i18n.ts.useEmojiCdnDescription
+					i18n.ts.useCdnDescription
 				}}</template></FormSwitch
 			>
 			<FormSwitch

packages/client/src/scripts/prism.ts

@@ -0,0 +1,29 @@
+import Prism from "prismjs";
+import "prismjs/plugins/autoloader/prism-autoloader.js";
+import { defaultStore } from "@/store";
+
+// TODO
+Prism.plugins.autoloader.languages_path =
+	"https://cdn.jsdelivr.net/npm/prismjs@1.29.0/components/";
+
+const nonExistingLanguagesCache = new Set<string>();
+export const loadLanguage = (lang: string) =>
+	new Promise<void>((resolve, reject) => {
+		// for now
+		if (!defaultStore.state.useEmojiCdn) return resolve();
+		// cached non-existing language
+		if (nonExistingLanguagesCache.has(lang)) return reject();
+		// load language with autoloader
+		Prism.plugins.autoloader.loadLanguages(
+			lang,
+			// loaded
+			() => resolve(),
+			// failed
+			() => {
+				nonExistingLanguagesCache.add(lang);
+				reject();
+			},
+		);
+	});
+
+export default Prism;

packages/firefish-js/package.json

@@ -18,10 +18,10 @@
 		"url": "https://code.naskya.net/naskya/firefish"
 	},
 	"devDependencies": {
-		"@swc/cli": "0.1.64",
+		"@swc/cli": "0.1.65",
 		"@swc/core": "1.3.105",
 		"@swc/types": "0.1.5",
-		"@types/node": "20.11.5",
+		"@types/node": "20.11.6",
 		"ts-node": "10.9.2",
 		"tsd": "0.30.4",
 		"typescript": "5.3.3"

packages/sw/package.json

@@ -9,7 +9,7 @@
 		"format": "pnpm biome format * --write"
 	},
 	"devDependencies": {
-		"@swc/cli": "0.1.64",
+		"@swc/cli": "0.1.65",
 		"@swc/core": "1.3.105",
 		"@swc/core-android-arm64": "1.3.11",
 		"firefish-js": "workspace:*",

packages/sw/tsconfig.json

@@ -21,7 +21,7 @@
 		"isolatedModules": true,
 		"baseUrl": ".",
 		"paths": {
-			"@/*": ["./src/*"],
+			"@/*": ["./src/*"]
 		},
 		"typeRoots": [
 			"node_modules/@types",