sup39/firefish
1
0
Fork 0
forked from naskya/firefish
Code Issues Pull requests Activity

refactor: remove native-utils/migration

Browse source
This commit is contained in:
naskya 2024-01-22 08:54:24 +09:00
parent cbf8fd9f20
commit 38e9e21d60
Signed by: naskya
GPG key ID: 712D413B3A9FED5C
21 changed files with 223 additions and 1717 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
Dockerfile
View file

@ -18,8 +18,6 @@ EOC
COPY packages/backend/native-utils/Cargo.toml packages/backend/native-utils/Cargo.toml
COPY packages/backend/native-utils/Cargo.lock packages/backend/native-utils/Cargo.lock
COPY packages/backend/native-utils/src/lib.rs packages/backend/native-utils/src/
COPY packages/backend/native-utils/migration/Cargo.toml packages/backend/native-utils/migration/Cargo.toml
COPY packages/backend/native-utils/migration/src/lib.rs packages/backend/native-utils/migration/src/
# Install cargo dependencies
RUN cargo fetch --locked --manifest-path /firefish/packages/backend/native-utils/Cargo.toml

9
Makefile
View file

@ -8,16 +8,11 @@ PACKAGE_JSONS += packages/megalodon/package.json
PACKAGE_JSONS += packages/sw/package.json
RUST_DIR := packages/backend/native-utils
CARGO_TOMLS := $(RUST_DIR)/Cargo.toml
CARGO_TOMLS += $(RUST_DIR)/migration/Cargo.toml
RUSTY_STUFF := $(foreach dir, $(RUST_DIR)/src, $(wildcard $(dir)/*))
RUSTY_STUFF += $(foreach dir, $(RUST_DIR)/migration/src, $(wildcard $(dir)/*))
RUSTY_STUFF += $(CARGO_TOMLS)
RUSTY_STUFF += $(RUST_DIR)/Cargo.toml
MIGRATIONS := $(foreach dir, packages/backend/migration, $(wildcard $(dir)/*))
MIGRATIONS += $(foreach dir, packages/backend/migration-neko, $(wildcard $(dir)/*))
MIGRATIONS += $(foreach dir, $(RUST_DIR)/migration/src, $(wildcard $(dir)/*))
UPDATE_SCRIPTS := update.sh
UPDATE_SCRIPTS += $(foreach dir, neko/update, $(wildcard $(dir)/*))
@ -57,7 +52,7 @@ update-lockfiles: pnpm-lock.yaml $(RUST_DIR)/Cargo.lock
pnpm-lock.yaml: pnpm-workspace.yaml $(PACKAGE_JSONS)
pnpm install
$(RUST_DIR)/Cargo.lock: $(CARGO_TOMLS)
$(RUST_DIR)/Cargo.lock: $(RUST_DIR)/Cargo.toml
cd $(RUST_DIR) && cargo update

2
docs/changes.md
View file

@ -11,6 +11,8 @@
- この Firefish はよく Firefish neko flavor と呼ばれていますが、これはこのバージョン番号に由来します。私自身は特にこの呼称を普及させたいわけではなかったのですが、[私のアカウント](https://post.naskya.net/@dev)の数割のフォロワーが英語圏の本家 Firefish の利用者であるため単に「この機能を追加しました！」などと投稿すると本家版の更新と誤解される恐れがありますし、「私の Firefish フォークにこの機能を追加しました！」などと投稿すると私が本家 Firefish の開発を離れて新たなハードフォークを作っているのだと誤解される恐れがあるため、投稿を機械翻訳で英訳されて読まれてもなるべく誤解されないための苦肉の策としてこの呼称を使い始めました。このソフトフォークはあくまで本家 Firefish の私による「味付け」です。
- 依存ライブラリをほとんど常に最新版にアップデート
- 本家版を更新する前にちゃんと動くか実験したいという意図もあります
- `native-utils/migration` を廃止
- Rust に移植できる部分は少しずつ移植していきますが、Rust によるデータベースのマイグレーションは（少なくとも当面の間）行いません
## クライアントの変更点

43
neko/index.js
View file

@ -250,6 +250,49 @@ switch (platform) {
loadError = e;
}
break;
case "riscv64":
if (isMusl()) {
localFileExisted = existsSync(
join(__dirname, "native-utils.linux-riscv64-musl.node"),
);
try {
if (localFileExisted) {
nativeBinding = require("./native-utils.linux-riscv64-musl.node");
} else {
nativeBinding = require("native-utils-linux-riscv64-musl");
}
} catch (e) {
loadError = e;
}
} else {
localFileExisted = existsSync(
join(__dirname, "native-utils.linux-riscv64-gnu.node"),
);
try {
if (localFileExisted) {
nativeBinding = require("./native-utils.linux-riscv64-gnu.node");
} else {
nativeBinding = require("native-utils-linux-riscv64-gnu");
}
} catch (e) {
loadError = e;
}
}
break;
case "s390x":
localFileExisted = existsSync(
join(__dirname, "native-utils.linux-s390x-gnu.node"),
);
try {
if (localFileExisted) {
nativeBinding = require("./native-utils.linux-s390x-gnu.node");
} else {
nativeBinding = require("native-utils-linux-s390x-gnu");
}
} catch (e) {
loadError = e;
}
break;
default:
throw new Error(`Unsupported architecture on Linux: ${arch}`);
}

18
neko/revert.sql
View file

@ -7,9 +7,25 @@ DELETE FROM "migrations" WHERE name IN (
'Vervis1699302371683',
'Pgroonga1698420787202',
'TruncateChartTables1694921638251',
'EmojiModerator1692825433698'
'EmojiModerator1692825433698',
'RemoveNativeUtilsMigration1705877093218'
);
-- remove-native-utils-migration
CREATE TABLE "seaql_migrations" (
version character varying NOT NULL,
applied_at bigint NOT NULL
);
INSERT INTO "seaql_migrations" (version, applied_at)
VALUES
('m20230531_180824_drop_reversi', 1705876632),
('m20230627_185451_index_note_url', 1705876632),
('m20230709_000510_move_antenna_to_cache', 1705876632),
('m20230806_170616_fix_antenna_stream_ids', 1705876632),
('m20230904_013244_is_indexable', 1705876632),
('m20231002_143323_remove_integrations', 1705876632)
;
-- remove-nsfw-detection
ALTER TABLE "user_profile" ADD "autoSensitive" boolean NOT NULL DEFAULT false;
ALTER TABLE "meta" ADD "enableSensitiveMediaDetectionForVideos" boolean NOT NULL DEFAULT false;

152
packages/backend/migration-neko/1705877093218-remove-native-utils-migration.js Normal file
View file

@ -0,0 +1,152 @@
export class RemoveNativeUtilsMigration1705877093218 {
name = "RemoveNativeUtilsMigration1705877093218";
async up(queryRunner) {
await queryRunner.query(`DROP TABLE IF EXISTS "reversi_game"`);
await queryRunner.query(`DROP TABLE IF EXISTS "reversi_matching"`);
await queryRunner.query(
`CREATE INDEX IF NOT EXISTS "IDX_note_url" ON "note" ("text")`,
);
await queryRunner.query(`DROP TABLE IF EXISTS "antenna_note"`);
await queryRunner.query(
`ALTER TABLE "user" ADD COLUMN IF NOT EXISTS "isIndexable" boolean NOT NULL DEFAULT true`,
);
await queryRunner.query(
`ALTER TABLE "user_profile" ADD COLUMN IF NOT EXISTS "isIndexable" boolean NOT NULL DEFAULT true`,
);
await queryRunner.query(
`ALTER TABLE "meta" DROP COLUMN IF EXISTS "enableTwitterIntegration"`,
);
await queryRunner.query(
`ALTER TABLE "meta" DROP COLUMN IF EXISTS "twitterConsumerKey"`,
);
await queryRunner.query(
`ALTER TABLE "meta" DROP COLUMN IF EXISTS "twitterConsumerSecret"`,
);
await queryRunner.query(
`ALTER TABLE "meta" DROP COLUMN IF EXISTS "enableGithubIntegration"`,
);
await queryRunner.query(
`ALTER TABLE "meta" DROP COLUMN IF EXISTS "githubClientId"`,
);
await queryRunner.query(
`ALTER TABLE "meta" DROP COLUMN IF EXISTS "githubClientSecret"`,
);
await queryRunner.query(
`ALTER TABLE "meta" DROP COLUMN IF EXISTS "enableDiscordIntegration"`,
);
await queryRunner.query(
`ALTER TABLE "meta" DROP COLUMN IF EXISTS "discordClientId"`,
);
await queryRunner.query(
`ALTER TABLE "meta" DROP COLUMN IF EXISTS "discordClientSecret"`,
);
await queryRunner.query(
`ALTER TABLE "user_profile" DROP COLUMN IF EXISTS "integrations"`,
);
await queryRunner.query(`DROP TABLE IF EXISTS "seaql_migrations"`);
}
async down(queryRunner) {
await queryRunner.query(
`CREATE TABLE IF NOT EXISTS "seaql_migrations" (version character varying NOT NULL, applied_at bigint NOT NULL)`,
);
await queryRunner.query(
`ALTER TABLE "user_profile" ADD COLUMN IF NOT EXISTS "integrations" jsonb DEFAULT '{}'`,
);
await queryRunner.query(
`ALTER TABLE "meta" ADD COLUMN IF NOT EXISTS "discordClientSecret" character varying(128)`,
);
await queryRunner.query(
`ALTER TABLE "meta" ADD COLUMN IF NOT EXISTS "discordClientId" character varying(128)`,
);
await queryRunner.query(
`ALTER TABLE "meta" ADD COLUMN IF NOT EXISTS "enableDiscordIntegration" boolean NOT NULL DEFAULT false`,
);
await queryRunner.query(
`ALTER TABLE "meta" ADD COLUMN IF NOT EXISTS "githubClientSecret" character varying(128)`,
);
await queryRunner.query(
`ALTER TABLE "meta" ADD COLUMN IF NOT EXISTS "githubClientId" character varying(128)`,
);
await queryRunner.query(
`ALTER TABLE "meta" ADD COLUMN IF NOT EXISTS "enableGithubIntegration" boolean NOT NULL DEFAULT false`,
);
await queryRunner.query(
`ALTER TABLE "meta" ADD COLUMN IF NOT EXISTS "twitterConsumerSecret" character varying(128)`,
);
await queryRunner.query(
`ALTER TABLE "meta" ADD COLUMN IF NOT EXISTS "twitterConsumerKey" character varying(128)`,
);
await queryRunner.query(
`ALTER TABLE "meta" ADD COLUMN IF NOT EXISTS "enableTwitterIntegration" boolean NOT NULL DEFAULT false`,
);
await queryRunner.query(
`ALTER TABLE "user_profile" DROP COLUMN IF EXISTS "isIndexable"`,
);
await queryRunner.query(
`ALTER TABLE "user" DROP COLUMN IF EXISTS "isIndexable"`,
);
await queryRunner.query(
`CREATE TABLE IF NOT EXISTS "antenna_note" ("id" character varying(32) NOT NULL, "noteId" character varying(32) NOT NULL, "antennaId" character varying(32) NOT NULL, "read" boolean NOT NULL DEFAULT false, CONSTRAINT "PK_fb28d94d0989a3872df19fd6ef8" PRIMARY KEY ("id"))`,
);
await queryRunner.query(
`CREATE INDEX IF NOT EXISTS "IDX_0d775946662d2575dfd2068a5f" ON "antenna_note" ("antennaId")`,
);
await queryRunner.query(
`CREATE INDEX IF NOT EXISTS "IDX_bd0397be22147e17210940e125" ON "antenna_note" ("noteId")`,
);
await queryRunner.query(
`CREATE UNIQUE INDEX IF NOT EXISTS "IDX_335a0bf3f904406f9ef3dd51c2" ON "antenna_note" ("noteId", "antennaId")`,
);
await queryRunner.query(
`CREATE INDEX "IDX_9937ea48d7ae97ffb4f3f063a4" ON "antenna_note" ("read")`,
);
await queryRunner.query(
`ALTER TABLE "antenna_note" ADD CONSTRAINT IF NOT EXISTS "FK_0d775946662d2575dfd2068a5f5" FOREIGN KEY ("antennaId") REFERENCES "antenna"("id") ON DELETE CASCADE ON UPDATE NO ACTION`,
);
await queryRunner.query(
`ALTER TABLE "antenna_note" ADD CONSTRAINT IF NOT EXISTS "FK_bd0397be22147e17210940e125b" FOREIGN KEY ("noteId") REFERENCES "note"("id") ON DELETE CASCADE ON UPDATE NO ACTION`,
);
await queryRunner.query(`DROP INDEX IF EXISTS "IDX_note_url"`);
await queryRunner.query(
`CREATE TABLE IF NOT EXISTS "reversi_matching" ("id" character varying(32) NOT NULL, "createdAt" TIMESTAMP WITH TIME ZONE NOT NULL, "parentId" character varying(32) NOT NULL, "childId" character varying(32) NOT NULL, CONSTRAINT "PK_880bd0afbab232f21c8b9d146cf" PRIMARY KEY ("id"))`,
);
await queryRunner.query(
`CREATE INDEX IF NOT EXISTS "IDX_b604d92d6c7aec38627f6eaf16" ON "reversi_matching" ("createdAt")`,
);
await queryRunner.query(
`CREATE INDEX IF NOT EXISTS "IDX_3b25402709dd9882048c2bbade" ON "reversi_matching" ("parentId")`,
);
await queryRunner.query(
`CREATE INDEX IF NOT EXISTS "IDX_e247b23a3c9b45f89ec1299d06" ON "reversi_matching" ("childId")`,
);
await queryRunner.query(
`ALTER TABLE "reversi_matching" ADD CONSTRAINT IF NOT EXISTS "FK_3b25402709dd9882048c2bbade0" FOREIGN KEY ("parentId") REFERENCES "user"("id") ON DELETE CASCADE ON UPDATE NO ACTION`,
);
await queryRunner.query(
`ALTER TABLE "reversi_matching" ADD CONSTRAINT IF NOT EXISTS "FK_e247b23a3c9b45f89ec1299d066" FOREIGN KEY ("childId") REFERENCES "user"("id") ON DELETE CASCADE ON UPDATE NO ACTION`,
);
await queryRunner.query(
`COMMENT ON COLUMN "reversi_matching"."createdAt" IS 'The created date of the ReversiMatching.'`,
);
await queryRunner.query(
`CREATE TABLE IF NOT EXISTS "reversi_game" ("id" character varying(32) NOT NULL, "createdAt" TIMESTAMP WITH TIME ZONE NOT NULL, "startedAt" TIMESTAMP WITH TIME ZONE, "user1Id" character varying(32) NOT NULL, "user2Id" character varying(32) NOT NULL, "user1Accepted" boolean NOT NULL DEFAULT false, "user2Accepted" boolean NOT NULL DEFAULT false, "black" integer, "isStarted" boolean NOT NULL DEFAULT false, "isEnded" boolean NOT NULL DEFAULT false, "winnerId" character varying(32), "surrendered" character varying(32), "logs" jsonb NOT NULL DEFAULT '[]', "map" character varying(64) array NOT NULL, "bw" character varying(32) NOT NULL, "isLlotheo" boolean NOT NULL DEFAULT false, "canPutEverywhere" boolean NOT NULL DEFAULT false, "loopedBoard" boolean NOT NULL DEFAULT false, "form1" jsonb DEFAULT null, "form2" jsonb DEFAULT null, "crc32" character varying(32), CONSTRAINT "PK_76b30eeba71b1193ad7c5311c3f" PRIMARY KEY ("id"))`,
);
await queryRunner.query(
`CREATE INDEX IF NOT EXISTS "IDX_b46ec40746efceac604142be1c" ON "reversi_game" ("createdAt")`,
);
await queryRunner.query(
`ALTER TABLE "reversi_game" ADD CONSTRAINT IF NOT EXISTS "FK_f7467510c60a45ce5aca6292743" FOREIGN KEY ("user1Id") REFERENCES "user"("id") ON DELETE CASCADE ON UPDATE NO ACTION`,
);
await queryRunner.query(
`ALTER TABLE "reversi_game" ADD CONSTRAINT IF NOT EXISTS "FK_6649a4e8c5d5cf32fb03b5da9f6" FOREIGN KEY ("user2Id") REFERENCES "user"("id") ON DELETE CASCADE ON UPDATE NO ACTION`,
);
await queryRunner.query(
`COMMENT ON COLUMN "reversi_game"."createdAt" IS 'The created date of the ReversiGame.'`,
);
await queryRunner.query(
`COMMENT ON COLUMN "reversi_game"."startedAt" IS 'The started date of the ReversiGame.'`,
);
}
}

373
packages/backend/native-utils/Cargo.lock generated
View file

@ -443,33 +443,6 @@ version = "1.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "acbf1af155f9b9ef647e42cdc158db4b64a1b61f743629225fde6f3e0be2a7c7"
[[package]]
name = "combine"
version = "4.6.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "35ed6e9d84f0b51a7f52daf1c7d71dd136fd7a3f41a8462b8cdb8c78d920fad4"
dependencies = [
"bytes",
"futures-core",
"memchr",
"pin-project-lite",
"tokio",
"tokio-util",
]
[[package]]
name = "console"
version = "0.15.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0e1f83fc076bd6dd27517eacdf25fef6c4dfe5f1d7448bafaaf3a26f13b5e4eb"
dependencies = [
"encode_unicode",
"lazy_static",
"libc",
"unicode-width",
"windows-sys 0.52.0",
]
[[package]]
name = "const-oid"
version = "0.9.6"
@ -649,12 +622,6 @@ dependencies = [
"serde",
]
[[package]]
name = "encode_unicode"
version = "0.3.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a357d28ed41a50f9c765dbfe56cbc04a64e53e5fc58ba79fbc34c10ef3df831f"
[[package]]
name = "encoding_rs"
version = "0.8.33"
@ -769,7 +736,6 @@ checksum = "645c6916888f6cb6350d2550b80fb63e734897a8498abe35cfb732b6487804b0"
dependencies = [
"futures-channel",
"futures-core",
"futures-executor",
"futures-io",
"futures-sink",
"futures-task",
@ -820,17 +786,6 @@ version = "0.3.30"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a44623e20b9681a318efdd71c299b6b222ed6f231972bfe2f224ebad6311f0c1"
[[package]]
name = "futures-macro"
version = "0.3.30"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "87750cf4b7a4c0625b1529e4c543c2182106e4dedc60a2a6455e00d212c489ac"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.48",
]
[[package]]
name = "futures-sink"
version = "0.3.30"
@ -852,7 +807,6 @@ dependencies = [
"futures-channel",
"futures-core",
"futures-io",
"futures-macro",
"futures-sink",
"futures-task",
"memchr",
@ -890,12 +844,6 @@ version = "0.28.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4271d37baee1b8c7e4b708028c57d816cf9d2434acb33a549475f78c181f6253"
[[package]]
name = "glob"
version = "0.3.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d2fabcfbdc87f4758337ca535fb41a6d701b65693ce38287d856d1674551ec9b"
[[package]]
name = "h2"
version = "0.3.24"
@ -1042,7 +990,7 @@ dependencies = [
"httpdate",
"itoa",
"pin-project-lite",
"socket2 0.5.5",
"socket2",
"tokio",
"tower-service",
"tracing",
@ -1092,20 +1040,6 @@ dependencies = [
"hashbrown 0.14.3",
]
[[package]]
name = "indicatif"
version = "0.17.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fb28741c9db9a713d93deb3bb9515c20788cef5815265bee4980e87bde7e0f25"
dependencies = [
"console",
"instant",
"number_prefix",
"portable-atomic",
"tokio",
"unicode-width",
]
[[package]]
name = "inherent"
version = "1.0.11"
@ -1117,15 +1051,6 @@ dependencies = [
"syn 2.0.48",
]
[[package]]
name = "instant"
version = "0.1.12"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7a5bbe824c507c5da5956355e86a746d82e0e1464f65d862cc5e71da70e94b2c"
dependencies = [
"cfg-if",
]
[[package]]
name = "ipnet"
version = "2.9.0"
@ -1268,15 +1193,6 @@ version = "0.4.20"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b5e6163cb8c49088c2c36f57875e58ccd8c87c7427f7fbd50ea6710b2f3f2e8f"
[[package]]
name = "matchers"
version = "0.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8263075bb86c5a1b1427b5ae862e8889656f126e9f77c484496e8b47cf5c5558"
dependencies = [
"regex-automata 0.1.10",
]
[[package]]
name = "md-5"
version = "0.10.6"
@ -1293,25 +1209,6 @@ version = "2.7.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "523dc4f511e55ab87b694dc30d0f820d60906ef06413f93d4d7a1385599cc149"
[[package]]
name = "migration"
version = "0.1.0"
dependencies = [
"basen",
"futures",
"indicatif",
"native-utils",
"redis",
"sea-orm",
"sea-orm-migration",
"serde",
"serde_json",
"serde_yaml",
"tokio",
"url",
"urlencoding",
]
[[package]]
name = "mime"
version = "0.3.17"
@ -1564,12 +1461,6 @@ dependencies = [
"libc",
]
[[package]]
name = "number_prefix"
version = "0.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "830b246a0e5f20af87141b25c173cd1b609bd7779a4617d6ec582abaf90870f3"
[[package]]
name = "object"
version = "0.32.2"
@ -1585,12 +1476,6 @@ version = "1.19.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92"
[[package]]
name = "openssl-probe"
version = "0.1.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf"
[[package]]
name = "ordered-float"
version = "3.9.2"
@ -1733,12 +1618,6 @@ version = "0.3.29"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2900ede94e305130c13ddd391e0ab7cbaeb783945ae07a279c268cb05109c6cb"
[[package]]
name = "portable-atomic"
version = "1.6.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7170ef9988bc169ba16dd36a7fa041e5c4cbeb6a35b76d4c03daded371eae7c0"
[[package]]
name = "powerfmt"
version = "0.2.0"
@ -1868,32 +1747,6 @@ dependencies = [
"getrandom",
]
[[package]]
name = "redis"
version = "0.24.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c580d9cbbe1d1b479e8d67cf9daf6a62c957e6846048408b80b43ac3f6af84cd"
dependencies = [
"async-trait",
"bytes",
"combine",
"futures-util",
"itoa",
"percent-encoding",
"pin-project-lite",
"rustls",
"rustls-native-certs",
"rustls-pemfile",
"rustls-webpki",
"ryu",
"sha1_smol",
"socket2 0.4.10",
"tokio",
"tokio-rustls",
"tokio-util",
"url",
]
[[package]]
name = "redox_syscall"
version = "0.4.1"
@ -1911,19 +1764,10 @@ checksum = "b62dbe01f0b06f9d8dc7d49e05a0785f153b00b2c227856282f671e0318c9b15"
dependencies = [
"aho-corasick",
"memchr",
"regex-automata 0.4.4",
"regex-automata",
"regex-syntax 0.8.2",
]
[[package]]
name = "regex-automata"
version = "0.1.10"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6c230d73fb8d8c1b9c0b3135c5142a8acee3a0558fb8db5cf1cb65f8d7862132"
dependencies = [
"regex-syntax 0.6.29",
]
[[package]]
name = "regex-automata"
version = "0.4.4"
@ -1935,12 +1779,6 @@ dependencies = [
"regex-syntax 0.8.2",
]
[[package]]
name = "regex-syntax"
version = "0.6.29"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f162c6dd7b008981e4d40210aca20b4bd0f9b60ca9271061b07f78537722f2e1"
[[package]]
name = "regex-syntax"
version = "0.7.5"
@ -2101,24 +1939,11 @@ version = "0.21.10"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f9d5a6813c0759e4609cd494e8e725babae6a2ca7b62a5536a13daaec6fcb7ba"
dependencies = [
"log",
"ring",
"rustls-webpki",
"sct",
]
[[package]]
name = "rustls-native-certs"
version = "0.6.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a9aace74cb666635c918e9c12bc0d348266037aa8eb599b5cba565709a8dff00"
dependencies = [
"openssl-probe",
"rustls-pemfile",
"schannel",
"security-framework",
]
[[package]]
name = "rustls-pemfile"
version = "1.0.4"
@ -2144,15 +1969,6 @@ version = "1.0.16"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f98d2aa92eebf49b69786be48e4477826b256916e84a57ff2a4f21923b48eb4c"
[[package]]
name = "schannel"
version = "0.1.23"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fbc91545643bcf3a0bbb6569265615222618bdf33ce4ffbbd13c4bbd4c093534"
dependencies = [
"windows-sys 0.52.0",
]
[[package]]
name = "schemars"
version = "0.8.16"
@ -2235,23 +2051,6 @@ dependencies = [
"uuid",
]
[[package]]
name = "sea-orm-cli"
version = "0.12.11"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "747f032e4abd58a6cacdb21275cc13448b4ff4b368f5f578ece465957ae7b417"
dependencies = [
"chrono",
"clap",
"dotenvy",
"glob",
"regex",
"sea-schema",
"tracing",
"tracing-subscriber",
"url",
]
[[package]]
name = "sea-orm-macros"
version = "0.12.11"
@ -2266,23 +2065,6 @@ dependencies = [
"unicode-ident",
]
[[package]]
name = "sea-orm-migration"
version = "0.12.11"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "412643baf225b37ae1313c0344c10bff6ec1fae72cdec1c7b591cc7c833134d2"
dependencies = [
"async-trait",
"clap",
"dotenvy",
"futures",
"sea-orm",
"sea-orm-cli",
"sea-schema",
"tracing",
"tracing-subscriber",
]
[[package]]
name = "sea-query"
version = "0.30.7"
@ -2295,7 +2077,6 @@ dependencies = [
"inherent",
"ordered-float",
"rust_decimal",
"sea-query-derive",
"serde_json",
"time",
"uuid",
@ -2317,71 +2098,12 @@ dependencies = [
"uuid",
]
[[package]]
name = "sea-query-derive"
version = "0.4.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "25a82fcb49253abcb45cdcb2adf92956060ec0928635eb21b4f7a6d8f25ab0bc"
dependencies = [
"heck",
"proc-macro2",
"quote",
"syn 2.0.48",
"thiserror",
]
[[package]]
name = "sea-schema"
version = "0.14.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "30d148608012d25222442d1ebbfafd1228dbc5221baf4ec35596494e27a2394e"
dependencies = [
"futures",
"sea-query",
"sea-schema-derive",
]
[[package]]
name = "sea-schema-derive"
version = "0.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c6f686050f76bffc4f635cda8aea6df5548666b830b52387e8bc7de11056d11e"
dependencies = [
"heck",
"proc-macro2",
"quote",
"syn 1.0.109",
]
[[package]]
name = "seahash"
version = "4.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1c107b6f4780854c8b126e228ea8869f4d7b71260f962fefb57b996b8959ba6b"
[[package]]
name = "security-framework"
version = "2.9.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "05b64fb303737d99b81884b2c63433e9ae28abebe5eb5045dcdd175dc2ecf4de"
dependencies = [
"bitflags 1.3.2",
"core-foundation",
"core-foundation-sys",
"libc",
"security-framework-sys",
]
[[package]]
name = "security-framework-sys"
version = "2.9.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e932934257d3b408ed8f30db49d85ea163bfe74961f017f405b025af298f0c7a"
dependencies = [
"core-foundation-sys",
"libc",
]
[[package]]
name = "semver"
version = "1.0.21"
@ -2466,12 +2188,6 @@ dependencies = [
"digest",
]
[[package]]
name = "sha1_smol"
version = "1.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ae1a47186c03a32177042e55dbc5fd5aee900b8e0069a8d70fba96a9375cd012"
[[package]]
name = "sha2"
version = "0.10.8"
@ -2493,15 +2209,6 @@ dependencies = [
"keccak",
]
[[package]]
name = "sharded-slab"
version = "0.1.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f40ca3c46823713e0d4209592e8d6e826aa57e928f09752619fc696c499637f6"
dependencies = [
"lazy_static",
]
[[package]]
name = "signal-hook-registry"
version = "1.4.1"
@ -2542,16 +2249,6 @@ version = "1.13.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e6ecd384b10a64542d77071bd64bd7b231f4ed5940fba55e98c3de13824cf3d7"
[[package]]
name = "socket2"
version = "0.4.10"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9f7916fc008ca5542385b89a3d3ce689953c143e9304a9bf8beec1de48994c0d"
dependencies = [
"libc",
"winapi",
]
[[package]]
name = "socket2"
version = "0.5.5"
@ -2969,16 +2666,6 @@ dependencies = [
"syn 2.0.48",
]
[[package]]
name = "thread_local"
version = "1.1.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3fdd6f064ccff2d6567adcb3873ca630700f00b5ad3f060c25b5dcfd9a4ce152"
dependencies = [
"cfg-if",
"once_cell",
]
[[package]]
name = "time"
version = "0.3.31"
@ -3037,7 +2724,7 @@ dependencies = [
"parking_lot",
"pin-project-lite",
"signal-hook-registry",
"socket2 0.5.5",
"socket2",
"tokio-macros",
"windows-sys 0.48.0",
]
@ -3053,16 +2740,6 @@ dependencies = [
"syn 2.0.48",
]
[[package]]
name = "tokio-rustls"
version = "0.24.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c28327cf380ac148141087fbfb9de9d7bd4e84ab5d2c28fbc911d753de8a7081"
dependencies = [
"rustls",
"tokio",
]
[[package]]
name = "tokio-stream"
version = "0.1.14"
@ -3143,21 +2820,6 @@ dependencies = [
"once_cell",
]
[[package]]
name = "tracing-subscriber"
version = "0.3.18"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ad0f048c97dbd9faa9b7df56362b8ebcaa52adb06b498c050d2f4e32f90a7a8b"
dependencies = [
"matchers",
"once_cell",
"regex",
"sharded-slab",
"thread_local",
"tracing",
"tracing-core",
]
[[package]]
name = "try-lock"
version = "0.2.5"
@ -3197,12 +2859,6 @@ version = "1.10.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1dd624098567895118886609431a7c3b8f516e41d30e0643f03d94592a147e36"
[[package]]
name = "unicode-width"
version = "0.1.11"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e51733f11c9c4f72aa0c160008246859e340b00807569a0da0e7a1079b27ba85"
[[package]]
name = "unicode_categories"
version = "0.1.1"
@ -3230,7 +2886,6 @@ dependencies = [
"form_urlencoded",
"idna",
"percent-encoding",
"serde",
]
[[package]]
@ -3369,28 +3024,6 @@ version = "1.4.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "22fc3756b8a9133049b26c7f61ab35416c130e8c09b660f5b3958b446f52cc50"
[[package]]
name = "winapi"
version = "0.3.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419"
dependencies = [
"winapi-i686-pc-windows-gnu",
"winapi-x86_64-pc-windows-gnu",
]
[[package]]
name = "winapi-i686-pc-windows-gnu"
version = "0.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6"
[[package]]
name = "winapi-x86_64-pc-windows-gnu"
version = "0.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f"
[[package]]
name = "windows-core"
version = "0.52.0"

3
packages/backend/native-utils/Cargo.toml
View file

@ -3,9 +3,6 @@ edition = "2021"
name = "native-utils"
version = "0.0.0"
[workspace]
members = ["migration"]
[features]
default = ["napi"]
napi = ["dep:napi-derive"]

37
packages/backend/native-utils/migration/Cargo.toml
View file

@ -1,37 +0,0 @@
[package]
name = "migration"
version = "0.1.0"
edition = "2021"
publish = false
[lib]
name = "migration"
path = "src/lib.rs"
[features]
default = []
convert = ["dep:native-utils", "dep:indicatif", "dep:futures"]
[dependencies]
serde_json = "1.0.111"
native-utils = { path = "../", optional = true }
indicatif = { version = "0.17.7", features = ["tokio"], optional = true }
tokio = { version = "1.35.1", features = ["full"] }
futures = { version = "0.3.30", optional = true }
serde_yaml = "0.9.30"
serde = { version = "1.0.195", features = ["derive"] }
urlencoding = "2.1.3"
redis = { version = "0.24.0", features = ["tokio-rustls-comp"] }
sea-orm = "0.12.11"
url = { version = "2.5.0", features = ["serde"] }
basen = "0.1.0"
[dependencies.sea-orm-migration]
version = "0.12.11"
features = [
# Enable at least one `ASYNC_RUNTIME` and `DATABASE_DRIVER` feature if you want to run migration via CLI.
# View the list of supported features at https://www.sea-ql.org/SeaORM/docs/install-and-config/database-and-async-runtime.
# e.g.
"runtime-tokio-rustls", # `ASYNC_RUNTIME` feature
"sqlx-postgres", # `DATABASE_DRIVER` feature
]

55
packages/backend/native-utils/migration/README.md
View file

@ -1,55 +0,0 @@
# Making migrations
For more information, please read https://www.sea-ql.org/SeaORM/docs/migration/setting-up-migration/
- Install `sea-orm-cli`
```sh
cargo install sea-orm-cli
```
- Generate
```sh
sea-orm-cli migrate generate ****
```
# Running Migrator CLI
- Generate a new migration file
```sh
cargo run -- migrate generate MIGRATION_NAME
```
- Apply all pending migrations
```sh
cargo run
```
```sh
cargo run -- up
```
- Apply first 10 pending migrations
```sh
cargo run -- up -n 10
```
- Rollback last applied migrations
```sh
cargo run -- down
```
- Rollback last 10 applied migrations
```sh
cargo run -- down -n 10
```
- Drop all tables from the database, then reapply all migrations
```sh
cargo run -- fresh
```
- Rollback all applied migrations, then reapply all migrations
```sh
cargo run -- refresh
```
- Rollback all applied migrations
```sh
cargo run -- reset
```
- Check the status of all migrations
```sh
cargo run -- status
```

35
packages/backend/native-utils/migration/src/lib.rs
View file

@ -1,35 +0,0 @@
pub use sea_orm_migration::prelude::*;
use basen::BASE36;
mod m20230531_180824_drop_reversi;
mod m20230627_185451_index_note_url;
mod m20230709_000510_move_antenna_to_cache;
mod m20230806_170616_fix_antenna_stream_ids;
mod m20230904_013244_is_indexable;
mod m20231002_143323_remove_integrations;
pub struct Migrator;
#[async_trait::async_trait]
impl MigratorTrait for Migrator {
fn migrations() -> Vec<Box<dyn MigrationTrait>> {
vec![
Box::new(m20230531_180824_drop_reversi::Migration),
Box::new(m20230627_185451_index_note_url::Migration),
Box::new(m20230709_000510_move_antenna_to_cache::Migration),
Box::new(m20230806_170616_fix_antenna_stream_ids::Migration),
Box::new(m20230904_013244_is_indexable::Migration),
Box::new(m20231002_143323_remove_integrations::Migration),
]
}
}
pub fn get_timestamp(id: &str) -> i64 {
const TIME_2000: i64 = 946_684_800_000;
let n: Option<u64> = BASE36.decode_var_len(&id[0..8]);
match n {
None => -1,
Some(n) => n as i64 + TIME_2000,
}
}

51
packages/backend/native-utils/migration/src/m20230531_180824_drop_reversi.rs
View file

@ -1,51 +0,0 @@
use sea_orm_migration::{
prelude::*,
sea_orm::{DbBackend, Statement},
};
#[derive(DeriveMigrationName)]
pub struct Migration;
#[async_trait::async_trait]
impl MigrationTrait for Migration {
async fn up(&self, manager: &SchemaManager) -> Result<(), DbErr> {
if manager.get_database_backend() == DbBackend::Sqlite {
return Ok(());
}
let db = manager.get_connection();
db.query_one(Statement::from_string(
DbBackend::Postgres,
Table::drop()
.table(ReversiGame::Table)
.if_exists()
.to_string(PostgresQueryBuilder),
))
.await?;
db.query_one(Statement::from_string(
DbBackend::Postgres,
Table::drop()
.table(ReversiMatching::Table)
.if_exists()
.to_string(PostgresQueryBuilder),
))
.await?;
Ok(())
}
async fn down(&self, _manager: &SchemaManager) -> Result<(), DbErr> {
// Replace the sample below with your own migration scripts
Ok(())
}
}
/// Learn more at https://docs.rs/sea-query#iden
#[derive(Iden)]
enum ReversiGame {
Table,
}
#[derive(Iden)]
enum ReversiMatching {
Table,
}

38
packages/backend/native-utils/migration/src/m20230627_185451_index_note_url.rs
View file

@ -1,38 +0,0 @@
use sea_orm_migration::prelude::*;
#[derive(DeriveMigrationName)]
pub struct Migration;
#[async_trait::async_trait]
impl MigrationTrait for Migration {
async fn up(&self, manager: &SchemaManager) -> Result<(), DbErr> {
manager
.create_index(
Index::create()
.name("IDX_note_url")
.table(Note::Table)
.col(Note::Url)
.if_not_exists()
.to_owned(),
)
.await
}
async fn down(&self, manager: &SchemaManager) -> Result<(), DbErr> {
manager
.drop_index(
Index::drop()
.name("IDX_note_url")
.table(Note::Table)
.to_owned(),
)
.await
}
}
/// Learn more at https://docs.rs/sea-query#iden
#[derive(Iden)]
enum Note {
Table,
Url,
}

248
packages/backend/native-utils/migration/src/m20230709_000510_move_antenna_to_cache.rs
View file

@ -1,248 +0,0 @@
use redis::streams::StreamMaxlen;
use sea_orm::Statement;
use sea_orm_migration::prelude::*;
use std::env;
#[derive(DeriveMigrationName)]
pub struct Migration;
#[async_trait::async_trait]
impl MigrationTrait for Migration {
async fn up(&self, manager: &SchemaManager) -> Result<(), DbErr> {
let cache_url = env::var("CACHE_URL").unwrap();
let skip_copy = env::var("ANTENNA_MIGRATION_SKIP").unwrap_or_default();
let copy_limit = env::var("ANTENNA_MIGRATION_COPY_LIMIT").unwrap_or_default();
let read_limit: u64 = env::var("ANTENNA_MIGRATION_READ_LIMIT")
.unwrap_or("10000".to_string())
.parse()
.unwrap();
let copy_limit: i64 = match copy_limit.parse() {
Ok(limit) => limit,
Err(_) => 0,
};
if skip_copy == "true" {
println!("Skipped antenna migration");
} else {
let prefix = env::var("CACHE_PREFIX").unwrap();
let db = manager.get_connection();
let bk = manager.get_database_backend();
let count_stmt =
Statement::from_string(bk, "SELECT COUNT(1) FROM antenna_note".to_owned());
let total_num = db
.query_one(count_stmt)
.await?
.unwrap()
.try_get_by_index::<i64>(0)?;
let copy_limit = if copy_limit > 0 {
copy_limit
} else {
total_num
};
println!(
"Copying {} out of {} entries in antenna_note.",
copy_limit, total_num
);
let stmt_base = Query::select()
.column((AntennaNote::Table, AntennaNote::Id))
.column(AntennaNote::AntennaId)
.column(AntennaNote::NoteId)
.from(AntennaNote::Table)
.order_by((AntennaNote::Table, AntennaNote::Id), Order::Asc)
.limit(read_limit)
.to_owned();
let mut stmt = stmt_base.clone();
let client = redis::Client::open(cache_url).unwrap();
let mut redis_conn = client.get_connection().unwrap();
let mut remaining = total_num;
let mut pagination: i64 = 0;
loop {
let res = db.query_all(bk.build(&stmt)).await?;
if res.len() == 0 {
break;
}
let val: Vec<(String, String, String)> = res
.iter()
.filter_map(|q| q.try_get_many_by_index().ok())
.collect();
remaining -= val.len() as i64;
if remaining <= copy_limit {
let mut pipe = redis::pipe();
for v in &val {
pipe.xadd_maxlen(
format!("{}:antennaTimeline:{}", prefix, v.1),
StreamMaxlen::Approx(200),
"*",
&[("note", v.2.to_owned())],
)
.ignore();
}
pipe.query::<()>(&mut redis_conn).unwrap_or(());
}
let copied = total_num - remaining;
let copied = std::cmp::min(copied, total_num);
pagination += 1;
if pagination % 10 == 0 {
println!(
"Migrating antenna [{:.2}%]",
(copied as f64 / total_num as f64) * 100_f64,
);
}
if let Some((last_id, _, _)) = val.last() {
stmt = stmt_base
.clone()
.and_where(
Expr::col((AntennaNote::Table, AntennaNote::Id)).gt(last_id.to_owned()),
)
.to_owned();
} else {
break;
}
}
println!("Migrating antenna [100.00%]");
}
manager
.drop_table(
Table::drop()
.table(AntennaNote::Table)
.if_exists()
.to_owned(),
)
.await?;
Ok(())
}
async fn down(&self, manager: &SchemaManager) -> Result<(), DbErr> {
manager
.create_table(
Table::create()
.table(AntennaNote::Table)
.if_not_exists()
.col(
ColumnDef::new(AntennaNote::Id)
.string_len(32)
.not_null()
.primary_key(),
)
.col(
ColumnDef::new(AntennaNote::NoteId)
.string_len(32)
.not_null(),
)
.col(
ColumnDef::new(AntennaNote::AntennaId)
.string_len(32)
.not_null(),
)
.col(
ColumnDef::new(AntennaNote::Read)
.boolean()
.default(false)
.not_null(),
)
.to_owned(),
)
.await?;
manager
.create_index(
Index::create()
.name("IDX_0d775946662d2575dfd2068a5f")
.table(AntennaNote::Table)
.col(AntennaNote::AntennaId)
.if_not_exists()
.to_owned(),
)
.await?;
manager
.create_index(
Index::create()
.name("IDX_bd0397be22147e17210940e125")
.table(AntennaNote::Table)
.col(AntennaNote::NoteId)
.if_not_exists()
.to_owned(),
)
.await?;
manager
.create_index(
Index::create()
.name("IDX_335a0bf3f904406f9ef3dd51c2")
.table(AntennaNote::Table)
.col(AntennaNote::NoteId)
.col(AntennaNote::AntennaId)
.unique()
.if_not_exists()
.to_owned(),
)
.await?;
manager
.create_index(
Index::create()
.name("IDX_9937ea48d7ae97ffb4f3f063a4")
.table(AntennaNote::Table)
.col(AntennaNote::Read)
.if_not_exists()
.to_owned(),
)
.await?;
manager
.create_foreign_key(
ForeignKey::create()
.name("FK_0d775946662d2575dfd2068a5f5")
.from(AntennaNote::Table, AntennaNote::AntennaId)
.to(Antenna::Table, Antenna::Id)
.on_delete(ForeignKeyAction::Cascade)
.to_owned(),
)
.await?;
manager
.create_foreign_key(
ForeignKey::create()
.name("FK_bd0397be22147e17210940e125b")
.from(AntennaNote::Table, AntennaNote::NoteId)
.to(Note::Table, Note::Id)
.on_delete(ForeignKeyAction::Cascade)
.to_owned(),
)
.await?;
Ok(())
}
}
/// Learn more at https://docs.rs/sea-query#iden
#[derive(Iden)]
enum AntennaNote {
Table,
Id,
#[iden = "noteId"]
NoteId,
#[iden = "antennaId"]
AntennaId,
Read,
}
#[derive(Iden)]
enum Antenna {
Table,
Id,
}
#[derive(Iden)]
enum Note {
Table,
Id,
}

60
packages/backend/native-utils/migration/src/m20230806_170616_fix_antenna_stream_ids.rs
View file

@ -1,60 +0,0 @@
use std::env;
use redis::Commands;
use sea_orm_migration::prelude::*;
use crate::get_timestamp;
#[derive(DeriveMigrationName)]
pub struct Migration;
#[async_trait::async_trait]
impl MigrationTrait for Migration {
async fn up(&self, _manager: &SchemaManager) -> Result<(), DbErr> {
// Replace the sample below with your own migration scripts
let cache_url = env::var("CACHE_URL").unwrap();
let prefix = env::var("CACHE_PREFIX").unwrap();
let client = redis::Client::open(cache_url).unwrap();
let mut redis_conn = client.get_connection().unwrap();
let keys: Vec<String> = redis_conn
.keys(format!("{}:antennaTimeline:*", prefix))
.unwrap();
let key_len = keys.len();
println!(
"Fixing corrupted stream IDs: {} timelines to be fixed",
key_len
);
for (i, key) in keys.iter().enumerate() {
let all_elems: Vec<Vec<Vec<String>>> = redis_conn.xrange_all(key).unwrap(); // Get all post IDs in stream
let stream_ids = all_elems
.iter()
.map(|v| format!("{}-*", get_timestamp(&v[1][1]))); // Get correct stream id with timestamp
redis_conn.del::<_, ()>(key).unwrap();
for (j, v) in stream_ids.enumerate() {
redis_conn
.xadd(key, v, &[("note", &all_elems[j][1][1])])
.unwrap_or(());
}
if i % 10 == 0 {
println!(
"Fixing streams [{:.2}%]",
(i as f64 / key_len as f64) * 100_f64
);
}
}
println!("Fixing streams [100.00%]");
Ok(())
}
async fn down(&self, _manager: &SchemaManager) -> Result<(), DbErr> {
// Replace the sample below with your own migration scripts
Ok(())
}
}

74
packages/backend/native-utils/migration/src/m20230904_013244_is_indexable.rs
View file

@ -1,74 +0,0 @@
use sea_orm_migration::prelude::*;
#[derive(DeriveMigrationName)]
pub struct Migration;
#[async_trait::async_trait]
impl MigrationTrait for Migration {
async fn up(&self, manager: &SchemaManager) -> Result<(), DbErr> {
manager
.alter_table(
Table::alter()
.table(User::Table)
.add_column(
ColumnDef::new(User::IsIndexable)
.boolean()
.not_null()
.default(true),
)
.to_owned(),
)
.await?;
manager
.alter_table(
Table::alter()
.table(UserProfile::Table)
.add_column(
ColumnDef::new(UserProfile::IsIndexable)
.boolean()
.not_null()
.default(true),
)
.to_owned(),
)
.await?;
Ok(())
}
async fn down(&self, manager: &SchemaManager) -> Result<(), DbErr> {
manager
.alter_table(
Table::alter()
.table(User::Table)
.drop_column(User::IsIndexable)
.to_owned(),
)
.await?;
manager
.alter_table(
Table::alter()
.table(UserProfile::Table)
.drop_column(UserProfile::IsIndexable)
.to_owned(),
)
.await?;
Ok(())
}
}
/// Learn more at https://docs.rs/sea-query#iden
#[derive(Iden)]
enum User {
Table,
#[iden = "isIndexable"]
IsIndexable,
}
#[derive(Iden)]
enum UserProfile {
Table,
#[iden = "isIndexable"]
IsIndexable,
}

117
packages/backend/native-utils/migration/src/m20231002_143323_remove_integrations.rs
View file

@ -1,117 +0,0 @@
use sea_orm_migration::prelude::*;
#[derive(DeriveMigrationName)]
pub struct Migration;
#[async_trait::async_trait]
impl MigrationTrait for Migration {
async fn up(&self, manager: &SchemaManager) -> Result<(), DbErr> {
manager
.alter_table(
Table::alter()
.table(UserProfile::Table)
.drop_column(UserProfile::Integrations)
.to_owned(),
)
.await?;
manager
.alter_table(
Table::alter()
.table(Meta::Table)
.drop_column(Meta::EnableTwitterIntegration)
.drop_column(Meta::TwitterConsumerKey)
.drop_column(Meta::TwitterConsumerSecret)
.drop_column(Meta::EnableGithubIntegration)
.drop_column(Meta::GithubClientId)
.drop_column(Meta::GithubClientSecret)
.drop_column(Meta::EnableDiscordIntegration)
.drop_column(Meta::DiscordClientId)
.drop_column(Meta::DiscordClientSecret)
.to_owned(),
)
.await?;
Ok(())
}
async fn down(&self, manager: &SchemaManager) -> Result<(), DbErr> {
manager
.alter_table(
Table::alter()
.table(Meta::Table)
.add_column(ColumnDef::new(Meta::DiscordClientSecret).string())
.add_column(ColumnDef::new(Meta::DiscordClientId).string())
.add_column(
ColumnDef::new(Meta::EnableDiscordIntegration)
.boolean()
.not_null()
.default(false),
)
.add_column(ColumnDef::new(Meta::GithubClientSecret).string())
.add_column(ColumnDef::new(Meta::GithubClientId).string())
.add_column(
ColumnDef::new(Meta::EnableGithubIntegration)
.boolean()
.not_null()
.default(false),
)
.add_column(ColumnDef::new(Meta::TwitterConsumerSecret).string())
.add_column(ColumnDef::new(Meta::TwitterConsumerKey).string())
.add_column(
ColumnDef::new(Meta::EnableTwitterIntegration)
.boolean()
.not_null()
.default(false),
)
.to_owned(),
)
.await?;
manager
.alter_table(
Table::alter()
.table(UserProfile::Table)
.add_column(
ColumnDef::new(UserProfile::Integrations)
.json()
.default("{}"),
)
.to_owned(),
)
.await?;
Ok(())
}
}
#[derive(Iden)]
enum UserProfile {
Table,
#[iden = "integrations"]
Integrations,
}
#[derive(Iden)]
enum Meta {
Table,
#[iden = "enableTwitterIntegration"]
EnableTwitterIntegration,
#[iden = "twitterConsumerKey"]
TwitterConsumerKey,
#[iden = "twitterConsumerSecret"]
TwitterConsumerSecret,
#[iden = "enableGithubIntegration"]
EnableGithubIntegration,
#[iden = "githubClientId"]
GithubClientId,
#[iden = "githubClientSecret"]
GithubClientSecret,
#[iden = "enableDiscordIntegration"]
EnableDiscordIntegration,
#[iden = "discordClientId"]
DiscordClientId,
#[iden = "discordClientSecret"]
DiscordClientSecret,
}

109
packages/backend/native-utils/migration/src/main.rs
View file

@ -1,109 +0,0 @@
use serde::Deserialize;
use std::env;
use std::fs;
use urlencoding::encode;
use sea_orm_migration::prelude::*;
const DB_URL_ENV: &str = "DATABASE_URL";
const CACHE_URL_ENV: &str = "CACHE_URL";
const CACHE_PREFIX_ENV: &str = "CACHE_PREFIX";
#[cfg(feature = "convert")]
mod vec_to_json;
#[tokio::main]
async fn main() {
let cwd = env::current_dir().unwrap();
let yml = fs::File::open(cwd.join("../../.config/default.yml"))
.expect("Failed to open '.config/default.yml'");
let config: Config = serde_yaml::from_reader(yml).expect("Failed to parse yaml");
if env::var_os(DB_URL_ENV).is_none() {
env::set_var(
DB_URL_ENV,
format!(
"postgres://{}:{}@{}:{}/{}",
config.db.user,
encode(&config.db.pass),
config.db.host,
config.db.port,
config.db.db,
),
);
};
if env::var_os(CACHE_URL_ENV).is_none() {
let redis_conf = match config.cache_server {
None => config.redis,
Some(conf) => conf,
};
let redis_proto = match redis_conf.tls {
None => "redis",
Some(_) => "rediss",
};
let redis_user = redis_conf.user.unwrap_or("default".to_string());
let redis_uri_userpass = format!(
"{}:{}",
redis_user,
encode(&redis_conf.pass.unwrap_or_default())
);
let redis_uri_hostport = format!("{}:{}", redis_conf.host, redis_conf.port);
let redis_uri = format!(
"{}://{}@{}/{}",
redis_proto, redis_uri_userpass, redis_uri_hostport, redis_conf.db
);
env::set_var(CACHE_URL_ENV, redis_uri);
env::set_var(
CACHE_PREFIX_ENV,
if redis_conf.prefix.is_empty() {
config.url.host_str().unwrap()
} else {
&redis_conf.prefix
},
);
}
cli::run_cli(migration::Migrator).await;
#[cfg(feature = "convert")]
vec_to_json::convert().await;
}
#[derive(Debug, PartialEq, Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct Config {
pub url: url::Url,
pub db: DbConfig,
pub redis: RedisConfig,
pub cache_server: Option<RedisConfig>,
}
#[derive(Debug, PartialEq, Deserialize)]
pub struct DbConfig {
pub host: String,
pub port: u32,
pub db: String,
pub user: String,
pub pass: String,
}
#[derive(Debug, PartialEq, Deserialize)]
pub struct RedisConfig {
pub host: String,
pub port: u32,
pub user: Option<String>,
pub pass: Option<String>,
pub tls: Option<TlsConfig>,
#[serde(default)]
pub db: u32,
#[serde(default)]
pub prefix: String,
}
#[derive(Debug, PartialEq, Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct TlsConfig {
pub host: String,
pub reject_unauthorized: bool,
}

498
packages/backend/native-utils/migration/src/vec_to_json.rs
View file

@ -1,498 +0,0 @@
use indicatif::{MultiProgress, ProgressBar, ProgressStyle};
use native_utils::model::entity::newtype::{I32Vec, StringVec};
use sea_orm_migration::{
prelude::*,
sea_orm::{Database, DbBackend, DbConn, Statement, TryGetable},
};
use serde_json::json;
use std::env;
use std::time::Duration;
pub async fn convert() {
let uri = env::var("DATABASE_URL").expect("Environment variable 'DATABASE_URL' not set");
let db = Database::connect(uri).await.expect("Unable to connect");
let mp = MultiProgress::new();
let handlers = vec![
tokio::spawn(to_json::<AccessToken, Vec<String>, StringVec>(
db.clone(),
mp.clone(),
AccessToken::Table,
AccessToken::Id,
AccessToken::Permission,
)),
tokio::spawn(to_json::<Antenna, Vec<String>, StringVec>(
db.clone(),
mp.clone(),
Antenna::Table,
Antenna::Id,
Antenna::Users,
)),
tokio::spawn(to_json::<App, Vec<String>, StringVec>(
db.clone(),
mp.clone(),
App::Table,
App::Id,
App::Permission,
)),
tokio::spawn(to_json::<Emoji, Vec<String>, StringVec>(
db.clone(),
mp.clone(),
Emoji::Table,
Emoji::Id,
Emoji::Aliases,
)),
tokio::spawn(to_json::<GalleryPost, Vec<String>, StringVec>(
db.clone(),
mp.clone(),
GalleryPost::Table,
GalleryPost::Id,
GalleryPost::FileIds,
)),
tokio::spawn(to_json::<GalleryPost, Vec<String>, StringVec>(
db.clone(),
mp.clone(),
GalleryPost::Table,
GalleryPost::Id,
GalleryPost::Tags,
)),
tokio::spawn(to_json::<Hashtag, Vec<String>, StringVec>(
db.clone(),
mp.clone(),
Hashtag::Table,
Hashtag::Id,
Hashtag::MentionedUserIds,
)),
tokio::spawn(to_json::<Hashtag, Vec<String>, StringVec>(
db.clone(),
mp.clone(),
Hashtag::Table,
Hashtag::Id,
Hashtag::MentionedLocalUserIds,
)),
tokio::spawn(to_json::<Hashtag, Vec<String>, StringVec>(
db.clone(),
mp.clone(),
Hashtag::Table,
Hashtag::Id,
Hashtag::MentionedRemoteUserIds,
)),
tokio::spawn(to_json::<Hashtag, Vec<String>, StringVec>(
db.clone(),
mp.clone(),
Hashtag::Table,
Hashtag::Id,
Hashtag::AttachedUserIds,
)),
tokio::spawn(to_json::<Hashtag, Vec<String>, StringVec>(
db.clone(),
mp.clone(),
Hashtag::Table,
Hashtag::Id,
Hashtag::AttachedLocalUserIds,
)),
tokio::spawn(to_json::<Hashtag, Vec<String>, StringVec>(
db.clone(),
mp.clone(),
Hashtag::Table,
Hashtag::Id,
Hashtag::AttachedRemoteUserIds,
)),
tokio::spawn(to_json::<MessagingMessage, Vec<String>, StringVec>(
db.clone(),
mp.clone(),
MessagingMessage::Table,
MessagingMessage::Id,
MessagingMessage::Reads,
)),
tokio::spawn(to_json::<Meta, Vec<String>, StringVec>(
db.clone(),
mp.clone(),
Meta::Table,
Meta::Id,
Meta::Langs,
)),
tokio::spawn(to_json::<Meta, Vec<String>, StringVec>(
db.clone(),
mp.clone(),
Meta::Table,
Meta::Id,
Meta::BlockedHosts,
)),
tokio::spawn(to_json::<Meta, Vec<String>, StringVec>(
db.clone(),
mp.clone(),
Meta::Table,
Meta::Id,
Meta::HiddenTags,
)),
tokio::spawn(to_json::<Meta, Vec<String>, StringVec>(
db.clone(),
mp.clone(),
Meta::Table,
Meta::Id,
Meta::PinnedUsers,
)),
tokio::spawn(to_json::<Meta, Vec<String>, StringVec>(
db.clone(),
mp.clone(),
Meta::Table,
Meta::Id,
Meta::PinnedPages,
)),
tokio::spawn(to_json::<Meta, Vec<String>, StringVec>(
db.clone(),
mp.clone(),
Meta::Table,
Meta::Id,
Meta::RecommendedInstances,
)),
tokio::spawn(to_json::<Meta, Vec<String>, StringVec>(
db.clone(),
mp.clone(),
Meta::Table,
Meta::Id,
Meta::SilencedHosts,
)),
tokio::spawn(to_json::<Note, Vec<String>, StringVec>(
db.clone(),
mp.clone(),
Note::Table,
Note::Id,
Note::FileIds,
)),
tokio::spawn(to_json::<Note, Vec<String>, StringVec>(
db.clone(),
mp.clone(),
Note::Table,
Note::Id,
Note::AttachedFileTypes,
)),
tokio::spawn(to_json::<Note, Vec<String>, StringVec>(
db.clone(),
mp.clone(),
Note::Table,
Note::Id,
Note::VisibleUserIds,
)),
tokio::spawn(to_json::<Note, Vec<String>, StringVec>(
db.clone(),
mp.clone(),
Note::Table,
Note::Id,
Note::Mentions,
)),
tokio::spawn(to_json::<Note, Vec<String>, StringVec>(
db.clone(),
mp.clone(),
Note::Table,
Note::Id,
Note::Emojis,
)),
tokio::spawn(to_json::<Note, Vec<String>, StringVec>(
db.clone(),
mp.clone(),
Note::Table,
Note::Id,
Note::Tags,
)),
tokio::spawn(to_json::<NoteEdit, Vec<String>, StringVec>(
db.clone(),
mp.clone(),
NoteEdit::Table,
NoteEdit::Id,
NoteEdit::FileIds,
)),
tokio::spawn(to_json::<Page, Vec<String>, StringVec>(
db.clone(),
mp.clone(),
Page::Table,
Page::Id,
Page::VisibleUserIds,
)),
tokio::spawn(to_json::<RegistryItem, Vec<String>, StringVec>(
db.clone(),
mp.clone(),
RegistryItem::Table,
RegistryItem::Id,
RegistryItem::Scope,
)),
tokio::spawn(to_json::<User, Vec<String>, StringVec>(
db.clone(),
mp.clone(),
User::Table,
User::Id,
User::Tags,
)),
tokio::spawn(to_json::<User, Vec<String>, StringVec>(
db.clone(),
mp.clone(),
User::Table,
User::Id,
User::Emojis,
)),
tokio::spawn(to_json::<Webhook, Vec<String>, StringVec>(
db.clone(),
mp.clone(),
Webhook::Table,
Webhook::Id,
Webhook::On,
)),
tokio::spawn(to_json::<Poll, Vec<String>, StringVec>(
db.clone(),
mp.clone(),
Poll::Table,
Poll::NoteId,
Poll::Choices,
)),
tokio::spawn(to_json::<Poll, Vec<i32>, I32Vec>(
db.clone(),
mp.clone(),
Poll::Table,
Poll::NoteId,
Poll::Votes,
)),
tokio::spawn(to_json::<UserProfile, Vec<String>, StringVec>(
db.clone(),
mp.clone(),
UserProfile::Table,
UserProfile::UserId,
UserProfile::MutingNotificationTypes,
)),
];
futures::future::join_all(handlers).await;
}
fn select_query<T: Iden + 'static>(table: T, id: T, col: T) -> String {
Query::select()
.column(id)
.column(col)
.from(table)
.to_string(PostgresQueryBuilder)
}
async fn get_vec<T: TryGetable>(db: &DbConn, query: String) -> Result<Vec<(String, T)>, DbErr> {
let res: Vec<(String, T)> = db
.query_all(Statement::from_string(DbBackend::Postgres, query))
.await?
.iter()
.filter_map(|r| r.try_get_many_by_index().ok())
.collect();
Ok(res)
}
async fn convert_col<T: Iden + Clone + 'static>(
db: &DbConn,
table: T,
col: T,
) -> Result<(), DbErr> {
let stmt = Table::alter()
.table(table)
.drop_column(col.to_owned())
.add_column(
ColumnDef::new(col.to_owned())
.json_binary()
.not_null()
.default(json!([])),
)
.to_string(PostgresQueryBuilder);
db.query_one(Statement::from_string(DbBackend::Postgres, stmt))
.await?;
Ok(())
}
async fn to_json<T, U, V>(
db: DbConn,
mp: MultiProgress,
table: T,
id: T,
col: T,
) -> Result<(), DbErr>
where
T: Iden + Clone + 'static,
U: TryGetable + IntoIterator + Clone,
V: From<U> + Into<SimpleExpr>,
{
let query = select_query(table.clone(), id.clone(), col.clone());
let loading = ProgressBar::new_spinner()
.with_style(ProgressStyle::with_template("{prefix} {msg} {spinner}").unwrap())
.with_prefix("[-]")
.with_message(format!(
"Loading data from {}.{}",
table.to_string(),
col.to_string()
));
let loading = mp.add(loading);
loading.enable_steady_tick(Duration::from_millis(100));
let res = get_vec::<U>(&db, query).await?;
let models: Vec<(String, V)> = res
.iter()
.filter(|(_, r)| r.clone().into_iter().count() > 0)
.map(|(id, r)| (id.clone(), <V>::from(r.clone())))
.collect();
loading.finish_and_clear();
convert_col(&db, table.clone(), col.clone()).await?;
let progress = ProgressBar::new(models.len() as u64)
.with_style(
ProgressStyle::with_template("{prefix} {msg} {wide_bar} {pos}/{len}")
.unwrap()
.progress_chars("##-"),
)
.with_prefix("[*]")
.with_message(format!("Copying {}.{}", table.to_string(), col.to_string()));
let progress = mp.add(progress);
for model in models {
progress.inc(1);
let q = Query::update()
.table(table.clone())
.values([(col.clone(), model.1.into())])
.and_where(Expr::col(id.clone()).eq(model.0))
.to_string(PostgresQueryBuilder);
db.query_one(Statement::from_string(DbBackend::Postgres, q))
.await?;
}
progress.finish_with_message(format!("Done {}.{}", table.to_string(), col.to_string()));
Ok(())
}
#[derive(Iden, Clone)]
enum AccessToken {
Table,
Id,
Permission,
}
#[derive(Iden, Clone)]
enum Antenna {
Table,
Id,
Users,
}
#[derive(Iden, Clone)]
enum App {
Table,
Id,
Permission,
}
#[derive(Iden, Clone)]
enum Emoji {
Table,
Id,
Aliases,
}
#[derive(Iden, Clone)]
enum GalleryPost {
Table,
Id,
#[iden = "fileIds"]
FileIds,
Tags,
}
#[derive(Iden, Clone)]
enum Hashtag {
Table,
Id,
#[iden = "mentionedUserIds"]
MentionedUserIds,
#[iden = "mentionedLocalUserIds"]
MentionedLocalUserIds,
#[iden = "mentionedRemoteUserIds"]
MentionedRemoteUserIds,
#[iden = "attachedUserIds"]
AttachedUserIds,
#[iden = "attachedLocalUserIds"]
AttachedLocalUserIds,
#[iden = "attachedRemoteUserIds"]
AttachedRemoteUserIds,
}
#[derive(Iden, Clone)]
enum MessagingMessage {
Table,
Id,
Reads,
}
#[derive(Iden, Clone)]
enum Meta {
Table,
Id,
Langs,
#[iden = "hiddenTags"]
HiddenTags,
#[iden = "blockedHosts"]
BlockedHosts,
#[iden = "pinnedUsers"]
PinnedUsers,
#[iden = "pinnedPages"]
PinnedPages,
#[iden = "recommendedInstances"]
RecommendedInstances,
#[iden = "silencedHosts"]
SilencedHosts,
}
#[derive(Iden, Clone)]
enum Note {
Table,
Id,
#[iden = "fileIds"]
FileIds,
#[iden = "attachedFileTypes"]
AttachedFileTypes,
#[iden = "visibleUserIds"]
VisibleUserIds,
Mentions,
Emojis,
Tags,
}
#[derive(Iden, Clone)]
enum NoteEdit {
Table,
Id,
#[iden = "fileIds"]
FileIds,
}
#[derive(Iden, Clone)]
enum Page {
Table,
Id,
#[iden = "visibleUserIds"]
VisibleUserIds,
}
#[derive(Iden, Clone)]
enum Poll {
Table,
#[iden = "noteId"]
NoteId,
Choices,
Votes, // I32Vec
}
#[derive(Iden, Clone)]
enum RegistryItem {
Table,
Id,
Scope,
}
#[derive(Iden, Clone)]
enum User {
Table,
Id,
Tags,
Emojis,
}
#[derive(Iden, Clone)]
enum UserProfile {
Table,
#[iden = "userId"]
UserId,
#[iden = "mutingNotificationTypes"]
MutingNotificationTypes,
}
#[derive(Iden, Clone)]
enum Webhook {
Table,
Id,
On,
}

8
packages/backend/native-utils/package.json
View file

@ -34,12 +34,8 @@
},
"scripts": {
"artifacts": "napi artifacts",
"build": "pnpm run build:napi && pnpm run build:migration",
"build:napi": "napi build --features napi --platform --release ./built/",
"build:migration": "pnpm run build:migration:cargo && pnpm run build:migration:copy",
"build:migration:cargo": "cargo build --locked --release --manifest-path ./migration/Cargo.toml",
"build:migration:copy": "cp -v ./target/release/migration ./built/migration",
"build:debug": "napi build --features napi --platform ./built/ && cargo build --locked --manifest-path ./migration/Cargo.toml && cp -v ./target/debug/migration ./built/migration",
"build": "napi build --features napi --platform --release ./built/",
"build:debug": "napi build --features napi --platform ./built/",
"prepublishOnly": "napi prepublish -t npm",
"universal": "napi universal",
"version": "napi version",

8
packages/backend/package.json
View file

@ -6,12 +6,8 @@
"scripts": {
"start": "pnpm node ./built/index.js",
"start:test": "NODE_ENV=test pnpm node ./built/index.js",
"migrate": "pnpm run migrate:typeorm && pnpm run migrate:cargo",
"migrate:typeorm": "typeorm migration:run -d ormconfig.js",
"migrate:cargo": "./native-utils/built/migration up",
"revertmigration": "pnpm run revertmigration:cargo && pnpm run revertmigration:typeorm",
"revertmigration:typeorm": "typeorm migration:revert -d ormconfig.js",
"revertmigration:cargo": "./native-utils/built/migration down",
"migrate": "typeorm migration:run -d ormconfig.js",
"revertmigration": "typeorm migration:revert -d ormconfig.js",
"check:connect": "node ./check_connect.js",
"build": "pnpm swc src -d built -D",
"build:debug": "pnpm swc src -d built -s -D",