From bc5f233b9ae29c14b3c816d1e4beed8c7a84adf1 Mon Sep 17 00:00:00 2001 From: naskya Date: Wed, 13 Mar 2024 17:02:05 +0900 Subject: [PATCH] refactor (backend): add note_file table to store (noteId, fileId) --- packages/backend-rs/src/model/entity/mod.rs | 1 + .../backend-rs/src/model/entity/note_file.rs | 20 ++++++++++++ .../backend-rs/src/model/entity/prelude.rs | 1 + .../src/model/entity/sea_orm_active_enums.rs | 4 +-- .../src/model/entity/user_profile.rs | 1 - packages/backend/src/db/postgre.ts | 2 ++ .../src/migration/1698420787202-pgroonga.ts | 13 -------- ...708452631156-drop-user-profile-language.ts | 15 --------- .../src/migration/1710304584214-note-file.ts | 32 +++++++++++++++++++ packages/backend/src/misc/schema.ts | 2 ++ .../backend/src/models/entities/note-file.ts | 25 +++++++++++++++ packages/backend/src/models/entities/note.ts | 1 + packages/backend/src/models/index.ts | 2 ++ .../src/models/repositories/note-file.ts | 4 +++ .../backend/src/models/schema/note-file.ts | 19 +++++++++++ .../backend/src/services/drive/delete-file.ts | 15 ++++++++- packages/backend/src/services/note/create.ts | 9 ++++++ packages/backend/src/services/note/delete.ts | 31 +++++++++++++++--- 18 files changed, 161 insertions(+), 36 deletions(-) create mode 100644 packages/backend-rs/src/model/entity/note_file.rs delete mode 100644 packages/backend/src/migration/1708452631156-drop-user-profile-language.ts create mode 100644 packages/backend/src/migration/1710304584214-note-file.ts create mode 100644 packages/backend/src/models/entities/note-file.ts create mode 100644 packages/backend/src/models/repositories/note-file.ts create mode 100644 packages/backend/src/models/schema/note-file.ts diff --git a/packages/backend-rs/src/model/entity/mod.rs b/packages/backend-rs/src/model/entity/mod.rs index 38be7561b..223b07fb4 100644 --- a/packages/backend-rs/src/model/entity/mod.rs +++ b/packages/backend-rs/src/model/entity/mod.rs @@ -35,6 +35,7 @@ pub mod muting; pub mod note; pub mod note_edit; pub mod note_favorite; +pub mod note_file; pub mod note_reaction; pub mod note_thread_muting; pub mod note_unread; diff --git a/packages/backend-rs/src/model/entity/note_file.rs b/packages/backend-rs/src/model/entity/note_file.rs new file mode 100644 index 000000000..d5d0d1752 --- /dev/null +++ b/packages/backend-rs/src/model/entity/note_file.rs @@ -0,0 +1,20 @@ +//! `SeaORM` Entity. Generated by sea-orm-codegen 0.12.12 + +use sea_orm::entity::prelude::*; + +#[derive(Clone, Debug, PartialEq, DeriveEntityModel, Eq)] +#[sea_orm(table_name = "note_file")] +#[napi_derive::napi(object, js_name = "NoteFile", use_nullable = true)] +pub struct Model { + #[sea_orm(column_name = "idNum", primary_key)] + pub id_num: i64, + #[sea_orm(column_name = "noteId")] + pub note_id: String, + #[sea_orm(column_name = "fileId")] + pub file_id: String, +} + +#[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)] +pub enum Relation {} + +impl ActiveModelBehavior for ActiveModel {} diff --git a/packages/backend-rs/src/model/entity/prelude.rs b/packages/backend-rs/src/model/entity/prelude.rs index 10d379536..0935ac9ff 100644 --- a/packages/backend-rs/src/model/entity/prelude.rs +++ b/packages/backend-rs/src/model/entity/prelude.rs @@ -33,6 +33,7 @@ pub use super::muting::Entity as Muting; pub use super::note::Entity as Note; pub use super::note_edit::Entity as NoteEdit; pub use super::note_favorite::Entity as NoteFavorite; +pub use super::note_file::Entity as NoteFile; pub use super::note_reaction::Entity as NoteReaction; pub use super::note_thread_muting::Entity as NoteThreadMuting; pub use super::note_unread::Entity as NoteUnread; diff --git a/packages/backend-rs/src/model/entity/sea_orm_active_enums.rs b/packages/backend-rs/src/model/entity/sea_orm_active_enums.rs index 00a253c0a..d4cb07288 100644 --- a/packages/backend-rs/src/model/entity/sea_orm_active_enums.rs +++ b/packages/backend-rs/src/model/entity/sea_orm_active_enums.rs @@ -145,8 +145,8 @@ pub enum UserEmojimodpermEnum { Full, #[sea_orm(string_value = "mod")] Mod, - #[sea_orm(string_value = "unauthorized")] - Unauthorized, + #[sea_orm(string_value = "none")] + None, } #[derive(Debug, PartialEq, Eq, EnumIter, DeriveActiveEnum)] #[napi_derive::napi] diff --git a/packages/backend-rs/src/model/entity/user_profile.rs b/packages/backend-rs/src/model/entity/user_profile.rs index 9b5feb73b..55fa305d8 100644 --- a/packages/backend-rs/src/model/entity/user_profile.rs +++ b/packages/backend-rs/src/model/entity/user_profile.rs @@ -60,7 +60,6 @@ pub struct Model { pub receive_announcement_email: bool, #[sea_orm(column_name = "emailNotificationTypes", column_type = "JsonBinary")] pub email_notification_types: Json, - pub lang: Option, #[sea_orm(column_name = "mutedInstances", column_type = "JsonBinary")] pub muted_instances: Json, #[sea_orm(column_name = "publicReactions")] diff --git a/packages/backend/src/db/postgre.ts b/packages/backend/src/db/postgre.ts index 9ec52bb77..74e681ee4 100644 --- a/packages/backend/src/db/postgre.ts +++ b/packages/backend/src/db/postgre.ts @@ -73,6 +73,7 @@ import { UserPending } from "@/models/entities/user-pending.js"; import { Webhook } from "@/models/entities/webhook.js"; import { UserIp } from "@/models/entities/user-ip.js"; import { NoteEdit } from "@/models/entities/note-edit.js"; +import { NoteFile } from "@/models/entities/note-file.js"; import { entities as charts } from "@/services/chart/entities.js"; import { dbLogger } from "./logger.js"; @@ -143,6 +144,7 @@ export const entities = [ Note, NoteEdit, NoteFavorite, + NoteFile, NoteReaction, NoteWatching, NoteThreadMuting, diff --git a/packages/backend/src/migration/1698420787202-pgroonga.ts b/packages/backend/src/migration/1698420787202-pgroonga.ts index 72a33f2b8..0fe56cf04 100644 --- a/packages/backend/src/migration/1698420787202-pgroonga.ts +++ b/packages/backend/src/migration/1698420787202-pgroonga.ts @@ -13,22 +13,9 @@ export class Pgroonga1698420787202 implements MigrationInterface { ); } -<<<<<<<< HEAD:packages/backend/migration-neko/1698420787202-pgroonga.js - async down(queryRunner) { - await queryRunner.query( - `DROP INDEX "public"."IDX_fcb770976ff8240af5799e3ffc"`, - ); - await queryRunner.query( - `DROP INDEX "public"."IDX_065d4d8f3b5adb4a08841eae3c"`, - ); - await queryRunner.query( - `DROP INDEX "public"."IDX_f27f5d88941e57442be75ba9c8"`, - ); -======== async down(queryRunner: QueryRunner): Promise { await queryRunner.query(`DROP INDEX "IDX_fcb770976ff8240af5799e3ffc"`); await queryRunner.query(`DROP INDEX "IDX_065d4d8f3b5adb4a08841eae3c"`); await queryRunner.query(`DROP INDEX "IDX_f27f5d88941e57442be75ba9c8"`); ->>>>>>>> bf9916740 (refactor (backend): move migrations inside backend/src):packages/backend/src/migration/1698420787202-pgroonga.ts } } diff --git a/packages/backend/src/migration/1708452631156-drop-user-profile-language.ts b/packages/backend/src/migration/1708452631156-drop-user-profile-language.ts deleted file mode 100644 index 568542cb6..000000000 --- a/packages/backend/src/migration/1708452631156-drop-user-profile-language.ts +++ /dev/null @@ -1,15 +0,0 @@ -import { MigrationInterface, QueryRunner } from "typeorm"; - -export class DropUserProfileLanguage1708452631156 - implements MigrationInterface -{ - async up(queryRunner: QueryRunner): Promise { - await queryRunner.query(`ALTER TABLE "user_profile" DROP COLUMN "lang"`); - } - - async down(queryRunner: QueryRunner): Promise { - await queryRunner.query( - `ALTER TABLE "user_profile" ADD COLUMN "lang" character varying(32)`, - ); - } -} diff --git a/packages/backend/src/migration/1710304584214-note-file.ts b/packages/backend/src/migration/1710304584214-note-file.ts new file mode 100644 index 000000000..d58d2a0cb --- /dev/null +++ b/packages/backend/src/migration/1710304584214-note-file.ts @@ -0,0 +1,32 @@ +import { MigrationInterface, QueryRunner, TableIndex } from "typeorm"; + +export class NoteFile1710304584214 implements MigrationInterface { + name = "NoteFile1710304584214"; + + public async up(queryRunner: QueryRunner): Promise { + await queryRunner.query( + `CREATE TABLE "note_file" ("idNum" bigserial PRIMARY KEY, "noteId" varchar(32) NOT NULL, "fileId" varchar(32) NOT NULL)`, + ); + await queryRunner.query(` + INSERT INTO "note_file" ("noteId", "fileId") SELECT * FROM ( + SELECT "id", UNNEST("fileIds") FROM "note" + ) + `); + await queryRunner.createIndices("note_file", [ + { + name: "IDX_note_file_noteId", + columnNames: ["noteId"], + isUnique: false, + }, + { + name: "IDX_note_file_fileId", + columnNames: ["fileId"], + isUnique: false, + }, + ] as TableIndex[]); + } + + public async down(queryRunner: QueryRunner): Promise { + await queryRunner.dropTable("note_file"); + } +} diff --git a/packages/backend/src/misc/schema.ts b/packages/backend/src/misc/schema.ts index 9d698527c..016b7159c 100644 --- a/packages/backend/src/misc/schema.ts +++ b/packages/backend/src/misc/schema.ts @@ -32,6 +32,7 @@ import { packedQueueCountSchema } from "@/models/schema/queue.js"; import { packedGalleryPostSchema } from "@/models/schema/gallery-post.js"; import { packedEmojiSchema } from "@/models/schema/emoji.js"; import { packedNoteEdit } from "@/models/schema/note-edit.js"; +import { packedNoteFileSchema } from "@/models/schema/note-file.js"; export const refs = { UserLite: packedUserLiteSchema, @@ -47,6 +48,7 @@ export const refs = { App: packedAppSchema, MessagingMessage: packedMessagingMessageSchema, Note: packedNoteSchema, + NoteFile: packedNoteFileSchema, NoteEdit: packedNoteEdit, NoteReaction: packedNoteReactionSchema, NoteFavorite: packedNoteFavoriteSchema, diff --git a/packages/backend/src/models/entities/note-file.ts b/packages/backend/src/models/entities/note-file.ts new file mode 100644 index 000000000..7a7ebd024 --- /dev/null +++ b/packages/backend/src/models/entities/note-file.ts @@ -0,0 +1,25 @@ +import { Entity, Index, Column, PrimaryGeneratedColumn } from "typeorm"; +import { Note } from "./note.js"; +import { DriveFile } from "./drive-file.js"; +import { id } from "../id.js"; + +@Entity() +export class NoteFile { + // We don't use this column but TypeORM requires a primary column + @PrimaryGeneratedColumn("increment") + public idNum: number; + + @Index("IDX_note_file_noteId", { unique: false }) + @Column({ + ...id(), + nullable: false, + }) + public noteId: Note["id"]; + + @Index("IDX_note_file_fileId", { unique: false }) + @Column({ + ...id(), + nullable: false, + }) + public fileId: DriveFile["id"]; +} diff --git a/packages/backend/src/models/entities/note.ts b/packages/backend/src/models/entities/note.ts index a61f167c1..904e1c7dd 100644 --- a/packages/backend/src/models/entities/note.ts +++ b/packages/backend/src/models/entities/note.ts @@ -151,6 +151,7 @@ export class Note { }) public score: number; + // TODO: drop this column and use note_files @Index() @Column({ ...id(), diff --git a/packages/backend/src/models/index.ts b/packages/backend/src/models/index.ts index 8ae12a63d..5d4ff5219 100644 --- a/packages/backend/src/models/index.ts +++ b/packages/backend/src/models/index.ts @@ -66,12 +66,14 @@ import { InstanceRepository } from "./repositories/instance.js"; import { Webhook } from "./entities/webhook.js"; import { UserIp } from "./entities/user-ip.js"; import { NoteEdit } from "./entities/note-edit.js"; +import { NoteFileRepository } from "./repositories/note-file.js"; export const Announcements = db.getRepository(Announcement); export const AnnouncementReads = db.getRepository(AnnouncementRead); export const Apps = AppRepository; export const Notes = NoteRepository; export const NoteEdits = db.getRepository(NoteEdit); +export const NoteFiles = NoteFileRepository; export const NoteFavorites = NoteFavoriteRepository; export const NoteWatchings = db.getRepository(NoteWatching); export const NoteThreadMutings = db.getRepository(NoteThreadMuting); diff --git a/packages/backend/src/models/repositories/note-file.ts b/packages/backend/src/models/repositories/note-file.ts new file mode 100644 index 000000000..f755fb5fe --- /dev/null +++ b/packages/backend/src/models/repositories/note-file.ts @@ -0,0 +1,4 @@ +import { db } from "@/db/postgre.js"; +import { NoteFile } from "@/models/entities/note-file.js"; + +export const NoteFileRepository = db.getRepository(NoteFile).extend({}); diff --git a/packages/backend/src/models/schema/note-file.ts b/packages/backend/src/models/schema/note-file.ts new file mode 100644 index 000000000..87c6ca083 --- /dev/null +++ b/packages/backend/src/models/schema/note-file.ts @@ -0,0 +1,19 @@ +export const packedNoteFileSchema = { + type: "object", + properties: { + noteId: { + type: "string", + optional: false, + nullable: false, + format: "id", + example: "xxxxxxxxxx", + }, + fileId: { + type: "string", + optional: false, + nullable: false, + format: "id", + example: "xxxxxxxxxx", + }, + }, +} as const; diff --git a/packages/backend/src/services/drive/delete-file.ts b/packages/backend/src/services/drive/delete-file.ts index 7d855af2a..5effc5042 100644 --- a/packages/backend/src/services/drive/delete-file.ts +++ b/packages/backend/src/services/drive/delete-file.ts @@ -1,10 +1,14 @@ import type { DriveFile } from "@/models/entities/drive-file.js"; import { InternalStorage } from "./internal-storage.js"; -import { DriveFiles } from "@/models/index.js"; +import { DriveFiles, NoteFiles } from "@/models/index.js"; import { createDeleteObjectStorageFileJob } from "@/queue/index.js"; import { fetchMeta } from "@/misc/backend-rs.js"; import { getS3 } from "./s3.js"; import { v4 as uuid } from "uuid"; +import Logger from "@/services/logger.js"; +import { inspect } from "node:util"; + +const logger = new Logger("delete-file"); export async function deleteFile(file: DriveFile, isExpired = false) { if (file.storedInternal) { @@ -79,6 +83,15 @@ async function postProcess(file: DriveFile, isExpired = false) { } else { DriveFiles.delete(file.id); } + + const noteFiles = await NoteFiles.findBy({ fileId: file.id }); + await Promise.all( + noteFiles.map(async (noteFile) => { + await NoteFiles.delete(noteFile); + }), + ).catch((e) => { + logger.error(inspect(e)); + }); } export async function deleteObjectStorageFile(key: string) { diff --git a/packages/backend/src/services/note/create.ts b/packages/backend/src/services/note/create.ts index 7e8bcabe0..83cb4d8df 100644 --- a/packages/backend/src/services/note/create.ts +++ b/packages/backend/src/services/note/create.ts @@ -33,6 +33,7 @@ import { Channels, ChannelFollowings, NoteThreadMutings, + NoteFiles, } from "@/models/index.js"; import type { DriveFile } from "@/models/entities/drive-file.js"; import type { App } from "@/models/entities/app.js"; @@ -342,6 +343,14 @@ export default async ( const note = await insertNote(user, data, tags, emojis, mentionedUsers); + await Promise.all( + note.fileIds.map(async (fileId) => { + await NoteFiles.insert({ noteId: note.id, fileId }); + }), + ).catch((e) => { + logger.error(inspect(e)); + }); + res(note); // Register host diff --git a/packages/backend/src/services/note/delete.ts b/packages/backend/src/services/note/delete.ts index 9df2ad548..18327459f 100644 --- a/packages/backend/src/services/note/delete.ts +++ b/packages/backend/src/services/note/delete.ts @@ -8,7 +8,7 @@ import renderTombstone from "@/remote/activitypub/renderer/tombstone.js"; import config from "@/config/index.js"; import type { User, ILocalUser, IRemoteUser } from "@/models/entities/user.js"; import type { Note, IMentionedRemoteUsers } from "@/models/entities/note.js"; -import { Notes, Users, Instances } from "@/models/index.js"; +import { Notes, NoteFiles, Users, Instances } from "@/models/index.js"; import { deliverToFollowers, deliverToUser, @@ -16,7 +16,10 @@ import { import { hasOtherRenoteOfThisNote } from "@/misc/backend-rs.js"; import { registerOrFetchInstanceDoc } from "@/services/register-or-fetch-instance-doc.js"; import { deliverToRelays } from "@/services/relay.js"; -// import meilisearch from "@/db/meilisearch.js"; +import Logger from "@/services/logger.js"; +import { inspect } from "node:util"; + +const logger = new Logger("delete-note"); /** * 投稿を削除します。 @@ -87,11 +90,23 @@ export default async function ( deliverToConcerned(user, note, content); } + const cascadingNotes = await findCascadingNotes(note); + + for await (const cascadingNote of cascadingNotes) { + await Promise.all( + cascadingNote.fileIds.map(async (fileId) => { + await NoteFiles.delete({ noteId: cascadingNote.id, fileId }); + }), + ).catch((e) => { + logger.error(inspect(e)); + }); + } + // also deliever delete activity to cascaded notes - const cascadingNotes = (await findCascadingNotes(note)).filter( + const cascadingFederatedNotes = cascadingNotes.filter( (note) => !note.localOnly, ); // filter out local-only notes - for (const cascadingNote of cascadingNotes) { + for (const cascadingNote of cascadingFederatedNotes) { if (!cascadingNote.user) continue; if (!Users.isLocalUser(cascadingNote.user)) continue; const content = renderActivity( @@ -116,6 +131,14 @@ export default async function ( id: note.id, userId: user.id, }); + + await Promise.all( + note.fileIds.map(async (fileId) => { + await NoteFiles.delete({ noteId: note.id, fileId }); + }), + ).catch((e) => { + logger.error(inspect(e)); + }); } // if (meilisearch) {