Firefish v1.0.5-dev5

This commit is contained in:
naskya 2023-08-10 04:45:41 +09:00
parent d4fb73de27
commit 23f0e8532f
Signed by: naskya
GPG key ID: 164DFF24E2D40139
38 changed files with 2925 additions and 1655 deletions

View file

@ -196,6 +196,12 @@ version = "0.21.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "604178f6c5c21f02dc555784810edfb88d34ac2c73b2eae109655649ee73ce3d"
[[package]]
name = "basen"
version = "0.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1dbe4bb73fd931c4d1aaf53b35d1286c8a948ad00ec92c8e3c856f15fd027f43"
[[package]]
name = "bigdecimal"
version = "0.3.1"
@ -1399,6 +1405,7 @@ name = "native-utils"
version = "0.0.0"
dependencies = [
"async-trait",
"basen",
"cfg-if",
"chrono",
"cuid2",
@ -1410,7 +1417,6 @@ dependencies = [
"once_cell",
"parse-display",
"pretty_assertions",
"radix_fmt",
"rand",
"schemars",
"sea-orm",
@ -1831,12 +1837,6 @@ version = "0.7.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "dc33ff2d4973d518d823d61aa239014831e521c75da58e3df4840d3f47749d09"
[[package]]
name = "radix_fmt"
version = "1.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ce082a9940a7ace2ad4a8b7d0b1eac6aa378895f18be598230c5f2284ac05426"
[[package]]
name = "rand"
version = "0.8.5"

View file

@ -31,11 +31,11 @@ serde_json = "1.0.96"
thiserror = "1.0.40"
tokio = { version = "1.28.1", features = ["full"] }
utoipa = "3.3.0"
radix_fmt = "1.0.0"
# Default enable napi4 feature, see https://nodejs.org/api/n-api.html#node-api-version-matrix
napi = { version = "2.13.1", default-features = false, features = ["napi6", "tokio_rt"], optional = true }
napi-derive = { version = "2.12.0", optional = true }
basen = "0.1.0"
[dev-dependencies]
pretty_assertions = "1.3.0"

View file

@ -10,11 +10,11 @@ path = "src/lib.rs"
[features]
default = []
convert = ["dep:native-utils", "dep:indicatif", "dep:futures"]
convert = ["dep:indicatif", "dep:futures"]
[dependencies]
serde_json = "1.0.96"
native-utils = { path = "../", optional = true }
native-utils = { path = "../" }
indicatif = { version = "0.17.4", features = ["tokio"], optional = true }
tokio = { version = "1.28.2", features = ["full"] }
futures = { version = "0.3.28", optional = true }

View file

@ -3,6 +3,7 @@ pub use sea_orm_migration::prelude::*;
mod m20230531_180824_drop_reversi;
mod m20230627_185451_index_note_url;
mod m20230709_000510_move_antenna_to_cache;
mod m20230806_170616_fix_antenna_stream_ids;
pub struct Migrator;
@ -13,6 +14,7 @@ impl MigratorTrait for Migrator {
Box::new(m20230531_180824_drop_reversi::Migration),
Box::new(m20230627_185451_index_note_url::Migration),
Box::new(m20230709_000510_move_antenna_to_cache::Migration),
Box::new(m20230806_170616_fix_antenna_stream_ids::Migration),
]
}
}

View file

@ -85,7 +85,7 @@ impl MigrationTrait for Migration {
)
.ignore();
}
pipe.query::<()>(&mut redis_conn).unwrap();
pipe.query::<()>(&mut redis_conn).unwrap_or(());
}
let copied = total_num - remaining;

View file

@ -0,0 +1,59 @@
use std::env;
use native_utils::util::id;
use redis::Commands;
use sea_orm_migration::prelude::*;
#[derive(DeriveMigrationName)]
pub struct Migration;
#[async_trait::async_trait]
impl MigrationTrait for Migration {
async fn up(&self, _manager: &SchemaManager) -> Result<(), DbErr> {
// Replace the sample below with your own migration scripts
let cache_url = env::var("CACHE_URL").unwrap();
let prefix = env::var("CACHE_PREFIX").unwrap();
let client = redis::Client::open(cache_url).unwrap();
let mut redis_conn = client.get_connection().unwrap();
let keys: Vec<String> = redis_conn
.keys(format!("{}:antennaTimeline:*", prefix))
.unwrap();
let key_len = keys.len();
println!(
"Fixing corrupted stream IDs: {} timelines to be fixed",
key_len
);
for (i, key) in keys.iter().enumerate() {
let all_elems: Vec<Vec<Vec<String>>> = redis_conn.xrange_all(key).unwrap(); // Get all post IDs in stream
let stream_ids = all_elems
.iter()
.map(|v| format!("{}-*", id::get_timestamp(&v[1][1]))); // Get correct stream id with timestamp
redis_conn.del::<_, ()>(key).unwrap();
for (j, v) in stream_ids.enumerate() {
redis_conn
.xadd(key, v, &[("note", &all_elems[j][1][1])])
.unwrap_or(());
}
if i % 10 == 0 {
println!(
"Fixing streams [{:.2}%]",
(i as f64 / key_len as f64) * 100_f64
);
}
}
println!("Fixing streams [100.00%]");
Ok(())
}
async fn down(&self, _manager: &SchemaManager) -> Result<(), DbErr> {
// Replace the sample below with your own migration scripts
Ok(())
}
}

View file

@ -1,9 +1,9 @@
//! ID generation utility based on [cuid2]
use basen::BASE36;
use cfg_if::cfg_if;
use chrono::Utc;
use once_cell::sync::OnceCell;
use radix_fmt::radix_36;
use std::cmp;
use crate::impl_into_napi_error;
@ -46,13 +46,21 @@ pub fn create_id(date_num: i64) -> Result<String, ErrorUninitialized> {
let time = cmp::max(date_num - TIME_2000, 0);
Ok(format!(
"{:0>8}{}",
radix_36(time).to_string(),
BASE36.encode_var_len(&(time as u64)),
gen.create_id()
))
}
}
}
pub fn get_timestamp(id: &str) -> i64 {
let n: Option<u64> = BASE36.decode_var_len(&id[0..8]);
match n {
None => -1,
Some(n) => n as i64 + TIME_2000,
}
}
cfg_if! {
if #[cfg(feature = "napi")] {
use napi_derive::napi;
@ -68,17 +76,23 @@ cfg_if! {
pub fn native_create_id(date_num: i64) -> String {
create_id(date_num).unwrap()
}
#[napi]
pub fn native_get_timestamp(id: String) -> i64 {
get_timestamp(&id)
}
}
}
#[cfg(test)]
mod unit_test {
use crate::util::id;
use chrono::Utc;
use pretty_assertions::{assert_eq, assert_ne};
use std::thread;
#[test]
fn can_generate_unique_ids() {
fn can_create_and_decode() {
assert_eq!(id::create_id(0), Err(id::ErrorUninitialized));
id::init_id(16, "");
assert_eq!(id::create_id(0).unwrap().len(), 16);
@ -86,5 +100,10 @@ mod unit_test {
let id1 = thread::spawn(|| id::create_id(0).unwrap());
let id2 = thread::spawn(|| id::create_id(0).unwrap());
assert_ne!(id1.join().unwrap(), id2.join().unwrap());
let now = Utc::now().timestamp_millis();
let test_id = id::create_id(now).unwrap();
let timestamp = id::get_timestamp(&test_id);
assert_eq!(now, timestamp);
}
}

View file

@ -26,16 +26,16 @@
"@tensorflow/tfjs-node": "3.21.1"
},
"dependencies": {
"@bull-board/api": "5.6.0",
"@bull-board/koa": "5.6.0",
"@bull-board/ui": "5.6.0",
"@bull-board/api": "5.7.2",
"@bull-board/koa": "5.7.2",
"@bull-board/ui": "5.7.2",
"@discordapp/twemoji": "14.1.2",
"@elastic/elasticsearch": "7.17.0",
"@koa/cors": "3.4.3",
"@koa/multer": "3.0.2",
"@koa/router": "9.0.1",
"@peertube/http-signature": "1.7.0",
"@redocly/openapi-core": "1.0.0-beta.131",
"@redocly/openapi-core": "1.0.2",
"@sinonjs/fake-timers": "9.1.2",
"@syuilo/aiscript": "0.11.1",
"@tensorflow/tfjs": "^4.2.0",
@ -49,9 +49,8 @@
"axios": "^1.4.0",
"bcryptjs": "2.4.3",
"blurhash": "2.0.5",
"bull": "4.10.4",
"bull": "4.11.2",
"cacheable-lookup": "7.0.0",
"firefish-js": "workspace:*",
"cbor": "8.1.0",
"chalk": "5.3.0",
"chalk-template": "0.4.0",
@ -65,6 +64,7 @@
"escape-regexp": "0.0.1",
"feed": "4.2.2",
"file-type": "17.1.6",
"firefish-js": "workspace:*",
"fluent-ffmpeg": "2.1.2",
"got": "12.5.3",
"gunzip-maybe": "^1.4.2",
@ -92,18 +92,18 @@
"meilisearch": "0.33.0",
"mfm-js": "0.23.3",
"mime-types": "2.1.35",
"msgpackr": "1.9.5",
"msgpackr": "1.9.6",
"multer": "1.4.4-lts.1",
"native-utils": "link:native-utils",
"nested-property": "4.0.0",
"node-fetch": "3.3.1",
"nodemailer": "6.9.3",
"node-fetch": "3.3.2",
"nodemailer": "6.9.4",
"nsfwjs": "2.4.2",
"oauth": "^0.10.0",
"os-utils": "0.0.14",
"otpauth": "^9.1.3",
"otpauth": "^9.1.4",
"parse5": "7.1.2",
"pg": "8.11.1",
"pg": "8.11.2",
"private-ip": "2.3.4",
"probe-image-size": "7.2.3",
"promise-limit": "2.7.0",
@ -113,37 +113,37 @@
"qs": "6.11.2",
"random-seed": "0.3.0",
"ratelimiter": "3.4.1",
"re2": "1.19.1",
"re2": "1.20.1",
"redis-lock": "0.1.4",
"redis-semaphore": "5.3.1",
"redis-semaphore": "5.4.0",
"reflect-metadata": "0.1.13",
"rename": "1.0.4",
"rndstr": "1.0.0",
"rss-parser": "3.13.0",
"sanitize-html": "2.10.0",
"sanitize-html": "2.11.0",
"seedrandom": "^3.0.5",
"semver": "7.5.4",
"sharp": "0.32.1",
"sharp": "0.32.4",
"sonic-channel": "^1.3.1",
"stringz": "2.1.0",
"summaly": "2.7.0",
"syslog-pro": "1.0.0",
"systeminformation": "5.17.17",
"systeminformation": "5.18.13",
"tar-stream": "^3.1.6",
"tesseract.js": "^3.0.3",
"tinycolor2": "1.5.2",
"tesseract.js": "^4.1.1",
"tinycolor2": "1.6.0",
"tmp": "0.2.1",
"twemoji-parser": "14.0.0",
"typeorm": "0.3.17",
"ulid": "2.3.0",
"uuid": "9.0.0",
"web-push": "3.6.3",
"web-push": "3.6.4",
"websocket": "1.0.34",
"xev": "3.0.2"
},
"devDependencies": {
"@swc/cli": "^0.1.62",
"@swc/core": "^1.3.68",
"@swc/core": "^1.3.75",
"@types/adm-zip": "^0.5.0",
"@types/bcryptjs": "2.4.2",
"@types/cbor": "6.0.0",
@ -153,7 +153,7 @@
"@types/jsdom": "21.1.1",
"@types/jsonld": "1.5.9",
"@types/jsrsasign": "10.5.8",
"@types/koa": "2.13.6",
"@types/koa": "2.13.8",
"@types/koa-bodyparser": "4.3.10",
"@types/koa-cors": "0.0.2",
"@types/koa-favicon": "2.0.21",
@ -167,7 +167,7 @@
"@types/mocha": "9.1.1",
"@types/node": "18.11.18",
"@types/node-fetch": "3.0.3",
"@types/nodemailer": "6.4.8",
"@types/nodemailer": "6.4.9",
"@types/oauth": "0.9.1",
"@types/probe-image-size": "^7.2.0",
"@types/pug": "2.0.6",
@ -188,7 +188,7 @@
"@types/websocket": "1.0.5",
"@types/ws": "8.5.5",
"cross-env": "7.0.3",
"eslint": "^8.44.0",
"eslint": "^8.46.0",
"execa": "6.1.0",
"json5-loader": "4.0.1",
"mocha": "10.2.0",
@ -199,7 +199,7 @@
"ts-node": "10.9.1",
"tsconfig-paths": "4.2.0",
"typescript": "5.1.6",
"webpack": "^5.88.1",
"webpack": "^5.88.2",
"ws": "8.13.0"
}
}

View file

@ -226,6 +226,14 @@ export default hasConfig
return null;
} else if (term.startsWith("domain:")) {
const domain = term.slice(7);
if (
domain.length === 0 ||
domain === "local" ||
domain === config.hostname
) {
constructedFilters.push("userHost NOT EXISTS");
return null;
}
constructedFilters.push(`userHost = ${domain}`);
return null;
} else if (term.startsWith("after:")) {

View file

@ -2,6 +2,7 @@ import config from "@/config/index.js";
import {
nativeCreateId,
nativeInitIdGenerator,
nativeGetTimestamp,
} from "native-utils/built/index.js";
const length = Math.min(Math.max(config.cuid?.length ?? 16, 16), 24);
@ -19,3 +20,7 @@ nativeInitIdGenerator(length, fingerprint);
export function genId(date?: Date): string {
return nativeCreateId((date ?? new Date()).getTime());
}
export function getTimestamp(id: string): number {
return nativeGetTimestamp(id);
}

View file

@ -29,7 +29,7 @@ export async function deleteAccount(
userId: user.id,
...(cursor ? { id: MoreThan(cursor) } : {}),
},
take: 100,
take: 10,
order: {
id: 1,
},

View file

@ -2,7 +2,7 @@ import define from "../../define.js";
import readNote from "@/services/note/read.js";
import { Antennas, Notes } from "@/models/index.js";
import { redisClient } from "@/db/redis.js";
import { genId } from "@/misc/gen-id.js";
import { genId, getTimestamp } from "@/misc/gen-id.js";
import { makePaginationQuery } from "../../common/make-pagination-query.js";
import { generateVisibilityQuery } from "../../common/generate-visibility-query.js";
import { generateMutedUserQuery } from "../../common/generate-muted-user-query.js";
@ -61,10 +61,22 @@ export default define(meta, paramDef, async (ps, user) => {
}
const limit = ps.limit + (ps.untilId ? 1 : 0) + (ps.sinceId ? 1 : 0); // untilIdに指定したものも含まれるため+1
let end = "+";
if (ps.untilDate) {
end = ps.untilDate.toString();
} else if (ps.untilId) {
end = getTimestamp(ps.untilId).toString();
}
let start = "-";
if (ps.sinceDate) {
start = ps.sinceDate.toString();
} else if (ps.sinceId) {
start = getTimestamp(ps.sinceId).toString();
}
const noteIdsRes = await redisClient.xrevrange(
`antennaTimeline:${antenna.id}`,
ps.untilDate ?? "+",
ps.sinceDate ?? "-",
end,
start,
"COUNT",
limit,
);

View file

@ -26,16 +26,15 @@ export const paramDef = {
} as const;
export default define(meta, paramDef, async (ps) => {
const worker = createWorker({
logger: (m) => console.log(m),
});
const worker = await createWorker();
await worker.load();
await worker.loadLanguage("eng");
await worker.initialize("eng");
const {
data: { text },
} = await worker.recognize(ps.url);
} = await worker.recognize(ps.url, {
rotateAuto: true,
});
await worker.terminate();
return text;

View file

@ -14,13 +14,15 @@ import { serverLogger } from "../index.js";
import { isMimeImage } from "@/misc/is-mime-image.js";
export async function proxyMedia(ctx: Koa.Context) {
const url = "url" in ctx.query ? ctx.query.url : `https://${ctx.params.url}`;
let url = "url" in ctx.query ? ctx.query.url : `https://${ctx.params.url}`;
if (typeof url !== "string") {
ctx.status = 400;
return;
}
url = url.replace("//", "/");
const { hostname } = new URL(url);
let resolvedIps;
try {

View file

@ -147,7 +147,7 @@
<span class="button-label-big">Refresh</span>
</button>
<p class="dont-worry">Don't worry, it's (probably) not your fault.</p>
<p>Please make sure your browser is up-to-date and any AdBlockers are off.</p>
<p>Please make sure your browser is up-to-date and any AdBlockers are off (given they can sometimes errouniously interfere with loading assets).</p>
<p>If the problem persists after refreshing, please contact your instance's administrator.<br>You may also try the following options:</p>
<a href="/flush">
<button class="button-small">

View file

@ -1,6 +1,6 @@
import type { Antenna } from "@/models/entities/antenna.js";
import type { Note } from "@/models/entities/note.js";
import { genId } from "@/misc/gen-id.js";
import { getTimestamp } from "@/misc/gen-id.js";
import { redisClient } from "@/db/redis.js";
import { publishAntennaStream } from "@/services/stream.js";
import type { User } from "@/models/entities/user.js";
@ -15,7 +15,7 @@ export async function addNoteToAntenna(
"MAXLEN",
"~",
"200",
"*",
`${getTimestamp(note.id)}-*`,
"note",
note.id,
);

View file

@ -36,13 +36,13 @@
"blurhash": "2.0.5",
"broadcast-channel": "5.1.0",
"browser-image-resizer": "github:misskey-dev/browser-image-resizer",
"chart.js": "4.3.2",
"chart.js": "4.3.3",
"chartjs-adapter-date-fns": "3.0.0",
"chartjs-chart-matrix": "^2.0.1",
"chartjs-plugin-gradient": "0.6.1",
"chartjs-plugin-zoom": "2.0.1",
"city-timezones": "^1.2.1",
"compare-versions": "6.0.0",
"compare-versions": "6.1.0",
"cropperjs": "2.0.0-beta.2",
"cross-env": "7.0.3",
"cypress": "10.11.0",
@ -65,20 +65,20 @@
"mfm-js": "0.23.3",
"paralint": "^1.2.1",
"photoswipe": "5.3.8",
"prettier": "3.0.0",
"prettier": "3.0.1",
"prettier-plugin-vue": "1.1.6",
"prismjs": "1.29.0",
"punycode": "2.3.0",
"querystring": "0.2.1",
"rndstr": "1.0.0",
"rollup": "3.27.0",
"rollup": "3.27.2",
"s-age": "1.1.2",
"sass": "1.64.1",
"sass": "1.64.2",
"seedrandom": "3.0.5",
"start-server-and-test": "1.15.2",
"strict-event-emitter-types": "2.0.0",
"stringz": "2.1.0",
"swiper": "10.0.4",
"swiper": "10.1.0",
"syuilo-password-strength": "0.0.1",
"textarea-caret": "3.1.0",
"three": "0.146.0",
@ -92,10 +92,10 @@
"unicode-emoji-json": "^0.4.0",
"uuid": "9.0.0",
"vanilla-tilt": "1.8.0",
"vite": "4.4.7",
"vite": "4.4.9",
"vite-plugin-compression": "^0.5.1",
"vue": "3.3.4",
"vue-draggable-plus": "^0.2.2",
"vue-draggable-plus": "^0.2.4",
"vue-isyourpasswordsafe": "^2.0.0",
"vue-plyr": "^7.0.0",
"vue-prism-editor": "2.0.0-alpha.2"

View file

@ -47,7 +47,7 @@ const commonNames = new Map<string, string>([
["bookwyrm", "BookWyrm"],
["bridgy-fed", "Bridgy Fed"],
["foundkey", "FoundKey"],
["gnusocial", "GNU Social"],
["gnusocial", "GNU social"],
["gotosocial", "GoToSocial"],
["microblogpub", "microblog.pub"],
["nextcloud social", "Nextcloud Social"],

View file

@ -106,7 +106,8 @@
:text="'@namekuji@firefish.social (Backend)'"
/></FormLink>
<FormLink to="/@dev@post.naskya.net"
><Mfm :text="'@dev@post.naskya.net (Fullstack)'"
><Mfm
:text="'@dev@post.naskya.net (Fullstack)'"
/></FormLink>
<FormLink to="/@panos@firefish.social"
><Mfm

View file

@ -24,6 +24,7 @@
<MkTime :time="announcement.createdAt" />
</div>
</div>
<hr class="_seperator" />
<div class="_content">
<Mfm :text="announcement.text" />
<img
@ -87,7 +88,13 @@ definePageMetadata({
padding: 14px 32px !important;
}
> ._seperator {
margin: 1rem;
}
> ._content {
padding: 2rem;
> img {
display: block;
max-height: 300px;

View file

@ -207,7 +207,7 @@ export function getUserMenu(user, router: Router = mainRouter) {
async function getConfirmed(text: string): Promise<boolean> {
const confirm = await os.confirm({
type: "warning",
title: "confirm",
title: i18n.ts.confirm,
text,
});

View file

@ -28,7 +28,7 @@ globalThis.addEventListener("activate", (ev) => {
});
function offlineContentHTML(): string {
return `<!doctype html>Offline. Service Worker @${_VERSION_} <button onclick="location.reload()">reload</button>`;
return `<!DOCTYPE html><meta charset="utf-8"><meta name="viewport" content="width=device-width, initial-scale=1.0"><title>Offline</title><style>*{font-family:BIZ UDGothic,Roboto,HelveticaNeue,Arial,sans-serif}body,html{background-color:#191724;color:#e0def4;justify-content:center;margin:auto;padding:10px;text-align:center}button{border-radius:999px;padding:0 12px 0 12px;border:none;cursor:pointer;margin-bottom:12px}.button-big{background:linear-gradient(90deg,#c4a7e7,#ebbcba);line-height:50px}.button-big:hover{background:#31748f}.button-label-big{color:#191724;font-weight:700;font-size:20px;padding:12px}.button-label-small{color:#9ccfd8;font-size:16px;padding:12px}p{font-size:16px}#msg,.dont-worry{font-size:18px}.icon-warning{color:#f6c177;height:4rem;padding-top:2rem}h1{font-size:32px}code{font-family:Fira,FiraCode,monospace}@media screen and (max-width:500px){details{width:50%}}</style><body><svg class=icon-warning class="icon icon-tabler icon-tabler-alert-triangle"fill=none stroke=currentColor stroke-linecap=round stroke-linejoin=round stroke-width=2 viewBox="0 0 24 24"xmlns=http://www.w3.org/2000/svg><path d="M0 0h24v24H0z"fill=none stroke=none></path><path d="M12 9v2m0 4v.01"></path><path d="M5 19h14a2 2 0 0 0 1.84 -2.75l-7.1 -12.25a2 2 0 0 0 -3.5 0l-7.1 12.25a2 2 0 0 0 1.75 2.75"></path></svg><h1>Looks like you're offline!</h1><button class=button-big onclick=location.reload(!0)><span class=button-label-big>Refresh</span></button><p class=dont-worry>Looks like Firefish couldn't connect to the server, probably because your device is offline.<p>The installed Service Worker is version <code>${_VERSION_}</code>`;
}
globalThis.addEventListener("fetch", (ev) => {