From 9490a9a34a672cdcf51feda24b5298bee76b30c9 Mon Sep 17 00:00:00 2001 From: aecsocket <43144841+aecsocket@users.noreply.github.com> Date: Sun, 6 Sep 2026 18:22:07 +0100 Subject: [PATCH 01/17] Ignore some nags inside code blocks --- Cargo.toml | 128 +++++++++++++++++++-------------------- apps/labrinth/Cargo.toml | 1 + 2 files changed, 65 insertions(+), 64 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 30ad1af6b7..79bbd5591f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,22 +1,22 @@ [workspace] resolver = "2" members = [ - "apps/app", - "apps/app-playground", - "apps/daedalus_client", - "apps/labrinth", - "packages/app-lib", - "packages/ariadne", - "packages/component-derive", - "packages/daedalus", - "packages/modrinth-content-management", - "packages/modrinth-log", - "packages/modrinth-maxmind", - "packages/modrinth-util", - "packages/neverbounce", - "packages/path-util", - "packages/serde-binhum", - "packages/xredis", + "apps/app", + "apps/app-playground", + "apps/daedalus_client", + "apps/labrinth", + "packages/app-lib", + "packages/ariadne", + "packages/component-derive", + "packages/daedalus", + "packages/modrinth-content-management", + "packages/modrinth-log", + "packages/modrinth-maxmind", + "packages/modrinth-util", + "packages/neverbounce", + "packages/path-util", + "packages/serde-binhum", + "packages/xredis", ] [workspace.package] @@ -39,16 +39,16 @@ ariadne = { path = "packages/ariadne" } async-minecraft-ping = { path = "packages/async-minecraft-ping" } async-recursion = "1.1.1" async-stripe = { version = "0.41.0", default-features = false, features = [ - "runtime-tokio-hyper-rustls", + "runtime-tokio-hyper-rustls", ] } async-trait = "0.1.89" async-tungstenite = { version = "0.31.0", default-features = false, features = ["futures-03-sink"] } async-walkdir = "2.1.0" async_zip = "0.0.18" aws-sdk-s3 = { version = "=1.122.0", default-features = false, features = [ - "default-https-client", - "rt-tokio", - "rustls", + "default-https-client", + "rt-tokio", + "rustls", ] } base64 = "0.22.1" bitflags = "2.9.4" @@ -99,10 +99,10 @@ hmac = "0.12.1" httpdate = "1.0.3" hyper = "1.7.0" hyper-rustls = { version = "0.27.7", default-features = false, features = [ - "aws-lc-rs", - "http1", - "native-tokio", - "tls12", + "aws-lc-rs", + "http1", + "native-tokio", + "tls12", ] } hyper-util = "0.1.17" iana-time-zone = "0.1.64" @@ -114,22 +114,22 @@ jemalloc_pprof = "0.8.1" json-patch = { version = "4.1.0", default-features = false } json5 = "1.3.1" lettre = { version = "0.11.19", default-features = false, features = [ - "aws-lc-rs", - "builder", - "hostname", - "pool", - "rustls", - "rustls-native-certs", - "smtp-transport", - "tokio1", - "tokio1-rustls", + "aws-lc-rs", + "builder", + "hostname", + "pool", + "rustls", + "rustls-native-certs", + "smtp-transport", + "tokio1", + "tokio1-rustls", ] } linkify = "0.11.0" lz4_flex = { version = "0.11.5", default-features = false, features = [ - "checked-decode", - "safe-decode", - "safe-encode", - "std", + "checked-decode", + "safe-decode", + "safe-encode", + "std", ] } maxminddb = "0.26.0" md5 = "0.8.0" @@ -159,8 +159,8 @@ pulldown-cmark = { version = "0.13.4", default-features = false } quartz_nbt = "0.2.9" quick-xml = "0.38.3" quote = { version = "1.0" } -rand = "=0.8.5" # Locked on 0.8 until argon2 and p256 update to 0.9 -rand_chacha = "=0.3.1" # Locked on 0.3 until we can update rand to 0.9 +rand = "=0.8.5" # Locked on 0.8 until argon2 and p256 update to 0.9 +rand_chacha = "=0.3.1" # Locked on 0.3 until we can update rand to 0.9 rdkafka = { version = "0.36.2", features = ["cmake-build"] } redis = "1.4.1" regex = "1.12.2" @@ -169,9 +169,9 @@ rgb = "0.8.52" rust_decimal = { version = "1.39.0", features = ["serde-with-float", "serde-with-str"] } rust_iso3166 = "0.1.14" rust-s3 = { version = "0.37.0", default-features = false, features = [ - "fail-on-err", - "tags", - "tokio-rustls-tls", + "fail-on-err", + "tags", + "tokio-rustls-tls", ] } rustls = "0.23.32" rustrict = { version = "0.7.39", default-features = false, features = ["censor"] } @@ -180,12 +180,12 @@ same-file = "1.0.6" scalar_api_reference = { version = "0.2.2", default-features = false } secrecy = "0.10.3" sentry = { version = "0.45.0", default-features = false, features = [ - "backtrace", - "contexts", - "debug-images", - "panic", - "reqwest", - "rustls", + "backtrace", + "contexts", + "debug-images", + "panic", + "reqwest", + "rustls", ] } serde = "1.0.228" serde-binhum = { path = "packages/serde-binhum" } @@ -216,8 +216,8 @@ tauri-plugin-opener = "2.5.0" tauri-plugin-os = "2.3.1" tauri-plugin-single-instance = "2.3.4" tauri-plugin-updater = { git = "https://github.com/modrinth/plugins-workspace", rev = "0d30f2aa28ec668ce187d527da1c475da3c01cbc", default-features = false, features = [ - "rustls-tls", - "zip", + "rustls-tls", + "zip", ] } tauri-plugin-window-state = "2.4.0" tempfile = "3.23.0" @@ -247,21 +247,21 @@ validator = "0.20.0" webauthn-rs = "0.5.5" webauthn-rs-proto = "0.5.5" webp = { version = "0.3.1", default-features = false } -webview2-com = "0.38.0" # Should be updated in lockstep with wry +webview2-com = "0.38.0" # Should be updated in lockstep with wry whatlang = "0.18.0" whoami = "1.6.1" -windows = "=0.61.3" # Locked on 0.61 until we can update windows-core to 0.62 -windows-core = "=0.61.2" # Locked on 0.61 until webview2-com updates to 0.62 +windows = "=0.61.3" # Locked on 0.61 until we can update windows-core to 0.62 +windows-core = "=0.61.2" # Locked on 0.61 until webview2-com updates to 0.62 winreg = "0.55.0" woothee = "0.13.0" xredis = { path = "packages/xredis" } yaserde = "0.12.0" zbus = "5.11.0" zip = { version = "6.0.0", default-features = false, features = [ - "bzip2", - "deflate", - "deflate64", - "zstd", + "bzip2", + "deflate", + "deflate64", + "zstd", ] } zxcvbn = "3.1.0" @@ -319,15 +319,15 @@ opt-level = 3 # Optimize for speed and reduce size on release builds [profile.release] -opt-level = "s" # Optimize for binary size -strip = true # Remove debug symbols -lto = true # Enables link to optimizations -panic = "abort" # Strip expensive panic clean-up logic +opt-level = "s" # Optimize for binary size +strip = true # Remove debug symbols +lto = true # Enables link to optimizations +panic = "abort" # Strip expensive panic clean-up logic # Specific profile for labrinth production builds [profile.release-labrinth] inherits = "release" opt-level = 2 -strip = false # Keep debug symbols for Sentry -lto = "thin" # Enable LTO but keep compile times reasonable -panic = "unwind" # Don't exit the whole app on panic in production +strip = false # Keep debug symbols for Sentry +lto = "thin" # Enable LTO but keep compile times reasonable +panic = "unwind" # Don't exit the whole app on panic in production diff --git a/apps/labrinth/Cargo.toml b/apps/labrinth/Cargo.toml index f2325ec3ba..6eb09144f3 100644 --- a/apps/labrinth/Cargo.toml +++ b/apps/labrinth/Cargo.toml @@ -82,6 +82,7 @@ paste = { workspace = true } path-util = { workspace = true } postcard = { workspace = true } prometheus = { workspace = true } +pulldown-cmark = { workspace = true } psl = { workspace = true } pulldown-cmark = { workspace = true } quick-xml = { workspace = true } From af438af9706e27705bb1a3d5967c532382fb7db8 Mon Sep 17 00:00:00 2001 From: tdgao Date: Mon, 14 Sep 2026 12:56:13 -0600 Subject: [PATCH 02/17] fmt --- apps/labrinth/Cargo.toml | 1 - 1 file changed, 1 deletion(-) diff --git a/apps/labrinth/Cargo.toml b/apps/labrinth/Cargo.toml index 6eb09144f3..f2325ec3ba 100644 --- a/apps/labrinth/Cargo.toml +++ b/apps/labrinth/Cargo.toml @@ -82,7 +82,6 @@ paste = { workspace = true } path-util = { workspace = true } postcard = { workspace = true } prometheus = { workspace = true } -pulldown-cmark = { workspace = true } psl = { workspace = true } pulldown-cmark = { workspace = true } quick-xml = { workspace = true } From ecc1de76742ebe60afd34bfba84ecd5a4963731d Mon Sep 17 00:00:00 2001 From: tdgao Date: Tue, 8 Sep 2026 10:55:38 -0600 Subject: [PATCH 03/17] fix: ensure all heading levels are validated --- .../validate/project/description/markdown.rs | 49 ++++++++++++++----- 1 file changed, 37 insertions(+), 12 deletions(-) diff --git a/apps/labrinth/src/validate/project/description/markdown.rs b/apps/labrinth/src/validate/project/description/markdown.rs index a102140974..5b6ccb21c2 100644 --- a/apps/labrinth/src/validate/project/description/markdown.rs +++ b/apps/labrinth/src/validate/project/description/markdown.rs @@ -6,10 +6,10 @@ use regex::Regex; use unicode_segmentation::UnicodeSegmentation; static HTML_HEADER: LazyLock = LazyLock::new(|| { - Regex::new(r"(?is)]*>(.*?)").unwrap() + Regex::new(r"(?is)]*>(.*?)").unwrap() }); static ADJACENT_HTML_HEADERS: LazyLock = - LazyLock::new(|| Regex::new(r"(?is)\s*\s* = LazyLock::new(|| { Regex::new(r"(?is)\s*(?:]*>\s*)*$").unwrap() }); @@ -101,7 +101,6 @@ impl<'a> DescriptionMarkdown<'a> { let markdown_headers = self .headings .iter() - .filter(|heading| is_primary_heading(heading.level)) .filter(|heading| header_is_long(&heading.text)) .count(); let without_code = self.replace_code(" "); @@ -130,8 +129,7 @@ impl<'a> DescriptionMarkdown<'a> { let [previous, current] = headings else { return false; }; - is_primary_heading(previous.level) - && previous.level == current.level + previous.level == current.level && self.markdown[previous.range.end..current.range.start] .trim() .is_empty() @@ -180,13 +178,6 @@ fn header_is_long(header: &str) -> bool { rendered.graphemes(true).count() > 80 } -fn is_primary_heading(level: HeadingLevel) -> bool { - matches!( - level, - HeadingLevel::H1 | HeadingLevel::H2 | HeadingLevel::H3 - ) -} - #[cfg(test)] mod tests { use super::DescriptionMarkdown; @@ -250,4 +241,38 @@ image: "![](/missing-alt.png)" .has_adjacent_same_level_headers() ); } + + #[test] + fn all_heading_levels_are_validated() { + for level in 1..=6 { + let prefix = "#".repeat(level); + let long_text = "heading ".repeat(12); + for (long_header, adjacent_headers, separated_headers) in [ + ( + format!("{prefix} {long_text}"), + format!("{prefix} First\n\n{prefix} Second"), + format!("{prefix} First\n\n```yaml\n# comment\n```\n\n{prefix} Second"), + ), + ( + format!("{long_text}"), + format!("First\n\nSecond"), + format!("First\n\n```yaml\n# comment\n```\n\nSecond"), + ), + ] { + let markdown = DescriptionMarkdown::parse(&long_header); + assert_eq!(markdown.long_header_count(), 1, "{long_header}"); + assert!(markdown.ends_with_header(), "{long_header}"); + assert!( + DescriptionMarkdown::parse(&adjacent_headers) + .has_adjacent_same_level_headers(), + "{adjacent_headers}" + ); + assert!( + !DescriptionMarkdown::parse(&separated_headers) + .has_adjacent_same_level_headers(), + "{separated_headers}" + ); + } + } + } } From 68ea4583f67099dd43420dfd9cb0ec7eb49dbb8f Mon Sep 17 00:00:00 2001 From: tdgao Date: Tue, 8 Sep 2026 11:07:23 -0600 Subject: [PATCH 04/17] feat: add new description matches summary with 80% similarity validator --- .../src/validate/project/description.rs | 11 ++- apps/labrinth/src/validate/project/mod.rs | 1 + apps/labrinth/src/validate/project/summary.rs | 48 +---------- apps/labrinth/src/validate/project/text.rs | 83 +++++++++++++++++++ apps/labrinth/tests/project.rs | 54 ++++++++++++ .../api-client/src/modules/labrinth/types.ts | 1 + .../moderation/src/data/nags/description.ts | 13 +++ 7 files changed, 165 insertions(+), 46 deletions(-) diff --git a/apps/labrinth/src/validate/project/description.rs b/apps/labrinth/src/validate/project/description.rs index 27dab4c4e3..269aea48d2 100644 --- a/apps/labrinth/src/validate/project/description.rs +++ b/apps/labrinth/src/validate/project/description.rs @@ -8,13 +8,14 @@ use super::text::{ extract_description_text, find_banned_description_link, has_image_without_alt_text, has_sufficient_english_blocks, js_string_length, non_standard_text_ratio, normalize_project_field_text, - profanity_matches, project_requires_english, + profanity_matches, project_requires_english, project_text_similarity, }; use super::{ProjectNag, ProjectNagKind, ProjectNagSeverity}; use crate::models::projects::Project; const MIN_DESCRIPTION_CHARS: usize = 125; +const MAX_DESCRIPTION_SUMMARY_SIMILARITY: f64 = 0.8; const MAX_PROFANITY_COUNT: usize = 2; const NON_STANDARD_TEXT_FAILURE_THRESHOLD: f64 = 0.05; @@ -91,6 +92,14 @@ pub(super) fn validate(project: &Project) -> Vec { ); } } + if project_text_similarity(&normalized_text, &project.summary) + >= MAX_DESCRIPTION_SUMMARY_SIMILARITY + { + nags.push(ProjectNag::new( + ProjectNagKind::ProjectDescriptionMatchesSummary, + ProjectNagSeverity::Required, + )); + } if has_spam { nags.push(ProjectNag::new( ProjectNagKind::ProjectDescriptionSpam, diff --git a/apps/labrinth/src/validate/project/mod.rs b/apps/labrinth/src/validate/project/mod.rs index 308870b7a7..28ea0df273 100644 --- a/apps/labrinth/src/validate/project/mod.rs +++ b/apps/labrinth/src/validate/project/mod.rs @@ -68,6 +68,7 @@ pub enum ProjectNagKind { ProjectDescriptionProfanity, ProjectDescriptionNonStandardText, ProjectDescriptionNonEnglish, + ProjectDescriptionMatchesSummary, AddDescription, DescriptionTooShort, ProjectDescriptionSpam, diff --git a/apps/labrinth/src/validate/project/summary.rs b/apps/labrinth/src/validate/project/summary.rs index 518bed6dfc..43ac090a06 100644 --- a/apps/labrinth/src/validate/project/summary.rs +++ b/apps/labrinth/src/validate/project/summary.rs @@ -3,7 +3,8 @@ use serde_json::json; use super::text::{ ProfanityKind, contains_spam, find_link_or_ip, has_non_standard_text, has_summary_formatting, is_likely_english_summary, js_string_length, - normalize_project_field_text, profanity_matches, project_requires_english, + normalize_project_field_text, profanity_matches, project_requires_english, + project_text_similarity, }; use super::{ProjectNag, ProjectNagKind, ProjectNagSeverity}; @@ -65,7 +66,7 @@ pub(super) fn validate(project: &Project) -> Vec { if !summary.is_empty() && !contains_link && !project.name.is_empty() - && summary_name_similarity(summary, &project.name) + && project_text_similarity(summary, &project.name) >= MAX_SUMMARY_NAME_SIMILARITY { nags.push(ProjectNag::new( @@ -135,46 +136,3 @@ fn requires_language_nag( && !contains_link && !has_spam } - -fn summary_name_similarity(summary: &str, name: &str) -> f64 { - let summary = normalized_for_similarity(summary); - let name = normalized_for_similarity(name); - let longest_length = summary.len().max(name.len()); - if longest_length == 0 { - return 0.0; - } - - 1.0 - levenshtein_distance(&summary, &name) as f64 / longest_length as f64 -} - -fn normalized_for_similarity(text: &str) -> Vec { - normalize_project_field_text(text) - .to_lowercase() - .chars() - .filter(|character| !character.is_whitespace()) - .collect() -} - -fn levenshtein_distance(left: &[char], right: &[char]) -> usize { - if left.len() > right.len() { - return levenshtein_distance(right, left); - } - - let mut previous_row = (0..=left.len()).collect::>(); - for (right_index, right_character) in right.iter().enumerate() { - let mut current_row = Vec::with_capacity(left.len() + 1); - current_row.push(right_index + 1); - for (left_index, left_character) in left.iter().enumerate() { - current_row.push( - (current_row[left_index] + 1) - .min(previous_row[left_index + 1] + 1) - .min( - previous_row[left_index] - + usize::from(left_character != right_character), - ), - ); - } - previous_row = current_row; - } - previous_row[left.len()] -} diff --git a/apps/labrinth/src/validate/project/text.rs b/apps/labrinth/src/validate/project/text.rs index c51df4d3d6..a4efd930ec 100644 --- a/apps/labrinth/src/validate/project/text.rs +++ b/apps/labrinth/src/validate/project/text.rs @@ -2,6 +2,7 @@ use std::collections::{BTreeMap, BTreeSet}; use std::sync::LazyLock; use linkify::{LinkFinder, LinkKind}; +use pulldown_cmark::{Event, Parser, Tag}; use regex::Regex; use unicode_normalization::UnicodeNormalization; use unicode_segmentation::UnicodeSegmentation; @@ -129,6 +130,49 @@ pub(super) fn normalize_project_field_text(text: &str) -> String { text.trim().nfc().collect() } +pub(super) fn project_text_similarity(left: &str, right: &str) -> f64 { + let left = normalized_for_similarity(left); + let right = normalized_for_similarity(right); + let longest_length = left.len().max(right.len()); + if longest_length == 0 { + return 0.0; + } + + 1.0 - levenshtein_distance(&left, &right) as f64 / longest_length as f64 +} + +fn normalized_for_similarity(text: &str) -> Vec { + normalize_project_field_text(text) + .to_lowercase() + .chars() + .filter(|character| !character.is_whitespace()) + .collect() +} + +fn levenshtein_distance(left: &[char], right: &[char]) -> usize { + if left.len() > right.len() { + return levenshtein_distance(right, left); + } + + let mut previous_row = (0..=left.len()).collect::>(); + for (right_index, right_character) in right.iter().enumerate() { + let mut current_row = Vec::with_capacity(left.len() + 1); + current_row.push(right_index + 1); + for (left_index, left_character) in left.iter().enumerate() { + current_row.push( + (current_row[left_index] + 1) + .min(previous_row[left_index + 1] + 1) + .min( + previous_row[left_index] + + usize::from(left_character != right_character), + ), + ); + } + previous_row = current_row; + } + previous_row[left.len()] +} + pub(super) fn js_string_length(text: &str) -> usize { text.encode_utf16().count() } @@ -838,6 +882,8 @@ pub(super) fn find_link_or_ip(text: &str) -> Option { pub(super) fn has_summary_formatting(summary: &str) -> bool { has_paired_html_formatting(summary) || MARKDOWN_LINK.is_match(summary) + || Parser::new(summary) + .any(|event| matches!(event, Event::Start(Tag::Emphasis))) || summary.lines().any(|line| { let line = line.trim_start(); line.starts_with('#') @@ -1075,3 +1121,40 @@ fn has_enough_language_content(text: &str) -> bool { WORD.find_iter(text).count() >= 8 && text.trim().graphemes(true).count() >= 35 } + +#[cfg(test)] +mod tests { + use super::{has_summary_formatting, project_text_similarity}; + + #[test] + fn summary_detects_markdown_emphasis() { + for summary in ["*this*", "Adds *new features* to Minecraft", "*a*", "_this_"] { + assert!(has_summary_formatting(summary), "{summary:?}"); + } + } + + #[test] + fn summary_allows_literal_asterisks() { + for summary in ["A single * asterisk", "2 * 3 * 4", r"\*this\*", "An unmatched *asterisk"] { + assert!(!has_summary_formatting(summary), "{summary:?}"); + } + } + + #[test] + fn similarity_ignores_case_whitespace_and_unicode_composition() { + assert_eq!(project_text_similarity(" Café tools ", "CAFE\u{301}\nTOOLS"), 1.0); + } + + #[test] + fn similarity_distinguishes_the_eighty_percent_boundary() { + assert!(project_text_similarity("abcde", "abcdx") >= 0.8); + assert!(project_text_similarity("abcde", "abcxy") < 0.8); + } + + #[test] + fn empty_fields_do_not_match() { + assert_eq!(project_text_similarity(" ", "\n"), 0.0); + assert_eq!(project_text_similarity("", "some text"), 0.0); + assert_eq!(project_text_similarity("some text", ""), 0.0); + } +} diff --git a/apps/labrinth/tests/project.rs b/apps/labrinth/tests/project.rs index 023aca2028..3e7373dc4c 100644 --- a/apps/labrinth/tests/project.rs +++ b/apps/labrinth/tests/project.rs @@ -658,6 +658,60 @@ async fn test_leaving_review_skips_validation() { .await; } +#[actix_rt::test] +async fn test_description_similarity_to_summary() { + with_test_environment( + None, + |test_env: TestEnvironment| async move { + let api = &test_env.api; + let project_slug = &test_env.dummy.project_alpha.project_slug; + let summary = "Explore new worlds with configurable tools and adventures."; + let response = api + .edit_project( + project_slug, + json!({ "status": "draft", "summary": summary }), + ADMIN_USER_PAT, + ) + .await; + assert_status!(&response, StatusCode::NO_CONTENT); + + for (description, expected_match) in [ + (summary.to_string(), true), + (format!("**{}**\n\n```yaml\nsetting: true\n```", summary.to_uppercase()), true), + (format!("{summary} Players can discover custom structures, configure individual features, and follow detailed installation instructions for their preferred loader."), false), + (String::new(), false), + ] { + let response = api + .edit_project( + project_slug, + json!({ "description": description }), + USER_USER_PAT, + ) + .await; + assert_status!(&response, StatusCode::NO_CONTENT); + + let request = test::TestRequest::get() + .uri(&format!("/v3/project/{project_slug}/validate")) + .append_pat(USER_USER_PAT) + .to_request(); + let response = api.call(request).await; + assert_status!(&response, StatusCode::OK); + let validation: serde_json::Value = test::read_body_json(response).await; + let matching_nag = validation["nags"] + .as_array() + .unwrap() + .iter() + .find(|nag| nag["kind"] == "project_description_matches_summary"); + assert_eq!(matching_nag.is_some(), expected_match, "{description}"); + if let Some(nag) = matching_nag { + assert_eq!(nag["severity"], "required"); + } + } + }, + ) + .await; +} + #[actix_rt::test] async fn test_plugin_and_datapack_validation_use_mod_tags() { with_test_environment( diff --git a/packages/api-client/src/modules/labrinth/types.ts b/packages/api-client/src/modules/labrinth/types.ts index ee90f01f86..e105386f05 100644 --- a/packages/api-client/src/modules/labrinth/types.ts +++ b/packages/api-client/src/modules/labrinth/types.ts @@ -1146,6 +1146,7 @@ export namespace Labrinth { | 'long-headers' | 'missing-alt-text' | 'project-description-banned-link' + | 'project-description-matches-summary' | 'project-description-non-english' | 'project-description-non-standard-text' | 'project-description-profanity' diff --git a/packages/moderation/src/data/nags/description.ts b/packages/moderation/src/data/nags/description.ts index abca386ac9..27c3b772a8 100644 --- a/packages/moderation/src/data/nags/description.ts +++ b/packages/moderation/src/data/nags/description.ts @@ -3,6 +3,14 @@ import { defineMessages } from '@modrinth/ui' import type { NagDefinitions } from './types.ts' const messages = defineMessages({ + reviewTitle: { + id: 'nags.project-description-content.title', + defaultMessage: 'Review the description', + }, + matchesSummary: { + id: 'project.text-validation.description-matches-summary', + defaultMessage: `Your project's description should provide unique information and not repeat the project's summary.`, + }, addTitle: { id: 'nags.add-description.title', defaultMessage: 'Add a description' }, add: { id: 'nags.add-description.description', @@ -77,6 +85,11 @@ const messages = defineMessages({ }) export const descriptionNags = { + 'project-description-matches-summary': { + title: messages.reviewTitle, + description: messages.matchesSummary, + destination: 'description', + }, 'add-description': { title: messages.addTitle, description: messages.add, From f1e7d68d207b6ac7147c991d9020649ebb1fc8cb Mon Sep 17 00:00:00 2001 From: tdgao Date: Tue, 8 Sep 2026 11:12:31 -0600 Subject: [PATCH 05/17] feat: hide publishing checklist for staff when theres no requred nags --- .../ui/moderation/ModerationProjectNags.vue | 11 +++++++++++ apps/frontend/src/composables/featureFlags.ts | 1 + 2 files changed, 12 insertions(+) diff --git a/apps/frontend/src/components/ui/moderation/ModerationProjectNags.vue b/apps/frontend/src/components/ui/moderation/ModerationProjectNags.vue index 065a5eb2b0..26ff89cf7d 100644 --- a/apps/frontend/src/components/ui/moderation/ModerationProjectNags.vue +++ b/apps/frontend/src/components/ui/moderation/ModerationProjectNags.vue @@ -141,6 +141,7 @@ import { type MessageDescriptor, useVIntl, } from '@modrinth/ui' +import { isStaff } from '@modrinth/utils' import type { Component } from 'vue' import { computed, nextTick, onBeforeUnmount, onMounted, ref, watch } from 'vue' @@ -222,6 +223,7 @@ const messages = defineMessages({ const { formatMessage } = useVIntl() const { addNotification } = injectNotificationManager() +const flags = useFeatureFlags() const props = withDefaults(defineProps(), { versions: () => [], @@ -419,6 +421,15 @@ function isNagComplete(nag: Nag): boolean { const visibleNags = computed(() => { const finalNags = applicableNags.value.filter((nag) => !isNagComplete(nag)) + if ( + isProcessing.value && + isStaff(props.currentMember?.user) && + !flags.value.alwaysShowPublishingChecklistForStaff && + !finalNags.some((nag) => nag.status === 'required') + ) { + return [] + } + if (props.project.status === 'draft') { finalNags.push({ id: 'submit-for-review', diff --git a/apps/frontend/src/composables/featureFlags.ts b/apps/frontend/src/composables/featureFlags.ts index 1165ef04cc..8cd1a685da 100644 --- a/apps/frontend/src/composables/featureFlags.ts +++ b/apps/frontend/src/composables/featureFlags.ts @@ -53,6 +53,7 @@ export const DEFAULT_FEATURE_FLAGS = validateValues({ alwaysIgnoreErrorBanner: false, showViewProdRouteBanner: false, showModeratorProjectMemberUi: false, + alwaysShowPublishingChecklistForStaff: false, archonApiStaging: false, showHostingAccessInstanceAuditLog: false, versionDevInfoCollapsed: true, From c549709d8dadc6819cf4418be248c389ae0ab429 Mon Sep 17 00:00:00 2001 From: tdgao Date: Tue, 8 Sep 2026 11:23:33 -0600 Subject: [PATCH 06/17] feat: only show disclosures suggestion if no disclosures added --- .../src/validate/project/disclosures.rs | 21 +++++++++++-------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/apps/labrinth/src/validate/project/disclosures.rs b/apps/labrinth/src/validate/project/disclosures.rs index 6749399cbd..67ff610671 100644 --- a/apps/labrinth/src/validate/project/disclosures.rs +++ b/apps/labrinth/src/validate/project/disclosures.rs @@ -7,15 +7,18 @@ pub(super) fn validate( project: &crate::models::projects::Project, disclosures: Option<&[ProjectDisclosure]>, ) -> Vec { - let (project_type, _) = - LegacyProject::get_project_type(&project.project_types); - let mut nags = vec![ - ProjectNag::new( - ProjectNagKind::CheckDisclosures, - ProjectNagSeverity::Suggestion, - ) - .with_details(serde_json::json!({ "project_type": project_type })), - ]; + let mut nags = Vec::new(); + if disclosures.is_some_and(|disclosures| disclosures.is_empty()) { + let (project_type, _) = + LegacyProject::get_project_type(&project.project_types); + nags.push( + ProjectNag::new( + ProjectNagKind::CheckDisclosures, + ProjectNagSeverity::Suggestion, + ) + .with_details(serde_json::json!({ "project_type": project_type })), + ); + } if disclosures.is_some_and(|disclosures| { disclosures.iter().any(disclosure_has_paired_html) From be4d76592817f38bc94e849a14fe65bdfc7ac516 Mon Sep 17 00:00:00 2001 From: tdgao Date: Tue, 8 Sep 2026 12:29:57 -0600 Subject: [PATCH 07/17] format --- .../src/validate/project/disclosures.rs | 24 +++++++++---------- 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/apps/labrinth/src/validate/project/disclosures.rs b/apps/labrinth/src/validate/project/disclosures.rs index 67ff610671..fc58e6073d 100644 --- a/apps/labrinth/src/validate/project/disclosures.rs +++ b/apps/labrinth/src/validate/project/disclosures.rs @@ -7,18 +7,18 @@ pub(super) fn validate( project: &crate::models::projects::Project, disclosures: Option<&[ProjectDisclosure]>, ) -> Vec { - let mut nags = Vec::new(); - if disclosures.is_some_and(|disclosures| disclosures.is_empty()) { - let (project_type, _) = - LegacyProject::get_project_type(&project.project_types); - nags.push( - ProjectNag::new( - ProjectNagKind::CheckDisclosures, - ProjectNagSeverity::Suggestion, - ) - .with_details(serde_json::json!({ "project_type": project_type })), - ); - } + let mut nags = Vec::new(); + if disclosures.is_some_and(|disclosures| disclosures.is_empty()) { + let (project_type, _) = + LegacyProject::get_project_type(&project.project_types); + nags.push( + ProjectNag::new( + ProjectNagKind::CheckDisclosures, + ProjectNagSeverity::Suggestion, + ) + .with_details(serde_json::json!({ "project_type": project_type })), + ); + } if disclosures.is_some_and(|disclosures| { disclosures.iter().any(disclosure_has_paired_html) From 12a2858102b6652eba4f29dbeb3ac60101b7ab10 Mon Sep 17 00:00:00 2001 From: tdgao Date: Wed, 9 Sep 2026 12:47:23 -0600 Subject: [PATCH 08/17] feat: switch language detection to use lingua, and tune for less false positives --- Cargo.lock | 837 +++++++++++++++++- Cargo.toml | 2 +- apps/labrinth/Cargo.toml | 2 +- .../src/routes/v3/projects/validate.rs | 28 +- .../labrinth/src/validate/project/language.rs | 487 ++++++++++ apps/labrinth/src/validate/project/mod.rs | 1 + apps/labrinth/src/validate/project/summary.rs | 4 +- apps/labrinth/src/validate/project/text.rs | 224 +++-- 8 files changed, 1437 insertions(+), 148 deletions(-) create mode 100644 apps/labrinth/src/validate/project/language.rs diff --git a/Cargo.lock b/Cargo.lock index 4b06422b59..28231e36b7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2529,6 +2529,15 @@ dependencies = [ "libc", ] +[[package]] +name = "counter" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "66e8e052be91f1c8aae2c1d81307d9f6e67f5f37001e3ddee419e971e73f03bc" +dependencies = [ + "num-traits", +] + [[package]] name = "cpufeatures" version = "0.2.17" @@ -3754,6 +3763,9 @@ name = "fastrand" version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be" +dependencies = [ + "getrandom 0.2.16", +] [[package]] name = "fax" @@ -3963,6 +3975,12 @@ dependencies = [ "libc", ] +[[package]] +name = "fst" +version = "0.4.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ab85b9b05e3978cc9a9cf8fea7f01b494e1a09ed3037e16ba39edc7a29eb61a" + [[package]] name = "funty" version = "2.0.0" @@ -5296,6 +5314,25 @@ version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "35a84fd5aa25fae5c0f4a33d9cac2ca017fc622cbd089be2229993514990f870" +[[package]] +name = "include_dir" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "923d117408f1e49d914f1a379a309cffe4f18c05cf4e3d12e613a15fc81bd0dd" +dependencies = [ + "include_dir_macros", +] + +[[package]] +name = "include_dir_macros" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7cab85a7ed0bd5f0e76d93846e0147172bed2e2d3f859bcc33a8d9699cad1a75" +dependencies = [ + "proc-macro2", + "quote", +] + [[package]] name = "indenter" version = "0.3.4" @@ -5746,6 +5783,7 @@ dependencies = [ "json-patch", "labrinth", "lettre", + "lingua", "linkify", "modrinth-content-management", "modrinth-util", @@ -5801,7 +5839,6 @@ dependencies = [ "webauthn-rs", "webauthn-rs-proto", "webp", - "whatlang", "woothee", "xredis", "yaserde", @@ -5999,6 +6036,778 @@ dependencies = [ "vcpkg", ] +[[package]] +name = "lingua" +version = "1.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f40d9129bb9fe42c95d1bd420d6891607eaff17df16ee15674aed2d05b0ec8f4" +dependencies = [ + "counter", + "dashmap", + "fastrand 2.3.0", + "fst", + "include_dir", + "itertools 0.14.0", + "lingua-afrikaans-language-model", + "lingua-albanian-language-model", + "lingua-arabic-language-model", + "lingua-armenian-language-model", + "lingua-azerbaijani-language-model", + "lingua-basque-language-model", + "lingua-belarusian-language-model", + "lingua-bengali-language-model", + "lingua-bokmal-language-model", + "lingua-bosnian-language-model", + "lingua-bulgarian-language-model", + "lingua-catalan-language-model", + "lingua-chinese-language-model", + "lingua-croatian-language-model", + "lingua-czech-language-model", + "lingua-danish-language-model", + "lingua-dutch-language-model", + "lingua-english-language-model", + "lingua-esperanto-language-model", + "lingua-estonian-language-model", + "lingua-finnish-language-model", + "lingua-french-language-model", + "lingua-ganda-language-model", + "lingua-georgian-language-model", + "lingua-german-language-model", + "lingua-greek-language-model", + "lingua-gujarati-language-model", + "lingua-hebrew-language-model", + "lingua-hindi-language-model", + "lingua-hungarian-language-model", + "lingua-icelandic-language-model", + "lingua-indonesian-language-model", + "lingua-irish-language-model", + "lingua-italian-language-model", + "lingua-japanese-language-model", + "lingua-kazakh-language-model", + "lingua-korean-language-model", + "lingua-latin-language-model", + "lingua-latvian-language-model", + "lingua-lithuanian-language-model", + "lingua-macedonian-language-model", + "lingua-malay-language-model", + "lingua-maori-language-model", + "lingua-marathi-language-model", + "lingua-mongolian-language-model", + "lingua-nynorsk-language-model", + "lingua-persian-language-model", + "lingua-polish-language-model", + "lingua-portuguese-language-model", + "lingua-punjabi-language-model", + "lingua-romanian-language-model", + "lingua-russian-language-model", + "lingua-serbian-language-model", + "lingua-shona-language-model", + "lingua-slovak-language-model", + "lingua-slovene-language-model", + "lingua-somali-language-model", + "lingua-sotho-language-model", + "lingua-spanish-language-model", + "lingua-swahili-language-model", + "lingua-swedish-language-model", + "lingua-tagalog-language-model", + "lingua-tamil-language-model", + "lingua-telugu-language-model", + "lingua-thai-language-model", + "lingua-tsonga-language-model", + "lingua-tswana-language-model", + "lingua-turkish-language-model", + "lingua-ukrainian-language-model", + "lingua-urdu-language-model", + "lingua-vietnamese-language-model", + "lingua-welsh-language-model", + "lingua-xhosa-language-model", + "lingua-yoruba-language-model", + "lingua-zulu-language-model", + "maplit", + "rayon", + "regex", + "serde", + "serde-wasm-bindgen", + "strum", + "strum_macros", + "wasm-bindgen", +] + +[[package]] +name = "lingua-afrikaans-language-model" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2810a47263f58358fa22acc425b330a334382f33fcee8f205be672a7ec6d8d4a" +dependencies = [ + "include_dir", +] + +[[package]] +name = "lingua-albanian-language-model" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "81be672350a5a37c9aae935dd8520295289cc02d098929484b963b60fb42e174" +dependencies = [ + "include_dir", +] + +[[package]] +name = "lingua-arabic-language-model" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5bcc254ff44209c2a50dea58a644f9c257a0aa5cd7e1739fd9c373fe4c7456cf" +dependencies = [ + "include_dir", +] + +[[package]] +name = "lingua-armenian-language-model" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4cf4e5825cb3a2b11d8bd7baa6788167b65ffbf8ce21dc9b9dcb5e6adee1d5f0" +dependencies = [ + "include_dir", +] + +[[package]] +name = "lingua-azerbaijani-language-model" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3fdcc775ff75bcc6de0ea5330f65e62644622e29c980a72b437780ba57bba838" +dependencies = [ + "include_dir", +] + +[[package]] +name = "lingua-basque-language-model" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5cbea869a4556800f2798c61559fd2e87fd65d7698ce27ad6fba5b1d9ec1fbe9" +dependencies = [ + "include_dir", +] + +[[package]] +name = "lingua-belarusian-language-model" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2887044d4c11388e15692bf953f70468282600c426e1cd6a208a93aeca3aedfd" +dependencies = [ + "include_dir", +] + +[[package]] +name = "lingua-bengali-language-model" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f94131563e0e0c571cebf03f867d0e1b7dd15278f7cab3e4ba8b0f8026072cc" +dependencies = [ + "include_dir", +] + +[[package]] +name = "lingua-bokmal-language-model" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b443b1ec0498bcf43594f9bf65ae8f5660b5bf21eb769950a245ced8e376ac23" +dependencies = [ + "include_dir", +] + +[[package]] +name = "lingua-bosnian-language-model" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "04f540ed0441bcdbb0b36c43f306bc2b0892921ceb95e865fe0b8b0c39bb82c4" +dependencies = [ + "include_dir", +] + +[[package]] +name = "lingua-bulgarian-language-model" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2f4679441ff2d94b21a3d76293ffd5310a012ea62247b7b2066939e2728cbfc0" +dependencies = [ + "include_dir", +] + +[[package]] +name = "lingua-catalan-language-model" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7366ac180de87eb19f32063b8eed1db270ab333f7a450d96250e53b14b0a750f" +dependencies = [ + "include_dir", +] + +[[package]] +name = "lingua-chinese-language-model" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "21ca7fa9f7671d684c82c168725f380fc873f14d6f4e8c82f0da681bcc0048d1" +dependencies = [ + "include_dir", +] + +[[package]] +name = "lingua-croatian-language-model" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d86b1346b98397e7cf87e8f3fe544fce6514918e6bde253a014eea7ca03c31d8" +dependencies = [ + "include_dir", +] + +[[package]] +name = "lingua-czech-language-model" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "348f06e4f90e1c2bc597ea3caf52abb9d3dd09ba227fa7cf4f2dc723a9810e98" +dependencies = [ + "include_dir", +] + +[[package]] +name = "lingua-danish-language-model" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "363953413132601a06fe0cae1fffb5d519d0cfb378049cc78b3893edd03a194f" +dependencies = [ + "include_dir", +] + +[[package]] +name = "lingua-dutch-language-model" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fe0da523f4726c05ed557b5a309315c1c161a5ec9ca6036c6c1c799ce492b698" +dependencies = [ + "include_dir", +] + +[[package]] +name = "lingua-english-language-model" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97102de08b134a49f1cce05a1b6f5bf08ef21fe858074ae2b794e7892c43dd4b" +dependencies = [ + "include_dir", +] + +[[package]] +name = "lingua-esperanto-language-model" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13d2a7ae84bfa3ed2a05815ca4a81d34cee7da1a77e830ab933c4debc21d6532" +dependencies = [ + "include_dir", +] + +[[package]] +name = "lingua-estonian-language-model" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7095f107a6e89147a9066db6a3f9e1b0a4706323d59a4bb58fa8218d74c1fde8" +dependencies = [ + "include_dir", +] + +[[package]] +name = "lingua-finnish-language-model" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f5b3c44089812704148c89dbe77fb34122c2f6e8182bbf0f72ea69c92e948e9e" +dependencies = [ + "include_dir", +] + +[[package]] +name = "lingua-french-language-model" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "45508227e42c9cc5eb202e17c4c40e38ea7b9be9421faeb3ab0fb7ac37d9c681" +dependencies = [ + "include_dir", +] + +[[package]] +name = "lingua-ganda-language-model" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f7d4b9fba87a72ae0f929fbe0bdc78f71be78d14475219718e70fd686b281082" +dependencies = [ + "include_dir", +] + +[[package]] +name = "lingua-georgian-language-model" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73f3568595e73295b905ec54cc2dbda915efaee5f7abf91d572e9d44a843e81d" +dependencies = [ + "include_dir", +] + +[[package]] +name = "lingua-german-language-model" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f584da803e8c135ea22dd3ed321a6b3e2ff3465559606be0a597924ecd465cb2" +dependencies = [ + "include_dir", +] + +[[package]] +name = "lingua-greek-language-model" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a5da9a688ceb41963f3555526a2e4431e0e5208a3320e56b72b1de998ba6b1c7" +dependencies = [ + "include_dir", +] + +[[package]] +name = "lingua-gujarati-language-model" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e495d0421f5e4d6b1d71ebf3a98b445c5ada6b03f033b6b62db2382e4d075663" +dependencies = [ + "include_dir", +] + +[[package]] +name = "lingua-hebrew-language-model" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc49d9a174de884bba595c8b575ee37fdb1c1a728eeff608d43e1a7378048545" +dependencies = [ + "include_dir", +] + +[[package]] +name = "lingua-hindi-language-model" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dca88af8dede1a30fcd79318b9c22f3662785fd87ed5b39881d19770651bf720" +dependencies = [ + "include_dir", +] + +[[package]] +name = "lingua-hungarian-language-model" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d7072b2cdf43438214e82b53c51ad85a16d657e5c07a5a77bef78a4cd39423c" +dependencies = [ + "include_dir", +] + +[[package]] +name = "lingua-icelandic-language-model" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "341fa17414af93a6ccf33c6efda9e65d1476a18785b075a16d53f66c9e19379d" +dependencies = [ + "include_dir", +] + +[[package]] +name = "lingua-indonesian-language-model" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fe1f2e05145c3fb4172c01c4563f729938c85f77742948444523dfcda2c3e80d" +dependencies = [ + "include_dir", +] + +[[package]] +name = "lingua-irish-language-model" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5099708a7fc7081e4171dbc0905691e5e02639d8899558a105720f28522261db" +dependencies = [ + "include_dir", +] + +[[package]] +name = "lingua-italian-language-model" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "83a87385ee42f6a0306df81066d4f77c225a337137fc15978a4b319c6bcf4bfe" +dependencies = [ + "include_dir", +] + +[[package]] +name = "lingua-japanese-language-model" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df0938f75de3ae5dcdc925d823ed409854ca14f6a653782b9a1ad5d899462fbe" +dependencies = [ + "include_dir", +] + +[[package]] +name = "lingua-kazakh-language-model" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af90d71a099b11f3b0677402a9894e95b0acb9459343735dfaec57af6be11ee6" +dependencies = [ + "include_dir", +] + +[[package]] +name = "lingua-korean-language-model" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aa87f6c43ff894fc75159c021480d2fdf96882bf5bd235f8916ceb6b7caae561" +dependencies = [ + "include_dir", +] + +[[package]] +name = "lingua-latin-language-model" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "757c1330e893bc532559397d976656a7bb0a42f391f7c946727ffb43a3761e46" +dependencies = [ + "include_dir", +] + +[[package]] +name = "lingua-latvian-language-model" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1b2f30766e9e1ce10d960ce3eb2da6514fd0ee18c8d617a566b03126140e656" +dependencies = [ + "include_dir", +] + +[[package]] +name = "lingua-lithuanian-language-model" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4e4a2a1994e48841c1d2d63099fbd68476615bf0d833fa14a3654f7043e38fc7" +dependencies = [ + "include_dir", +] + +[[package]] +name = "lingua-macedonian-language-model" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "779ea3952ecade3aff6a34eea6f432a0d10652fafd9723a1b2258551b08f83b4" +dependencies = [ + "include_dir", +] + +[[package]] +name = "lingua-malay-language-model" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "10936b2ac22076f9dbb968c71aacbf16c71ed36a92da10f077850d7ec5ded341" +dependencies = [ + "include_dir", +] + +[[package]] +name = "lingua-maori-language-model" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d5bbc1e34392b0081209eec7d6b201a062612717c0670dad451066d9026261da" +dependencies = [ + "include_dir", +] + +[[package]] +name = "lingua-marathi-language-model" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb68fa15d9a57a2ca2a71f8f59f455267bd8a8793216449697c2dd04898915a0" +dependencies = [ + "include_dir", +] + +[[package]] +name = "lingua-mongolian-language-model" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a72e6a2a3c9ecb22dbd8870bae40c167f767defd69f05c58e5467d233b6360d5" +dependencies = [ + "include_dir", +] + +[[package]] +name = "lingua-nynorsk-language-model" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2f5e7e54f8e04a5286aab3a8e22f395b42e9341cab1fdafdd04660a17e81fe8d" +dependencies = [ + "include_dir", +] + +[[package]] +name = "lingua-persian-language-model" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ddf084e05d33ed66d64461e6397b571fdf74fe93c13c4ac4c84a633e33d2d07b" +dependencies = [ + "include_dir", +] + +[[package]] +name = "lingua-polish-language-model" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72eb03e7940b3178c152138a16976e374ed0a3ddc80dc6f0b56984ab1189cb67" +dependencies = [ + "include_dir", +] + +[[package]] +name = "lingua-portuguese-language-model" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "432eda7984456055033ffa168037be9afe0f5c9ecd891bf5f27435247d496b47" +dependencies = [ + "include_dir", +] + +[[package]] +name = "lingua-punjabi-language-model" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "620ec0c678bfca1b99d5379f8024cfed24149556a0d55957799d870f3c71df26" +dependencies = [ + "include_dir", +] + +[[package]] +name = "lingua-romanian-language-model" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1051aa082754acfc52110456e29fbbf6d2c648da9a4753a861c34f19325cfab8" +dependencies = [ + "include_dir", +] + +[[package]] +name = "lingua-russian-language-model" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4fc0850578299531b50192be2c1de1af651f1289784253645e737fe456a6a138" +dependencies = [ + "include_dir", +] + +[[package]] +name = "lingua-serbian-language-model" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df46949714d864f81138b3590878c89f06cf5d0af0e758a1e1217dd13983a896" +dependencies = [ + "include_dir", +] + +[[package]] +name = "lingua-shona-language-model" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85ca30d90634fc427e79dddfa555524f5983a630ce809aeb02b708f434937965" +dependencies = [ + "include_dir", +] + +[[package]] +name = "lingua-slovak-language-model" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "529a596f3d90b2051b3ead90e864f6ad9e14b43716c3f252268e9e6e43ab28de" +dependencies = [ + "include_dir", +] + +[[package]] +name = "lingua-slovene-language-model" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db52978574611533d873b727281d1c1d0ab2b031331654c1729723b682bc76a1" +dependencies = [ + "include_dir", +] + +[[package]] +name = "lingua-somali-language-model" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be79a11e6d682c7f1ecb342fe41169a59d6adb5eac3b4de77973fb7bbf936e39" +dependencies = [ + "include_dir", +] + +[[package]] +name = "lingua-sotho-language-model" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c476a7019592446989258b7f204cc7b5429b42a4d7eb686b80461a5f790e08bc" +dependencies = [ + "include_dir", +] + +[[package]] +name = "lingua-spanish-language-model" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "56395a8d96c892130a9efb433c5f042977fb75ae5fb6e0058c8814a947459137" +dependencies = [ + "include_dir", +] + +[[package]] +name = "lingua-swahili-language-model" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "218763c238936496b53d88535e1a4db1eda30c278af682f02d6b836edb08162a" +dependencies = [ + "include_dir", +] + +[[package]] +name = "lingua-swedish-language-model" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77c965c5b11d6e1e619a98e393d16f73a0754cb05b5b067b537fd25e9b74bec0" +dependencies = [ + "include_dir", +] + +[[package]] +name = "lingua-tagalog-language-model" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fe7e48a2d97dc20ebe3618c794d45c363299e4930cc2ff2fd21b174130d75e8d" +dependencies = [ + "include_dir", +] + +[[package]] +name = "lingua-tamil-language-model" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "212e6ed4c50fc9b06278f5b401ba17eeece43ccc84515e95ece76efd16e55d9b" +dependencies = [ + "include_dir", +] + +[[package]] +name = "lingua-telugu-language-model" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af1a68d768b356c89f0a68280ab9a4adec487ba3b0ba1d2dda244a86ed712797" +dependencies = [ + "include_dir", +] + +[[package]] +name = "lingua-thai-language-model" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2582eeabf02e39680856c3f88d3b93fcdb351009407d38a4735326c8b0a33bce" +dependencies = [ + "include_dir", +] + +[[package]] +name = "lingua-tsonga-language-model" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "86e34d4bff2ed22b8b666b67a9a019c94fc01097764d7d7bc0a4f943abf88efd" +dependencies = [ + "include_dir", +] + +[[package]] +name = "lingua-tswana-language-model" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9aa7f52c30b53ed63ac9bf9ddefaf746c983e668f917f4133d4cda39a009b7e1" +dependencies = [ + "include_dir", +] + +[[package]] +name = "lingua-turkish-language-model" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a2ab6b46c596ebd7f58e0db7a3d732c42a2fa48b77094a0737e920a5c47dca0" +dependencies = [ + "include_dir", +] + +[[package]] +name = "lingua-ukrainian-language-model" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b86175a68d53b3b1e3aaf2e2fa61d31e8f03df7e0fcb51d7e1dd62fce86cb393" +dependencies = [ + "include_dir", +] + +[[package]] +name = "lingua-urdu-language-model" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d308fdae7c8caa05ffe455e8a2e568e5f018cb2742f588a810ecf306b890622" +dependencies = [ + "include_dir", +] + +[[package]] +name = "lingua-vietnamese-language-model" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "45ef9a92dce65c6e9fea7f2343e51de8b5b9cd764fa8ae3592c4ba3edd992d4a" +dependencies = [ + "include_dir", +] + +[[package]] +name = "lingua-welsh-language-model" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1057abcd15da2c81baafe25cefa148b5b56d382d645af749084987e1a6d576e2" +dependencies = [ + "include_dir", +] + +[[package]] +name = "lingua-xhosa-language-model" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d165e4dfe98f8ed1a9f768e5fefe13d472ad1c5600e7ceab6a1a4a942be4c2ee" +dependencies = [ + "include_dir", +] + +[[package]] +name = "lingua-yoruba-language-model" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6da42216997d892ce8822eae591084555469e1d32e44fe6e181996fb200492d3" +dependencies = [ + "include_dir", +] + +[[package]] +name = "lingua-zulu-language-model" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e429650453aa15779df3560ce9150c1cdfa48bbb8975e95a925cf868b2a3dc0a" +dependencies = [ + "include_dir", +] + [[package]] name = "linked-hash-map" version = "0.5.6" @@ -6121,6 +6930,12 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c41e0c4fef86961ac6d6f8a82609f55f31b05e4fce149ac5710e439df7619ba4" +[[package]] +name = "maplit" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3e2e65a1a2e43cfcb47a895c4c8b10d1f4a61097f9f254f183aee60cad9c651d" + [[package]] name = "mappings" version = "0.7.1" @@ -9800,6 +10615,17 @@ dependencies = [ "typeid", ] +[[package]] +name = "serde-wasm-bindgen" +version = "0.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8302e169f0eddcc139c70f139d19d6467353af16f9fce27e8c30158036a1e16b" +dependencies = [ + "js-sys", + "serde", + "wasm-bindgen", +] + [[package]] name = "serde_bytes" version = "0.11.19" @@ -13283,15 +14109,6 @@ dependencies = [ "winapi", ] -[[package]] -name = "whatlang" -version = "0.18.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f5e8f38b596e2a359b755342473520a99421e43658548c79489ee221b728c107" -dependencies = [ - "hashbrown 0.15.5", -] - [[package]] name = "which" version = "7.0.3" diff --git a/Cargo.toml b/Cargo.toml index 79bbd5591f..aa2fc058bd 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -124,6 +124,7 @@ lettre = { version = "0.11.19", default-features = false, features = [ "tokio1", "tokio1-rustls", ] } +lingua = "=1.8.0" linkify = "0.11.0" lz4_flex = { version = "0.11.5", default-features = false, features = [ "checked-decode", @@ -248,7 +249,6 @@ webauthn-rs = "0.5.5" webauthn-rs-proto = "0.5.5" webp = { version = "0.3.1", default-features = false } webview2-com = "0.38.0" # Should be updated in lockstep with wry -whatlang = "0.18.0" whoami = "1.6.1" windows = "=0.61.3" # Locked on 0.61 until we can update windows-core to 0.62 windows-core = "=0.61.2" # Locked on 0.61 until webview2-com updates to 0.62 diff --git a/apps/labrinth/Cargo.toml b/apps/labrinth/Cargo.toml index f2325ec3ba..9f8599feb5 100644 --- a/apps/labrinth/Cargo.toml +++ b/apps/labrinth/Cargo.toml @@ -72,6 +72,7 @@ image = { workspace = true, features = [ itertools = { workspace = true } json-patch = { workspace = true } lettre = { workspace = true } +lingua = { workspace = true } linkify = { workspace = true } modrinth-content-management = { workspace = true } modrinth-util = { workspace = true, features = ["decimal", "sentry", "utoipa"] } @@ -138,7 +139,6 @@ webauthn-rs = { workspace = true, features = [ ] } webauthn-rs-proto = { workspace = true } webp = { workspace = true } -whatlang = { workspace = true } woothee = { workspace = true } xredis = { workspace = true } yaserde = { workspace = true, features = ["derive"] } diff --git a/apps/labrinth/src/routes/v3/projects/validate.rs b/apps/labrinth/src/routes/v3/projects/validate.rs index 877242e7b2..8fe0e2d745 100644 --- a/apps/labrinth/src/routes/v3/projects/validate.rs +++ b/apps/labrinth/src/routes/v3/projects/validate.rs @@ -67,12 +67,17 @@ pub(crate) async fn ensure_project_is_valid_for_review( .collect::>(); let project = Project::from(reloaded_project.clone()); - if has_required_nags_with_context( - &project, - &versions, - &available_categories, - &disclosures, - ) { + let has_required_nags = web::block(move || { + has_required_nags_with_context( + &project, + &versions, + &available_categories, + &disclosures, + ) + }) + .await + .wrap_internal_err("validating project for review")?; + if has_required_nags { return Err(ApiError::Request(eyre!( "project must have no required validation nags before or while under review" ))); @@ -157,12 +162,15 @@ pub async fn validate( .collect::>(); let project = Project::from(project); - Ok(web::Json(ProjectValidationResponse { - nags: validate_project( + let nags = web::block(move || { + validate_project( &project, &versions, &available_categories, &disclosures, - ), - })) + ) + }) + .await + .wrap_internal_err("validating project")?; + Ok(web::Json(ProjectValidationResponse { nags })) } diff --git a/apps/labrinth/src/validate/project/language.rs b/apps/labrinth/src/validate/project/language.rs new file mode 100644 index 0000000000..bb1ccc8362 --- /dev/null +++ b/apps/labrinth/src/validate/project/language.rs @@ -0,0 +1,487 @@ +use std::collections::{HashMap, VecDeque}; +use std::sync::{Arc, LazyLock, Mutex}; + +use lingua::{Language, LanguageDetector, LanguageDetectorBuilder}; +use regex::Regex; +use unicode_normalization::UnicodeNormalization; +use unicode_segmentation::UnicodeSegmentation; + +const MIN_ENGLISH_TO_BEST_RATIO: f64 = 0.5; +const MIN_DESCRIPTION_CONFIDENCE: f64 = 0.35; +const MIN_DESCRIPTION_MARGIN: f64 = 0.15; +const MIN_NON_ENGLISH_CONFIDENCE: f64 = 0.8; +const MIN_NON_LATIN_CONFIDENCE: f64 = 0.5; +const MIN_DESCRIPTION_ENGLISH_PROPORTION: f64 = 0.2; +const MIN_PASSAGE_WORDS: usize = 4; +const MIN_PASSAGE_CHARS: usize = 25; + +static DETECTOR: LazyLock = + LazyLock::new(|| DetectorState::new()); + +struct BoundedCache { + entries: HashMap, (T, usize)>, + order: VecDeque>, + bytes: usize, +} + +impl BoundedCache { + fn new() -> Self { + Self { + entries: HashMap::new(), + order: VecDeque::new(), + bytes: 0, + } + } + + fn get(&self, text: &str) -> Option { + self.entries.get(text).map(|(value, _)| value.clone()) + } + + fn insert(&mut self, text: &str, value: T, value_bytes: usize) { + const MAX_BYTES: usize = 2 * 1024 * 1024; + const MAX_ENTRIES: usize = 2048; + let bytes = text.len() + value_bytes; + if bytes > MAX_BYTES || self.entries.contains_key(text) { + return; + } + while self.bytes + bytes > MAX_BYTES + || self.entries.len() >= MAX_ENTRIES + { + let Some(key) = self.order.pop_front() else { + break; + }; + if let Some((_, bytes)) = self.entries.remove(&key) { + self.bytes -= bytes; + } + } + let key: Arc = text.into(); + self.order.push_back(key.clone()); + self.entries.insert(key, (value, bytes)); + self.bytes += bytes; + } +} + +struct DetectorState { + detector: LanguageDetector, + scores: Mutex>, + spans: Mutex>>, +} + +impl DetectorState { + fn new() -> Self { + Self { + detector: LanguageDetectorBuilder::from_all_languages().build(), + scores: Mutex::new(BoundedCache::new()), + spans: Mutex::new(BoundedCache::new()), + } + } +} + +static WORD: LazyLock = + LazyLock::new(|| Regex::new(r"[\p{L}\p{M}\p{N}]+").unwrap()); + +static NON_LATIN_LETTER: LazyLock = + LazyLock::new(|| Regex::new(r"[\p{Alphabetic}&&[^\p{Latin}]]").unwrap()); +const MIN_DESCRIPTION_WORDS: usize = 8; +const MIN_DESCRIPTION_CHARS: usize = 35; +const MIN_ENGLISH_GRAMMATICAL_WORDS: usize = 2; + +struct Passage { + text: String, + language: Option, + confidence: f64, + runner_up_confidence: f64, + eligible: bool, + qualifies_as_english: bool, + confidently_non_english: bool, + alphabetic_words: usize, + english_grammatical_words: usize, +} + +#[derive(Clone, Copy, Default)] +struct Detection { + runner_up_confidence: f64, + language: Option, + confidence: f64, + english_confidence: f64, +} + +impl Detection { + fn is_english(&self) -> bool { + self.english_to_best_ratio() >= MIN_ENGLISH_TO_BEST_RATIO + } + + fn is_confidently_non_english(&self, text: &str) -> bool { + let has_non_latin_evidence = + NON_LATIN_LETTER.find_iter(text).count() >= 5; + let minimum_confidence = if has_non_latin_evidence { + MIN_NON_LATIN_CONFIDENCE + } else { + MIN_NON_ENGLISH_CONFIDENCE + }; + self.confidence >= minimum_confidence + && !self.is_english() + && (has_non_latin_evidence + || (text.graphemes(true).count() >= MIN_PASSAGE_CHARS + && alphabetic_word_count(text) >= MIN_PASSAGE_WORDS)) + } + + fn english_to_best_ratio(&self) -> f64 { + if self.confidence > 0.0 { + self.english_confidence / self.confidence + } else { + 0.0 + } + } +} + +fn detect(text: &str, detector: &DetectorState) -> Detection { + if let Some(value) = detector + .scores + .lock() + .unwrap_or_else(|error| error.into_inner()) + .get(text) + { + return value; + } + let scores = detector.detector.compute_language_confidence_values(text); + let english_confidence = scores + .iter() + .find(|(language, _)| *language == Language::English) + .map_or(0.0, |(_, confidence)| *confidence); + let best = scores.first().filter(|(_, confidence)| *confidence > 0.0); + let detection = Detection { + runner_up_confidence: scores.get(1).map_or(0.0, |(_, score)| *score), + language: best.map(|(language, _)| *language), + confidence: best.map_or(0.0, |(_, confidence)| *confidence), + english_confidence, + }; + detector + .scores + .lock() + .unwrap_or_else(|error| error.into_inner()) + .insert(text, detection, std::mem::size_of::()); + detection +} + +/// Use Lingua's inferred boundaries, then rescore each span independently so +/// English can qualify even when another language has the highest score. +fn mixed_language_passages( + text: &str, + detector: &DetectorState, +) -> Vec { + let cached = detector + .spans + .lock() + .unwrap_or_else(|error| error.into_inner()) + .get(text); + let spans = cached.unwrap_or_else(|| { + let spans: Arc<[(usize, usize)]> = detector + .detector + .detect_multiple_languages_of(text) + .into_iter() + .map(|span| (span.start_index(), span.end_index())) + .collect(); + detector + .spans + .lock() + .unwrap_or_else(|error| error.into_inner()) + .insert(text, spans.clone(), std::mem::size_of_val(spans.as_ref())); + spans + }); + spans + .iter() + .map(|(start, end)| text[*start..*end].trim().to_owned()) + .collect() +} + +fn has_enough_description_content(text: &str) -> bool { + alphabetic_word_count(text) >= MIN_DESCRIPTION_WORDS + && text.trim().graphemes(true).count() >= MIN_DESCRIPTION_CHARS +} + +/// Keep script runs disjoint so surrounding foreign text cannot qualify a short English fragment. +fn script_passages(text: &str) -> Vec<&str> { + let mut passages = Vec::new(); + let mut start = 0; + let mut previous = None; + for (index, character) in text.char_indices() { + if !character.is_alphabetic() { + continue; + } + let mut buffer = [0; 4]; + let non_latin = + NON_LATIN_LETTER.is_match(character.encode_utf8(&mut buffer)); + if previous.is_some_and(|previous| previous != non_latin) { + passages.push(text[start..index].trim()); + start = index; + } + previous = Some(non_latin); + } + if !text[start..].trim().is_empty() { + passages.push(text[start..].trim()); + } + passages +} + +fn description_passage(text: &str) -> Passage { + let mut passage = classify_passage( + text.to_owned(), + has_enough_description_content(text), + 1.0, + &DETECTOR, + has_enough_description_content(text) + || NON_LATIN_LETTER.find_iter(text).count() >= 5, + ); + let strong = passage.confidence >= MIN_DESCRIPTION_CONFIDENCE + && passage.confidence - passage.runner_up_confidence + >= MIN_DESCRIPTION_MARGIN; + let foreign_eligible = + passage.eligible || NON_LATIN_LETTER.find_iter(text).count() >= 5; + passage.qualifies_as_english = passage.eligible + && strong + && (passage.english_grammatical_words >= MIN_ENGLISH_GRAMMATICAL_WORDS + || has_lowercase_prose(text)) + && passage.language == Some(Language::English); + passage.confidently_non_english = foreign_eligible + && strong + && passage + .language + .is_some_and(|language| language != Language::English); + passage.eligible = foreign_eligible; + passage +} + +/// Capitalized names and acronyms alone do not establish English prose. +fn has_lowercase_prose(text: &str) -> bool { + WORD.find_iter(text) + .filter(|word| { + let word = word.as_str(); + word.chars().all(char::is_lowercase) && word.chars().count() > 1 + }) + .take(2) + .count() + >= 2 +} + +fn summary_translation_passage(text: &str) -> Passage { + let mut passage = description_passage(text); + if passage.qualifies_as_english + && passage.english_grammatical_words < MIN_ENGLISH_GRAMMATICAL_WORDS + { + passage.qualifies_as_english = false; + } + passage +} + +/// Bound detector input without overlapping or splitting words. +fn bounded_passages(text: &str) -> Vec<&str> { + let mut passages = Vec::new(); + let mut start = 0; + for (index, character) in text.char_indices() { + if index - start >= 4000 && character.is_whitespace() { + passages.push(text[start..index].trim()); + start = index + character.len_utf8(); + } + } + if !text[start..].trim().is_empty() { + passages.push(text[start..].trim()); + } + passages +} + +/// Only replace a coherent block when Lingua finds independently supported English and foreign portions. +fn description_passages(text: &str) -> Vec { + bounded_passages(text) + .into_iter() + .flat_map(script_passages) + .flat_map(|text| { + let whole = description_passage(text); + if whole.alphabetic_words < MIN_DESCRIPTION_WORDS * 2 { + return vec![whole]; + } + let spans = mixed_language_passages(text, &DETECTOR); + if spans.len() <= 1 { + return vec![whole]; + } + let candidates = spans + .iter() + .map(|span| description_passage(span)) + .collect::>(); + if candidates.iter().any(|span| span.qualifies_as_english) + && candidates.iter().any(|span| span.confidently_non_english) + { + candidates + } else { + vec![whole] + } + }) + .collect() +} + +fn alphabetic_word_count(text: &str) -> usize { + WORD.find_iter(text) + .filter(|word| word.as_str().chars().any(char::is_alphabetic)) + .count() +} + +/// Require grammatical context before treating technical names as an English translation. +fn english_grammatical_word_count(text: &str) -> usize { + WORD.find_iter(text) + .filter(|word| { + matches!( + word.as_str().to_lowercase().as_str(), + "a" | "an" + | "the" | "this" + | "that" | "these" + | "those" | "it" + | "its" | "you" | "your" + | "we" | "our" | "they" + | "their" | "to" + | "for" | "with" + | "from" | "in" | "on" + | "of" | "and" | "or" + | "by" | "into" | "is" + | "are" | "be" | "can" + | "will" | "has" + | "have" | "does" + | "not" + ) + }) + .count() +} + +fn is_eligible_passage(text: &str) -> bool { + alphabetic_word_count(text) >= MIN_PASSAGE_WORDS + && text.graphemes(true).count() >= MIN_PASSAGE_CHARS +} + +fn classify_passage( + text: String, + eligible: bool, + minimum_ratio: f64, + detector: &DetectorState, + should_detect: bool, +) -> Passage { + let detection = if should_detect { + detect(&text, detector) + } else { + Detection::default() + }; + Passage { + runner_up_confidence: detection.runner_up_confidence, + qualifies_as_english: eligible + && detection.english_to_best_ratio() >= minimum_ratio, + confidently_non_english: eligible + && detection.is_confidently_non_english(&text), + alphabetic_words: alphabetic_word_count(&text), + english_grammatical_words: english_grammatical_word_count(&text), + text, + language: detection.language, + confidence: detection.confidence, + eligible, + } +} + +/// A foreign summary needs a contiguous English passage, not scattered product names. +fn summary_rescue_passages(text: &str) -> Vec { + script_passages(text) + .into_iter() + .filter(|text| !NON_LATIN_LETTER.is_match(text)) + .flat_map(|text| { + let normalized = + text.split_whitespace().collect::>().join(" "); + let whole = summary_translation_passage(&normalized); + if whole.qualifies_as_english + || !whole.eligible + || whole.english_grammatical_words + < MIN_ENGLISH_GRAMMATICAL_WORDS + { + return vec![whole]; + } + let spans = mixed_language_passages(&normalized, &DETECTOR); + if spans.len() <= 1 { + return vec![whole]; + } + spans + .into_iter() + .map(|span| summary_translation_passage(&span)) + .collect() + }) + .collect() +} + +pub(super) fn is_likely_english_summary(text: &str) -> bool { + let normalized: String = text.nfkc().collect(); + let mut passages = vec![classify_passage( + normalized.clone(), + true, + MIN_ENGLISH_TO_BEST_RATIO, + &DETECTOR, + true, + )]; + let requires_strong_rescue = passages[0].confidently_non_english; + if requires_strong_rescue { + passages.extend(summary_rescue_passages(&normalized)); + } else if NON_LATIN_LETTER.is_match(&normalized) { + let latin = NON_LATIN_LETTER.replace_all(&normalized, " "); + passages[0].qualifies_as_english = false; + passages.push(classify_passage( + latin.to_string(), + is_eligible_passage(&latin), + MIN_ENGLISH_TO_BEST_RATIO, + &DETECTOR, + is_eligible_passage(&latin), + )); + } else if !passages[0].qualifies_as_english { + passages.extend( + mixed_language_passages(&normalized, &DETECTOR) + .into_iter() + .map(|text| { + let eligible = is_eligible_passage(&text); + classify_passage( + text, + eligible, + MIN_ENGLISH_TO_BEST_RATIO, + &DETECTOR, + eligible, + ) + }), + ); + } + passages.iter().any(|passage| passage.qualifies_as_english) + || !passages + .iter() + .any(|passage| passage.confidently_non_english) +} + +pub(super) fn has_sufficient_english_blocks(blocks: &[String]) -> bool { + let passages = blocks + .iter() + .flat_map(|block| { + let normalized: String = block.nfkc().collect(); + description_passages(&normalized) + }) + .collect::>(); + let english_utf16_length: usize = passages + .iter() + .filter(|passage| passage.qualifies_as_english) + .map(|passage| passage.text.encode_utf16().count()) + .sum(); + let total_utf16_length: usize = passages + .iter() + .filter(|passage| passage.eligible) + .map(|passage| passage.text.encode_utf16().count()) + .sum(); + let has_foreign_evidence = passages + .iter() + .any(|passage| passage.confidently_non_english); + let english_proportion = if total_utf16_length == 0 { + 0.0 + } else { + english_utf16_length as f64 / total_utf16_length as f64 + }; + + !has_foreign_evidence + || english_proportion >= MIN_DESCRIPTION_ENGLISH_PROPORTION +} diff --git a/apps/labrinth/src/validate/project/mod.rs b/apps/labrinth/src/validate/project/mod.rs index 28ea0df273..a9f94f6242 100644 --- a/apps/labrinth/src/validate/project/mod.rs +++ b/apps/labrinth/src/validate/project/mod.rs @@ -9,6 +9,7 @@ mod description; mod disclosures; mod gallery; mod icon; +mod language; mod license; mod links; mod moderation; diff --git a/apps/labrinth/src/validate/project/summary.rs b/apps/labrinth/src/validate/project/summary.rs index 43ac090a06..23f14ea2ec 100644 --- a/apps/labrinth/src/validate/project/summary.rs +++ b/apps/labrinth/src/validate/project/summary.rs @@ -132,7 +132,7 @@ fn requires_language_nag( contains_link: bool, has_spam: bool, ) -> bool { - is_non_english_text(project, normalized_summary) - && !contains_link + !contains_link && !has_spam + && is_non_english_text(project, normalized_summary) } diff --git a/apps/labrinth/src/validate/project/text.rs b/apps/labrinth/src/validate/project/text.rs index a4efd930ec..0dda85d15d 100644 --- a/apps/labrinth/src/validate/project/text.rs +++ b/apps/labrinth/src/validate/project/text.rs @@ -1,4 +1,4 @@ -use std::collections::{BTreeMap, BTreeSet}; +use std::collections::BTreeMap; use std::sync::LazyLock; use linkify::{LinkFinder, LinkKind}; @@ -7,15 +7,16 @@ use regex::Regex; use unicode_normalization::UnicodeNormalization; use unicode_segmentation::UnicodeSegmentation; use url::Url; -use whatlang::{Detector, Lang}; use crate::models::exp::minecraft::Language; use crate::models::projects::Project; +pub(super) use super::language::{ + has_sufficient_english_blocks, is_likely_english_summary, +}; + static WORD: LazyLock = LazyLock::new(|| Regex::new(r"[\p{L}\p{M}\p{N}]+").unwrap()); -static NON_LATIN_LETTER: LazyLock = - LazyLock::new(|| Regex::new(r"[\p{Alphabetic}&&[^\p{Latin}]]").unwrap()); static SPAM_TOKEN: LazyLock = LazyLock::new(|| { Regex::new(r#"[\p{L}\p{M}\p{N}]+(?:['_\u{2019}.:+/-][\p{L}\p{M}\p{N}]+)*"#) .unwrap() @@ -25,7 +26,6 @@ static SUMMARY_LINK_FINDER: LazyLock = LazyLock::new(|| { finder.kinds(&[LinkKind::Url]).url_must_have_scheme(false); finder }); -static LANGUAGE_DETECTOR: LazyLock = LazyLock::new(Detector::new); static MARKDOWN_LINK: LazyLock = LazyLock::new(|| Regex::new(r"!?\[([^\]]*)\]\([^)]+\)").unwrap()); static HTML_TAG: LazyLock = @@ -39,8 +39,6 @@ static HTML_CLOSE_TAG: LazyLock = LazyLock::new(|| Regex::new(r"(?is)").unwrap()); static CODE_BLOCK: LazyLock = LazyLock::new(|| Regex::new(r"(?s)```.*?```").unwrap()); -static DESCRIPTION_BLOCK_BREAK: LazyLock = - LazyLock::new(|| Regex::new(r"\n\s*\n+").unwrap()); static INLINE_CODE: LazyLock = LazyLock::new(|| Regex::new(r"`([^`]*)`").unwrap()); static MARKDOWN_IMAGE: LazyLock = @@ -919,7 +917,7 @@ pub(super) fn has_paired_html_formatting(text: &str) -> bool { }) } -pub(super) fn extract_description_text(markdown: &str) -> String { +fn strip_description_markup(markdown: &str) -> String { let without_code = INLINE_CODE.replace_all(markdown, " "); let with_image_alt = MARKDOWN_IMAGE.replace_all(&without_code, "$1"); let without_links = MARKDOWN_LINK.replace_all(&with_image_alt, " "); @@ -934,8 +932,11 @@ pub(super) fn extract_description_text(markdown: &str) -> String { .map_or_else(|| " ".to_owned(), |alt| alt.as_str().to_owned()) }, ); - let without_html = HTML_TAG.replace_all(&with_html_image_alt, " "); - without_html + HTML_TAG.replace_all(&with_html_image_alt, " ").into_owned() +} + +pub(super) fn extract_description_text(markdown: &str) -> String { + strip_description_markup(markdown) .lines() .map(|line| line.trim_start_matches(['>', '#'])) .collect::>() @@ -946,12 +947,96 @@ pub(super) fn extract_description_text(markdown: &str) -> String { .join(" ") } +static LANGUAGE_CONFIG_ENTRY: LazyLock = LazyLock::new(|| { + Regex::new(r"^([A-Za-z_][A-Za-z0-9_.-]*):(?:\s+(.*))?$").unwrap() +}); +static LANGUAGE_IDENTIFIER: LazyLock = LazyLock::new(|| { + Regex::new(r"\b[A-Za-z][A-Za-z0-9_-]*(?:\.[A-Za-z][A-Za-z0-9_-]*)+\b") + .unwrap() +}); +static LANGUAGE_COMMAND: LazyLock = LazyLock::new(|| { + Regex::new(r"(^|[\s|,(])/[A-Za-z][A-Za-z0-9_:./-]*").unwrap() +}); +static LANGUAGE_ARGUMENT: LazyLock = + LazyLock::new(|| Regex::new(r"\[[A-Za-z][A-Za-z0-9_ /|.-]*\]").unwrap()); + +/// Remove machine syntax before punctuation normalization turns identifiers into words. +fn description_language_input(markdown: &str) -> String { + let readable = strip_description_markup(markdown); + let readable = text_without_explicit_links(&readable); + let mut in_yaml = false; + readable + .lines() + .map(|line| { + let trimmed = line.trim(); + if matches!(trimmed, "yaml" | "yml") { + in_yaml = true; + return String::new(); + } + if let Some(entry) = LANGUAGE_CONFIG_ENTRY.captures(trimmed) { + let value = entry.get(2).map_or("", |value| value.as_str()); + let machine_value = + matches!(value, "true" | "false" | "null" | "~") + || value.parse::().is_ok() + || value.starts_with(['"', '\'']); + if in_yaml || entry[1].contains('_') || machine_value { + return String::new(); + } + } else if !trimmed.is_empty() { + in_yaml = false; + } + let has_command = LANGUAGE_COMMAND.is_match(line); + let without_commands = LANGUAGE_COMMAND.replace_all(line, "$1"); + let without_identifiers = + LANGUAGE_IDENTIFIER.replace_all(&without_commands, " "); + if has_command { + LANGUAGE_ARGUMENT + .replace_all(&without_identifiers, " ") + .into_owned() + } else { + without_identifiers.into_owned() + } + }) + .collect::>() + .join("\n") +} + pub(super) fn extract_description_blocks(markdown: &str) -> Vec { - DESCRIPTION_BLOCK_BREAK - .split(markdown) - .map(extract_description_text) - .filter(|block| !block.is_empty()) - .collect() + let readable = description_language_input(markdown); + let mut blocks = Vec::new(); + let mut paragraph = Vec::new(); + let mut heading_only = false; + let mut blank = false; + for line in readable.lines() { + let line = line.trim(); + if line.is_empty() { + blank = true; + continue; + } + let heading = line.starts_with('#'); + let metadata_list = line + .split_once(':') + .is_some_and(|(_, values)| values.matches(',').count() >= 3); + if (heading + || metadata_list + || (blank + && !heading_only + && (WORD.find_iter(¶graph.join(" ")).count() >= 8 + || WORD.find_iter(line).count() >= 8))) + && !paragraph.is_empty() + { + blocks.push(extract_description_text(¶graph.join("\n"))); + paragraph.clear(); + } + paragraph.push(line); + heading_only = heading; + blank = false; + } + if !paragraph.is_empty() { + blocks.push(extract_description_text(¶graph.join("\n"))); + } + blocks.retain(|block| !block.is_empty()); + blocks } pub(super) fn has_image_without_alt_text(markdown: &str) -> bool { @@ -1013,115 +1098,6 @@ pub(super) fn project_requires_english(project: &Project) -> bool { || is_english_server } -pub(super) fn is_likely_english_summary(text: &str) -> bool { - let detection_text = normalize_language_text(text); - if has_dominant_non_latin_script(&detection_text) { - return false; - } - - if !has_enough_language_content(&detection_text) { - return true; - } - - LANGUAGE_DETECTOR - .detect(&detection_text) - .is_none_or(|info| info.lang() == Lang::Eng || !info.is_reliable()) -} - -pub(super) fn has_sufficient_english_blocks(blocks: &[String]) -> bool { - let mut english_chunks = 0; - let mut non_english_chunks = 0; - - for block in blocks { - let detection_text = normalize_language_text(block); - if has_dominant_non_latin_script(&detection_text) { - non_english_chunks += 1; - - let latin_text = NON_LATIN_LETTER.replace_all(&detection_text, " "); - if has_enough_language_content(&latin_text) - && LANGUAGE_DETECTOR - .detect(&latin_text) - .is_some_and(|info| info.lang() == Lang::Eng) - { - english_chunks += 1; - } - continue; - } - - for chunk in language_chunks(&detection_text) { - let Some(info) = LANGUAGE_DETECTOR.detect(&chunk) else { - continue; - }; - - if info.lang() == Lang::Eng { - english_chunks += 1; - } else if info.is_reliable() { - non_english_chunks += 1; - } - } - } - - let classified_chunks = english_chunks + non_english_chunks; - classified_chunks == 0 || english_chunks * 10 >= classified_chunks * 3 -} - -fn normalize_language_text(text: &str) -> String { - text.nfkc().collect() -} - -fn has_dominant_non_latin_script(text: &str) -> bool { - const MIN_NON_LATIN_LETTERS: usize = 5; - - let non_latin_letters = NON_LATIN_LETTER.find_iter(text).count(); - let alphabetic_letters = text - .chars() - .filter(|character| character.is_alphabetic()) - .count(); - non_latin_letters >= MIN_NON_LATIN_LETTERS - && non_latin_letters * 2 >= alphabetic_letters -} - -fn language_chunks(block: &str) -> Vec { - const CHUNK_WORDS: usize = 24; - const CHUNK_STRIDE_WORDS: usize = 12; - - let words = WORD - .find_iter(block) - .map(|word| word.as_str()) - .collect::>(); - if words.len() < 8 { - return Vec::new(); - } - if words.len() <= CHUNK_WORDS { - let chunk = words.join(" "); - return has_enough_language_content(&chunk) - .then_some(chunk) - .into_iter() - .collect(); - } - - let mut starts = BTreeSet::new(); - let mut start = 0; - while start + 8 <= words.len() { - starts.insert(start); - start += CHUNK_STRIDE_WORDS; - } - starts.insert(words.len() - CHUNK_WORDS); - - starts - .into_iter() - .map(|start| { - words[start..(start + CHUNK_WORDS).min(words.len())].join(" ") - }) - .filter(|chunk| has_enough_language_content(chunk)) - .collect() -} - -fn has_enough_language_content(text: &str) -> bool { - WORD.find_iter(text).count() >= 8 - && text.trim().graphemes(true).count() >= 35 -} - #[cfg(test)] mod tests { use super::{has_summary_formatting, project_text_similarity}; From d0c84ccd687c93ffdbe38dec720b7679a96441e4 Mon Sep 17 00:00:00 2001 From: tdgao Date: Mon, 14 Sep 2026 10:51:47 -0600 Subject: [PATCH 09/17] refactor: clean up description preprocessing --- apps/labrinth/src/validate/project/text.rs | 302 +++++++++++---------- 1 file changed, 153 insertions(+), 149 deletions(-) diff --git a/apps/labrinth/src/validate/project/text.rs b/apps/labrinth/src/validate/project/text.rs index 0dda85d15d..479f0dff9f 100644 --- a/apps/labrinth/src/validate/project/text.rs +++ b/apps/labrinth/src/validate/project/text.rs @@ -12,7 +12,7 @@ use crate::models::exp::minecraft::Language; use crate::models::projects::Project; pub(super) use super::language::{ - has_sufficient_english_blocks, is_likely_english_summary, + has_sufficient_english_blocks, is_likely_english_summary, }; static WORD: LazyLock = @@ -129,46 +129,46 @@ pub(super) fn normalize_project_field_text(text: &str) -> String { } pub(super) fn project_text_similarity(left: &str, right: &str) -> f64 { - let left = normalized_for_similarity(left); - let right = normalized_for_similarity(right); - let longest_length = left.len().max(right.len()); - if longest_length == 0 { - return 0.0; - } + let left = normalized_for_similarity(left); + let right = normalized_for_similarity(right); + let longest_length = left.len().max(right.len()); + if longest_length == 0 { + return 0.0; + } - 1.0 - levenshtein_distance(&left, &right) as f64 / longest_length as f64 + 1.0 - levenshtein_distance(&left, &right) as f64 / longest_length as f64 } fn normalized_for_similarity(text: &str) -> Vec { - normalize_project_field_text(text) - .to_lowercase() - .chars() - .filter(|character| !character.is_whitespace()) - .collect() + normalize_project_field_text(text) + .to_lowercase() + .chars() + .filter(|character| !character.is_whitespace()) + .collect() } fn levenshtein_distance(left: &[char], right: &[char]) -> usize { - if left.len() > right.len() { - return levenshtein_distance(right, left); - } + if left.len() > right.len() { + return levenshtein_distance(right, left); + } - let mut previous_row = (0..=left.len()).collect::>(); - for (right_index, right_character) in right.iter().enumerate() { - let mut current_row = Vec::with_capacity(left.len() + 1); - current_row.push(right_index + 1); - for (left_index, left_character) in left.iter().enumerate() { - current_row.push( - (current_row[left_index] + 1) - .min(previous_row[left_index + 1] + 1) - .min( - previous_row[left_index] - + usize::from(left_character != right_character), - ), - ); - } - previous_row = current_row; - } - previous_row[left.len()] + let mut previous_row = (0..=left.len()).collect::>(); + for (right_index, right_character) in right.iter().enumerate() { + let mut current_row = Vec::with_capacity(left.len() + 1); + current_row.push(right_index + 1); + for (left_index, left_character) in left.iter().enumerate() { + current_row.push( + (current_row[left_index] + 1) + .min(previous_row[left_index + 1] + 1) + .min( + previous_row[left_index] + + usize::from(left_character != right_character), + ), + ); + } + previous_row = current_row; + } + previous_row[left.len()] } pub(super) fn js_string_length(text: &str) -> usize { @@ -880,8 +880,8 @@ pub(super) fn find_link_or_ip(text: &str) -> Option { pub(super) fn has_summary_formatting(summary: &str) -> bool { has_paired_html_formatting(summary) || MARKDOWN_LINK.is_match(summary) - || Parser::new(summary) - .any(|event| matches!(event, Event::Start(Tag::Emphasis))) + || Parser::new(summary) + .any(|event| matches!(event, Event::Start(Tag::Emphasis))) || summary.lines().any(|line| { let line = line.trim_start(); line.starts_with('#') @@ -947,96 +947,87 @@ pub(super) fn extract_description_text(markdown: &str) -> String { .join(" ") } -static LANGUAGE_CONFIG_ENTRY: LazyLock = LazyLock::new(|| { - Regex::new(r"^([A-Za-z_][A-Za-z0-9_.-]*):(?:\s+(.*))?$").unwrap() -}); -static LANGUAGE_IDENTIFIER: LazyLock = LazyLock::new(|| { - Regex::new(r"\b[A-Za-z][A-Za-z0-9_-]*(?:\.[A-Za-z][A-Za-z0-9_-]*)+\b") - .unwrap() -}); -static LANGUAGE_COMMAND: LazyLock = LazyLock::new(|| { - Regex::new(r"(^|[\s|,(])/[A-Za-z][A-Za-z0-9_:./-]*").unwrap() -}); -static LANGUAGE_ARGUMENT: LazyLock = - LazyLock::new(|| Regex::new(r"\[[A-Za-z][A-Za-z0-9_ /|.-]*\]").unwrap()); +fn strip_dotted_identifiers(text: &str) -> String { + static IDENTIFIER: LazyLock = LazyLock::new(|| { + // matches dotted identifiers such as `com.example.mod` or `config.enabled` + Regex::new(r"\b[A-Za-z][A-Za-z0-9_-]*(?:\.[A-Za-z][A-Za-z0-9_-]*)+\b") + .unwrap() + }); + + IDENTIFIER.replace_all(text, " ").into_owned() +} + +fn strip_command_arguments(text: &str) -> String { + static ARGUMENT: LazyLock = LazyLock::new(|| { + // matches bracketed argument placeholders such as `[player]` or `[on|off]` + Regex::new(r"\[[A-Za-z][A-Za-z0-9_ /|.-]*\]").unwrap() + }); + + ARGUMENT.replace_all(text, " ").into_owned() +} + +fn strip_technical_syntax(line: &str) -> String { + static COMMAND: LazyLock = LazyLock::new(|| { + // matches slash commands such as `/help` or `/minecraft:give`. + Regex::new(r"(^|[\s|,(])/[A-Za-z][A-Za-z0-9_:./-]*").unwrap() + }); + + let without_commands = COMMAND.replace_all(line, "$1"); + let without_identifiers = strip_dotted_identifiers(&without_commands); + + if COMMAND.is_match(line) { + strip_command_arguments(&without_identifiers) + } else { + without_identifiers + } +} -/// Remove machine syntax before punctuation normalization turns identifiers into words. fn description_language_input(markdown: &str) -> String { - let readable = strip_description_markup(markdown); - let readable = text_without_explicit_links(&readable); - let mut in_yaml = false; - readable - .lines() - .map(|line| { - let trimmed = line.trim(); - if matches!(trimmed, "yaml" | "yml") { - in_yaml = true; - return String::new(); - } - if let Some(entry) = LANGUAGE_CONFIG_ENTRY.captures(trimmed) { - let value = entry.get(2).map_or("", |value| value.as_str()); - let machine_value = - matches!(value, "true" | "false" | "null" | "~") - || value.parse::().is_ok() - || value.starts_with(['"', '\'']); - if in_yaml || entry[1].contains('_') || machine_value { - return String::new(); - } - } else if !trimmed.is_empty() { - in_yaml = false; - } - let has_command = LANGUAGE_COMMAND.is_match(line); - let without_commands = LANGUAGE_COMMAND.replace_all(line, "$1"); - let without_identifiers = - LANGUAGE_IDENTIFIER.replace_all(&without_commands, " "); - if has_command { - LANGUAGE_ARGUMENT - .replace_all(&without_identifiers, " ") - .into_owned() - } else { - without_identifiers.into_owned() - } - }) - .collect::>() - .join("\n") + let readable = strip_description_markup(markdown); + let readable = text_without_explicit_links(&readable); + readable + .lines() + .map(strip_technical_syntax) + .collect::>() + .join("\n") } pub(super) fn extract_description_blocks(markdown: &str) -> Vec { - let readable = description_language_input(markdown); - let mut blocks = Vec::new(); - let mut paragraph = Vec::new(); - let mut heading_only = false; - let mut blank = false; - for line in readable.lines() { - let line = line.trim(); - if line.is_empty() { - blank = true; - continue; - } - let heading = line.starts_with('#'); - let metadata_list = line - .split_once(':') - .is_some_and(|(_, values)| values.matches(',').count() >= 3); - if (heading - || metadata_list - || (blank - && !heading_only - && (WORD.find_iter(¶graph.join(" ")).count() >= 8 - || WORD.find_iter(line).count() >= 8))) - && !paragraph.is_empty() - { - blocks.push(extract_description_text(¶graph.join("\n"))); - paragraph.clear(); - } - paragraph.push(line); - heading_only = heading; - blank = false; - } - if !paragraph.is_empty() { - blocks.push(extract_description_text(¶graph.join("\n"))); - } - blocks.retain(|block| !block.is_empty()); - blocks + let readable = description_language_input(markdown); + let mut blocks = Vec::new(); + let mut paragraph = Vec::new(); + let mut heading_only = false; + let mut blank = false; + for line in readable.lines() { + let line = line.trim(); + if line.is_empty() { + blank = true; + continue; + } + let heading = line.starts_with('#'); + let metadata_list = line + .split_once(':') + .is_some_and(|(_, values)| values.matches(',').count() >= 3); + if (heading + || metadata_list + || (blank + && !heading_only + && (WORD.find_iter(¶graph.join(" ")).count() >= 8 + || WORD.find_iter(line).count() >= 8))) + && !paragraph.is_empty() + { + blocks.push(extract_description_text(¶graph.join("\n"))); + paragraph.clear(); + } + paragraph.push(line); + heading_only = heading; + blank = false; + } + if !paragraph.is_empty() { + blocks.push(extract_description_text(¶graph.join("\n"))); + } + blocks.retain(|block| !block.is_empty()); + blocks } pub(super) fn has_image_without_alt_text(markdown: &str) -> bool { @@ -1100,37 +1091,50 @@ pub(super) fn project_requires_english(project: &Project) -> bool { #[cfg(test)] mod tests { - use super::{has_summary_formatting, project_text_similarity}; - - #[test] - fn summary_detects_markdown_emphasis() { - for summary in ["*this*", "Adds *new features* to Minecraft", "*a*", "_this_"] { - assert!(has_summary_formatting(summary), "{summary:?}"); - } - } + use super::{has_summary_formatting, project_text_similarity}; + + #[test] + fn summary_detects_markdown_emphasis() { + for summary in [ + "*this*", + "Adds *new features* to Minecraft", + "*a*", + "_this_", + ] { + assert!(has_summary_formatting(summary), "{summary:?}"); + } + } - #[test] - fn summary_allows_literal_asterisks() { - for summary in ["A single * asterisk", "2 * 3 * 4", r"\*this\*", "An unmatched *asterisk"] { - assert!(!has_summary_formatting(summary), "{summary:?}"); - } - } + #[test] + fn summary_allows_literal_asterisks() { + for summary in [ + "A single * asterisk", + "2 * 3 * 4", + r"\*this\*", + "An unmatched *asterisk", + ] { + assert!(!has_summary_formatting(summary), "{summary:?}"); + } + } - #[test] - fn similarity_ignores_case_whitespace_and_unicode_composition() { - assert_eq!(project_text_similarity(" Café tools ", "CAFE\u{301}\nTOOLS"), 1.0); - } + #[test] + fn similarity_ignores_case_whitespace_and_unicode_composition() { + assert_eq!( + project_text_similarity(" Café tools ", "CAFE\u{301}\nTOOLS"), + 1.0 + ); + } - #[test] - fn similarity_distinguishes_the_eighty_percent_boundary() { - assert!(project_text_similarity("abcde", "abcdx") >= 0.8); - assert!(project_text_similarity("abcde", "abcxy") < 0.8); - } + #[test] + fn similarity_distinguishes_the_eighty_percent_boundary() { + assert!(project_text_similarity("abcde", "abcdx") >= 0.8); + assert!(project_text_similarity("abcde", "abcxy") < 0.8); + } - #[test] - fn empty_fields_do_not_match() { - assert_eq!(project_text_similarity(" ", "\n"), 0.0); - assert_eq!(project_text_similarity("", "some text"), 0.0); - assert_eq!(project_text_similarity("some text", ""), 0.0); - } + #[test] + fn empty_fields_do_not_match() { + assert_eq!(project_text_similarity(" ", "\n"), 0.0); + assert_eq!(project_text_similarity("", "some text"), 0.0); + assert_eq!(project_text_similarity("some text", ""), 0.0); + } } From a40401748eea59ac1f6be7ab1601abf63f59b972 Mon Sep 17 00:00:00 2001 From: tdgao Date: Mon, 14 Sep 2026 11:36:56 -0600 Subject: [PATCH 10/17] remove: unnecessary caching --- .../labrinth/src/validate/project/language.rs | 118 ++---------------- 1 file changed, 13 insertions(+), 105 deletions(-) diff --git a/apps/labrinth/src/validate/project/language.rs b/apps/labrinth/src/validate/project/language.rs index bb1ccc8362..6b285941db 100644 --- a/apps/labrinth/src/validate/project/language.rs +++ b/apps/labrinth/src/validate/project/language.rs @@ -1,5 +1,4 @@ -use std::collections::{HashMap, VecDeque}; -use std::sync::{Arc, LazyLock, Mutex}; +use std::sync::LazyLock; use lingua::{Language, LanguageDetector, LanguageDetectorBuilder}; use regex::Regex; @@ -15,67 +14,8 @@ const MIN_DESCRIPTION_ENGLISH_PROPORTION: f64 = 0.2; const MIN_PASSAGE_WORDS: usize = 4; const MIN_PASSAGE_CHARS: usize = 25; -static DETECTOR: LazyLock = - LazyLock::new(|| DetectorState::new()); - -struct BoundedCache { - entries: HashMap, (T, usize)>, - order: VecDeque>, - bytes: usize, -} - -impl BoundedCache { - fn new() -> Self { - Self { - entries: HashMap::new(), - order: VecDeque::new(), - bytes: 0, - } - } - - fn get(&self, text: &str) -> Option { - self.entries.get(text).map(|(value, _)| value.clone()) - } - - fn insert(&mut self, text: &str, value: T, value_bytes: usize) { - const MAX_BYTES: usize = 2 * 1024 * 1024; - const MAX_ENTRIES: usize = 2048; - let bytes = text.len() + value_bytes; - if bytes > MAX_BYTES || self.entries.contains_key(text) { - return; - } - while self.bytes + bytes > MAX_BYTES - || self.entries.len() >= MAX_ENTRIES - { - let Some(key) = self.order.pop_front() else { - break; - }; - if let Some((_, bytes)) = self.entries.remove(&key) { - self.bytes -= bytes; - } - } - let key: Arc = text.into(); - self.order.push_back(key.clone()); - self.entries.insert(key, (value, bytes)); - self.bytes += bytes; - } -} - -struct DetectorState { - detector: LanguageDetector, - scores: Mutex>, - spans: Mutex>>, -} - -impl DetectorState { - fn new() -> Self { - Self { - detector: LanguageDetectorBuilder::from_all_languages().build(), - scores: Mutex::new(BoundedCache::new()), - spans: Mutex::new(BoundedCache::new()), - } - } -} +static DETECTOR: LazyLock = + LazyLock::new(|| LanguageDetectorBuilder::from_all_languages().build()); static WORD: LazyLock = LazyLock::new(|| Regex::new(r"[\p{L}\p{M}\p{N}]+").unwrap()); @@ -135,63 +75,31 @@ impl Detection { } } -fn detect(text: &str, detector: &DetectorState) -> Detection { - if let Some(value) = detector - .scores - .lock() - .unwrap_or_else(|error| error.into_inner()) - .get(text) - { - return value; - } - let scores = detector.detector.compute_language_confidence_values(text); +fn detect(text: &str, detector: &LanguageDetector) -> Detection { + let scores = detector.compute_language_confidence_values(text); let english_confidence = scores .iter() .find(|(language, _)| *language == Language::English) .map_or(0.0, |(_, confidence)| *confidence); let best = scores.first().filter(|(_, confidence)| *confidence > 0.0); - let detection = Detection { + Detection { runner_up_confidence: scores.get(1).map_or(0.0, |(_, score)| *score), language: best.map(|(language, _)| *language), confidence: best.map_or(0.0, |(_, confidence)| *confidence), english_confidence, - }; - detector - .scores - .lock() - .unwrap_or_else(|error| error.into_inner()) - .insert(text, detection, std::mem::size_of::()); - detection + } } /// Use Lingua's inferred boundaries, then rescore each span independently so /// English can qualify even when another language has the highest score. fn mixed_language_passages( text: &str, - detector: &DetectorState, + detector: &LanguageDetector, ) -> Vec { - let cached = detector - .spans - .lock() - .unwrap_or_else(|error| error.into_inner()) - .get(text); - let spans = cached.unwrap_or_else(|| { - let spans: Arc<[(usize, usize)]> = detector - .detector - .detect_multiple_languages_of(text) - .into_iter() - .map(|span| (span.start_index(), span.end_index())) - .collect(); - detector - .spans - .lock() - .unwrap_or_else(|error| error.into_inner()) - .insert(text, spans.clone(), std::mem::size_of_val(spans.as_ref())); - spans - }); - spans - .iter() - .map(|(start, end)| text[*start..*end].trim().to_owned()) + detector + .detect_multiple_languages_of(text) + .into_iter() + .map(|span| text[span.start_index()..span.end_index()].trim().to_owned()) .collect() } @@ -360,7 +268,7 @@ fn classify_passage( text: String, eligible: bool, minimum_ratio: f64, - detector: &DetectorState, + detector: &LanguageDetector, should_detect: bool, ) -> Passage { let detection = if should_detect { From 1b532fda7aa01d7b4fd8231f9d95b27aa8b739f5 Mon Sep 17 00:00:00 2001 From: tdgao Date: Mon, 14 Sep 2026 12:22:45 -0600 Subject: [PATCH 11/17] feat: tune description block extraction --- .../src/validate/project/description.rs | 27 +- .../labrinth/src/validate/project/language.rs | 641 ++++++++++-------- apps/labrinth/src/validate/project/summary.rs | 15 +- apps/labrinth/src/validate/project/text.rs | 109 +-- 4 files changed, 433 insertions(+), 359 deletions(-) diff --git a/apps/labrinth/src/validate/project/description.rs b/apps/labrinth/src/validate/project/description.rs index 269aea48d2..6d76de6062 100644 --- a/apps/labrinth/src/validate/project/description.rs +++ b/apps/labrinth/src/validate/project/description.rs @@ -7,14 +7,16 @@ use super::text::{ ProfanityKind, contains_description_spam, extract_description_blocks, extract_description_text, find_banned_description_link, has_image_without_alt_text, has_sufficient_english_blocks, - js_string_length, non_standard_text_ratio, normalize_project_field_text, - profanity_matches, project_requires_english, project_text_similarity, + is_confidently_non_english_short_text, js_string_length, + non_standard_text_ratio, normalize_project_field_text, profanity_matches, + project_requires_english, project_text_similarity, }; use super::{ProjectNag, ProjectNagKind, ProjectNagSeverity}; use crate::models::projects::Project; const MIN_DESCRIPTION_CHARS: usize = 125; +const MIN_DESCRIPTION_LANGUAGE_CHARS: usize = 35; const MAX_DESCRIPTION_SUMMARY_SIMILARITY: f64 = 0.8; const MAX_PROFANITY_COUNT: usize = 2; const NON_STANDARD_TEXT_FAILURE_THRESHOLD: f64 = 0.05; @@ -92,14 +94,14 @@ pub(super) fn validate(project: &Project) -> Vec { ); } } - if project_text_similarity(&normalized_text, &project.summary) - >= MAX_DESCRIPTION_SUMMARY_SIMILARITY - { - nags.push(ProjectNag::new( - ProjectNagKind::ProjectDescriptionMatchesSummary, - ProjectNagSeverity::Required, - )); - } + if project_text_similarity(&normalized_text, &project.summary) + >= MAX_DESCRIPTION_SUMMARY_SIMILARITY + { + nags.push(ProjectNag::new( + ProjectNagKind::ProjectDescriptionMatchesSummary, + ProjectNagSeverity::Required, + )); + } if has_spam { nags.push(ProjectNag::new( ProjectNagKind::ProjectDescriptionSpam, @@ -160,8 +162,11 @@ fn is_non_english_text( text: &str, blocks: &[String], ) -> bool { + let length = js_string_length(text); project_requires_english(project) - && js_string_length(text) >= MIN_DESCRIPTION_CHARS + && length >= MIN_DESCRIPTION_LANGUAGE_CHARS + && (length >= MIN_DESCRIPTION_CHARS + || is_confidently_non_english_short_text(&blocks.join(" "))) && !has_sufficient_english_blocks(blocks) } diff --git a/apps/labrinth/src/validate/project/language.rs b/apps/labrinth/src/validate/project/language.rs index 6b285941db..3d6daef67b 100644 --- a/apps/labrinth/src/validate/project/language.rs +++ b/apps/labrinth/src/validate/project/language.rs @@ -9,387 +9,428 @@ const MIN_ENGLISH_TO_BEST_RATIO: f64 = 0.5; const MIN_DESCRIPTION_CONFIDENCE: f64 = 0.35; const MIN_DESCRIPTION_MARGIN: f64 = 0.15; const MIN_NON_ENGLISH_CONFIDENCE: f64 = 0.8; +const MIN_DOMINANT_NON_ENGLISH_CONFIDENCE: f64 = 0.6; +const MIN_DOMINANT_NON_ENGLISH_MARGIN: f64 = 0.6; +const MAX_DOMINANT_ENGLISH_CONFIDENCE: f64 = 0.05; const MIN_NON_LATIN_CONFIDENCE: f64 = 0.5; +const MIN_SHORT_TEXT_CONFIDENCE: f64 = 0.95; +const MIN_SHORT_TEXT_MARGIN: f64 = 0.5; const MIN_DESCRIPTION_ENGLISH_PROPORTION: f64 = 0.2; const MIN_PASSAGE_WORDS: usize = 4; const MIN_PASSAGE_CHARS: usize = 25; static DETECTOR: LazyLock = - LazyLock::new(|| LanguageDetectorBuilder::from_all_languages().build()); + LazyLock::new(|| LanguageDetectorBuilder::from_all_languages().build()); static WORD: LazyLock = - LazyLock::new(|| Regex::new(r"[\p{L}\p{M}\p{N}]+").unwrap()); + LazyLock::new(|| Regex::new(r"[\p{L}\p{M}\p{N}]+").unwrap()); static NON_LATIN_LETTER: LazyLock = - LazyLock::new(|| Regex::new(r"[\p{Alphabetic}&&[^\p{Latin}]]").unwrap()); + LazyLock::new(|| Regex::new(r"[\p{Alphabetic}&&[^\p{Latin}]]").unwrap()); const MIN_DESCRIPTION_WORDS: usize = 8; const MIN_DESCRIPTION_CHARS: usize = 35; const MIN_ENGLISH_GRAMMATICAL_WORDS: usize = 2; struct Passage { - text: String, - language: Option, - confidence: f64, - runner_up_confidence: f64, - eligible: bool, - qualifies_as_english: bool, - confidently_non_english: bool, - alphabetic_words: usize, - english_grammatical_words: usize, + text: String, + language: Option, + confidence: f64, + runner_up_confidence: f64, + eligible: bool, + qualifies_as_english: bool, + confidently_non_english: bool, + alphabetic_words: usize, + english_grammatical_words: usize, } #[derive(Clone, Copy, Default)] struct Detection { - runner_up_confidence: f64, - language: Option, - confidence: f64, - english_confidence: f64, + runner_up_confidence: f64, + language: Option, + confidence: f64, + english_confidence: f64, } impl Detection { - fn is_english(&self) -> bool { - self.english_to_best_ratio() >= MIN_ENGLISH_TO_BEST_RATIO - } + fn is_english(&self) -> bool { + self.english_to_best_ratio() >= MIN_ENGLISH_TO_BEST_RATIO + } - fn is_confidently_non_english(&self, text: &str) -> bool { - let has_non_latin_evidence = - NON_LATIN_LETTER.find_iter(text).count() >= 5; - let minimum_confidence = if has_non_latin_evidence { - MIN_NON_LATIN_CONFIDENCE - } else { - MIN_NON_ENGLISH_CONFIDENCE - }; - self.confidence >= minimum_confidence - && !self.is_english() - && (has_non_latin_evidence - || (text.graphemes(true).count() >= MIN_PASSAGE_CHARS - && alphabetic_word_count(text) >= MIN_PASSAGE_WORDS)) - } + fn is_confidently_non_english(&self, text: &str) -> bool { + let has_non_latin_evidence = + NON_LATIN_LETTER.find_iter(text).count() >= 5; + let minimum_confidence = if has_non_latin_evidence { + MIN_NON_LATIN_CONFIDENCE + } else { + MIN_NON_ENGLISH_CONFIDENCE + }; + let clearly_dominant = self.confidence + >= MIN_DOMINANT_NON_ENGLISH_CONFIDENCE + && self.confidence - self.runner_up_confidence + >= MIN_DOMINANT_NON_ENGLISH_MARGIN + && self.english_confidence < MAX_DOMINANT_ENGLISH_CONFIDENCE; + (self.confidence >= minimum_confidence || clearly_dominant) + && !self.is_english() + && (has_non_latin_evidence + || (text.graphemes(true).count() >= MIN_PASSAGE_CHARS + && alphabetic_word_count(text) >= MIN_PASSAGE_WORDS)) + } - fn english_to_best_ratio(&self) -> f64 { - if self.confidence > 0.0 { - self.english_confidence / self.confidence - } else { - 0.0 - } - } + fn english_to_best_ratio(&self) -> f64 { + if self.confidence > 0.0 { + self.english_confidence / self.confidence + } else { + 0.0 + } + } } fn detect(text: &str, detector: &LanguageDetector) -> Detection { - let scores = detector.compute_language_confidence_values(text); - let english_confidence = scores - .iter() - .find(|(language, _)| *language == Language::English) - .map_or(0.0, |(_, confidence)| *confidence); - let best = scores.first().filter(|(_, confidence)| *confidence > 0.0); - Detection { - runner_up_confidence: scores.get(1).map_or(0.0, |(_, score)| *score), - language: best.map(|(language, _)| *language), - confidence: best.map_or(0.0, |(_, confidence)| *confidence), - english_confidence, - } + let scores = detector.compute_language_confidence_values(text); + let english_confidence = scores + .iter() + .find(|(language, _)| *language == Language::English) + .map_or(0.0, |(_, confidence)| *confidence); + let best = scores.first().filter(|(_, confidence)| *confidence > 0.0); + Detection { + runner_up_confidence: scores.get(1).map_or(0.0, |(_, score)| *score), + language: best.map(|(language, _)| *language), + confidence: best.map_or(0.0, |(_, confidence)| *confidence), + english_confidence, + } +} + +/// Short fields need stronger evidence before bypassing the usual length gate. +pub(super) fn is_confidently_non_english_short_text(text: &str) -> bool { + let normalized: String = text.nfkc().collect(); + let detection = detect(&normalized, &DETECTOR); + detection.confidence >= MIN_SHORT_TEXT_CONFIDENCE + && detection.confidence - detection.runner_up_confidence + >= MIN_SHORT_TEXT_MARGIN + && detection.is_confidently_non_english(&normalized) } /// Use Lingua's inferred boundaries, then rescore each span independently so /// English can qualify even when another language has the highest score. fn mixed_language_passages( - text: &str, - detector: &LanguageDetector, + text: &str, + detector: &LanguageDetector, ) -> Vec { - detector - .detect_multiple_languages_of(text) - .into_iter() - .map(|span| text[span.start_index()..span.end_index()].trim().to_owned()) - .collect() + detector + .detect_multiple_languages_of(text) + .into_iter() + .map(|span| { + text[span.start_index()..span.end_index()].trim().to_owned() + }) + .collect() } fn has_enough_description_content(text: &str) -> bool { - alphabetic_word_count(text) >= MIN_DESCRIPTION_WORDS - && text.trim().graphemes(true).count() >= MIN_DESCRIPTION_CHARS + alphabetic_word_count(text) >= MIN_DESCRIPTION_WORDS + && text.trim().graphemes(true).count() >= MIN_DESCRIPTION_CHARS } /// Keep script runs disjoint so surrounding foreign text cannot qualify a short English fragment. fn script_passages(text: &str) -> Vec<&str> { - let mut passages = Vec::new(); - let mut start = 0; - let mut previous = None; - for (index, character) in text.char_indices() { - if !character.is_alphabetic() { - continue; - } - let mut buffer = [0; 4]; - let non_latin = - NON_LATIN_LETTER.is_match(character.encode_utf8(&mut buffer)); - if previous.is_some_and(|previous| previous != non_latin) { - passages.push(text[start..index].trim()); - start = index; - } - previous = Some(non_latin); - } - if !text[start..].trim().is_empty() { - passages.push(text[start..].trim()); - } - passages + let mut passages = Vec::new(); + let mut start = 0; + let mut previous = None; + for (index, character) in text.char_indices() { + if !character.is_alphabetic() { + continue; + } + let mut buffer = [0; 4]; + let non_latin = + NON_LATIN_LETTER.is_match(character.encode_utf8(&mut buffer)); + if previous.is_some_and(|previous| previous != non_latin) { + passages.push(text[start..index].trim()); + start = index; + } + previous = Some(non_latin); + } + if !text[start..].trim().is_empty() { + passages.push(text[start..].trim()); + } + passages } fn description_passage(text: &str) -> Passage { - let mut passage = classify_passage( - text.to_owned(), - has_enough_description_content(text), - 1.0, - &DETECTOR, - has_enough_description_content(text) - || NON_LATIN_LETTER.find_iter(text).count() >= 5, - ); - let strong = passage.confidence >= MIN_DESCRIPTION_CONFIDENCE - && passage.confidence - passage.runner_up_confidence - >= MIN_DESCRIPTION_MARGIN; - let foreign_eligible = - passage.eligible || NON_LATIN_LETTER.find_iter(text).count() >= 5; - passage.qualifies_as_english = passage.eligible - && strong - && (passage.english_grammatical_words >= MIN_ENGLISH_GRAMMATICAL_WORDS - || has_lowercase_prose(text)) - && passage.language == Some(Language::English); - passage.confidently_non_english = foreign_eligible - && strong - && passage - .language - .is_some_and(|language| language != Language::English); - passage.eligible = foreign_eligible; - passage + let mut passage = classify_passage( + text.to_owned(), + has_enough_description_content(text), + 1.0, + &DETECTOR, + has_enough_description_content(text) + || NON_LATIN_LETTER.find_iter(text).count() >= 5, + ); + let strong = passage.confidence >= MIN_DESCRIPTION_CONFIDENCE + && passage.confidence - passage.runner_up_confidence + >= MIN_DESCRIPTION_MARGIN; + let foreign_eligible = + passage.eligible || NON_LATIN_LETTER.find_iter(text).count() >= 5; + passage.qualifies_as_english = passage.eligible + && strong + && (passage.english_grammatical_words >= MIN_ENGLISH_GRAMMATICAL_WORDS + || has_lowercase_prose(text)) + && passage.language == Some(Language::English); + passage.confidently_non_english = foreign_eligible + && strong + && passage + .language + .is_some_and(|language| language != Language::English); + passage.eligible = foreign_eligible; + passage } /// Capitalized names and acronyms alone do not establish English prose. fn has_lowercase_prose(text: &str) -> bool { - WORD.find_iter(text) - .filter(|word| { - let word = word.as_str(); - word.chars().all(char::is_lowercase) && word.chars().count() > 1 - }) - .take(2) - .count() - >= 2 + WORD.find_iter(text) + .filter(|word| { + let word = word.as_str(); + word.chars().all(char::is_lowercase) && word.chars().count() > 1 + }) + .take(2) + .count() + >= 2 } fn summary_translation_passage(text: &str) -> Passage { - let mut passage = description_passage(text); - if passage.qualifies_as_english - && passage.english_grammatical_words < MIN_ENGLISH_GRAMMATICAL_WORDS - { - passage.qualifies_as_english = false; - } - passage + let mut passage = description_passage(text); + if passage.qualifies_as_english + && passage.english_grammatical_words < MIN_ENGLISH_GRAMMATICAL_WORDS + { + passage.qualifies_as_english = false; + } + passage } /// Bound detector input without overlapping or splitting words. fn bounded_passages(text: &str) -> Vec<&str> { - let mut passages = Vec::new(); - let mut start = 0; - for (index, character) in text.char_indices() { - if index - start >= 4000 && character.is_whitespace() { - passages.push(text[start..index].trim()); - start = index + character.len_utf8(); - } - } - if !text[start..].trim().is_empty() { - passages.push(text[start..].trim()); - } - passages + let mut passages = Vec::new(); + let mut start = 0; + for (index, character) in text.char_indices() { + if index - start >= 4000 && character.is_whitespace() { + passages.push(text[start..index].trim()); + start = index + character.len_utf8(); + } + } + if !text[start..].trim().is_empty() { + passages.push(text[start..].trim()); + } + passages } /// Only replace a coherent block when Lingua finds independently supported English and foreign portions. fn description_passages(text: &str) -> Vec { - bounded_passages(text) - .into_iter() - .flat_map(script_passages) - .flat_map(|text| { - let whole = description_passage(text); - if whole.alphabetic_words < MIN_DESCRIPTION_WORDS * 2 { - return vec![whole]; - } - let spans = mixed_language_passages(text, &DETECTOR); - if spans.len() <= 1 { - return vec![whole]; - } - let candidates = spans - .iter() - .map(|span| description_passage(span)) - .collect::>(); - if candidates.iter().any(|span| span.qualifies_as_english) - && candidates.iter().any(|span| span.confidently_non_english) - { - candidates - } else { - vec![whole] - } - }) - .collect() + bounded_passages(text) + .into_iter() + .flat_map(script_passages) + .flat_map(|text| { + let whole = description_passage(text); + if whole.alphabetic_words < MIN_DESCRIPTION_WORDS * 2 { + return vec![whole]; + } + let spans = mixed_language_passages(text, &DETECTOR); + if spans.len() <= 1 { + return vec![whole]; + } + let candidates = spans + .iter() + .map(|span| description_passage(span)) + .collect::>(); + if candidates.iter().any(|span| span.qualifies_as_english) + && candidates.iter().any(|span| span.confidently_non_english) + { + candidates + } else { + vec![whole] + } + }) + .collect() } fn alphabetic_word_count(text: &str) -> usize { - WORD.find_iter(text) - .filter(|word| word.as_str().chars().any(char::is_alphabetic)) - .count() + WORD.find_iter(text) + .filter(|word| word.as_str().chars().any(char::is_alphabetic)) + .count() } /// Require grammatical context before treating technical names as an English translation. fn english_grammatical_word_count(text: &str) -> usize { - WORD.find_iter(text) - .filter(|word| { - matches!( - word.as_str().to_lowercase().as_str(), - "a" | "an" - | "the" | "this" - | "that" | "these" - | "those" | "it" - | "its" | "you" | "your" - | "we" | "our" | "they" - | "their" | "to" - | "for" | "with" - | "from" | "in" | "on" - | "of" | "and" | "or" - | "by" | "into" | "is" - | "are" | "be" | "can" - | "will" | "has" - | "have" | "does" - | "not" - ) - }) - .count() + WORD.find_iter(text) + .filter(|word| { + matches!( + word.as_str().to_lowercase().as_str(), + "a" | "an" + | "the" + | "this" + | "that" + | "these" + | "those" + | "it" + | "its" + | "you" + | "your" + | "we" + | "our" + | "they" + | "their" + | "to" + | "for" + | "with" + | "from" + | "in" + | "on" + | "of" + | "and" + | "or" + | "by" + | "into" + | "is" + | "are" + | "be" + | "can" + | "will" + | "has" + | "have" + | "does" + | "not" + ) + }) + .count() } fn is_eligible_passage(text: &str) -> bool { - alphabetic_word_count(text) >= MIN_PASSAGE_WORDS - && text.graphemes(true).count() >= MIN_PASSAGE_CHARS + alphabetic_word_count(text) >= MIN_PASSAGE_WORDS + && text.graphemes(true).count() >= MIN_PASSAGE_CHARS } fn classify_passage( - text: String, - eligible: bool, - minimum_ratio: f64, - detector: &LanguageDetector, - should_detect: bool, + text: String, + eligible: bool, + minimum_ratio: f64, + detector: &LanguageDetector, + should_detect: bool, ) -> Passage { - let detection = if should_detect { - detect(&text, detector) - } else { - Detection::default() - }; - Passage { - runner_up_confidence: detection.runner_up_confidence, - qualifies_as_english: eligible - && detection.english_to_best_ratio() >= minimum_ratio, - confidently_non_english: eligible - && detection.is_confidently_non_english(&text), - alphabetic_words: alphabetic_word_count(&text), - english_grammatical_words: english_grammatical_word_count(&text), - text, - language: detection.language, - confidence: detection.confidence, - eligible, - } + let detection = if should_detect { + detect(&text, detector) + } else { + Detection::default() + }; + Passage { + runner_up_confidence: detection.runner_up_confidence, + qualifies_as_english: eligible + && detection.english_to_best_ratio() >= minimum_ratio, + confidently_non_english: eligible + && detection.is_confidently_non_english(&text), + alphabetic_words: alphabetic_word_count(&text), + english_grammatical_words: english_grammatical_word_count(&text), + text, + language: detection.language, + confidence: detection.confidence, + eligible, + } } /// A foreign summary needs a contiguous English passage, not scattered product names. fn summary_rescue_passages(text: &str) -> Vec { - script_passages(text) - .into_iter() - .filter(|text| !NON_LATIN_LETTER.is_match(text)) - .flat_map(|text| { - let normalized = - text.split_whitespace().collect::>().join(" "); - let whole = summary_translation_passage(&normalized); - if whole.qualifies_as_english - || !whole.eligible - || whole.english_grammatical_words - < MIN_ENGLISH_GRAMMATICAL_WORDS - { - return vec![whole]; - } - let spans = mixed_language_passages(&normalized, &DETECTOR); - if spans.len() <= 1 { - return vec![whole]; - } - spans - .into_iter() - .map(|span| summary_translation_passage(&span)) - .collect() - }) - .collect() + script_passages(text) + .into_iter() + .filter(|text| !NON_LATIN_LETTER.is_match(text)) + .flat_map(|text| { + let normalized = + text.split_whitespace().collect::>().join(" "); + let whole = summary_translation_passage(&normalized); + if whole.qualifies_as_english + || !whole.eligible + || whole.english_grammatical_words + < MIN_ENGLISH_GRAMMATICAL_WORDS + { + return vec![whole]; + } + let spans = mixed_language_passages(&normalized, &DETECTOR); + if spans.len() <= 1 { + return vec![whole]; + } + spans + .into_iter() + .map(|span| summary_translation_passage(&span)) + .collect() + }) + .collect() } pub(super) fn is_likely_english_summary(text: &str) -> bool { - let normalized: String = text.nfkc().collect(); - let mut passages = vec![classify_passage( - normalized.clone(), - true, - MIN_ENGLISH_TO_BEST_RATIO, - &DETECTOR, - true, - )]; - let requires_strong_rescue = passages[0].confidently_non_english; - if requires_strong_rescue { - passages.extend(summary_rescue_passages(&normalized)); - } else if NON_LATIN_LETTER.is_match(&normalized) { - let latin = NON_LATIN_LETTER.replace_all(&normalized, " "); - passages[0].qualifies_as_english = false; - passages.push(classify_passage( - latin.to_string(), - is_eligible_passage(&latin), - MIN_ENGLISH_TO_BEST_RATIO, - &DETECTOR, - is_eligible_passage(&latin), - )); - } else if !passages[0].qualifies_as_english { - passages.extend( - mixed_language_passages(&normalized, &DETECTOR) - .into_iter() - .map(|text| { - let eligible = is_eligible_passage(&text); - classify_passage( - text, - eligible, - MIN_ENGLISH_TO_BEST_RATIO, - &DETECTOR, - eligible, - ) - }), - ); - } - passages.iter().any(|passage| passage.qualifies_as_english) - || !passages - .iter() - .any(|passage| passage.confidently_non_english) + let normalized: String = text.nfkc().collect(); + let mut passages = vec![classify_passage( + normalized.clone(), + true, + MIN_ENGLISH_TO_BEST_RATIO, + &DETECTOR, + true, + )]; + let requires_strong_rescue = passages[0].confidently_non_english; + if requires_strong_rescue { + passages.extend(summary_rescue_passages(&normalized)); + } else if NON_LATIN_LETTER.is_match(&normalized) { + let latin = NON_LATIN_LETTER.replace_all(&normalized, " "); + passages[0].qualifies_as_english = false; + passages.push(classify_passage( + latin.to_string(), + is_eligible_passage(&latin), + MIN_ENGLISH_TO_BEST_RATIO, + &DETECTOR, + is_eligible_passage(&latin), + )); + } else if !passages[0].qualifies_as_english { + passages.extend( + mixed_language_passages(&normalized, &DETECTOR) + .into_iter() + .map(|text| { + let eligible = is_eligible_passage(&text); + classify_passage( + text, + eligible, + MIN_ENGLISH_TO_BEST_RATIO, + &DETECTOR, + eligible, + ) + }), + ); + } + passages.iter().any(|passage| passage.qualifies_as_english) + || !passages + .iter() + .any(|passage| passage.confidently_non_english) } pub(super) fn has_sufficient_english_blocks(blocks: &[String]) -> bool { - let passages = blocks - .iter() - .flat_map(|block| { - let normalized: String = block.nfkc().collect(); - description_passages(&normalized) - }) - .collect::>(); - let english_utf16_length: usize = passages - .iter() - .filter(|passage| passage.qualifies_as_english) - .map(|passage| passage.text.encode_utf16().count()) - .sum(); - let total_utf16_length: usize = passages - .iter() - .filter(|passage| passage.eligible) - .map(|passage| passage.text.encode_utf16().count()) - .sum(); - let has_foreign_evidence = passages - .iter() - .any(|passage| passage.confidently_non_english); - let english_proportion = if total_utf16_length == 0 { - 0.0 - } else { - english_utf16_length as f64 / total_utf16_length as f64 - }; + let passages = blocks + .iter() + .flat_map(|block| { + let normalized: String = block.nfkc().collect(); + description_passages(&normalized) + }) + .collect::>(); + let english_utf16_length: usize = passages + .iter() + .filter(|passage| passage.qualifies_as_english) + .map(|passage| passage.text.encode_utf16().count()) + .sum(); + let total_utf16_length: usize = passages + .iter() + .filter(|passage| passage.eligible) + .map(|passage| passage.text.encode_utf16().count()) + .sum(); + let has_foreign_evidence = passages + .iter() + .any(|passage| passage.confidently_non_english); + let english_proportion = if total_utf16_length == 0 { + 0.0 + } else { + english_utf16_length as f64 / total_utf16_length as f64 + }; - !has_foreign_evidence - || english_proportion >= MIN_DESCRIPTION_ENGLISH_PROPORTION + !has_foreign_evidence + || english_proportion >= MIN_DESCRIPTION_ENGLISH_PROPORTION } diff --git a/apps/labrinth/src/validate/project/summary.rs b/apps/labrinth/src/validate/project/summary.rs index 23f14ea2ec..89b006b869 100644 --- a/apps/labrinth/src/validate/project/summary.rs +++ b/apps/labrinth/src/validate/project/summary.rs @@ -1,8 +1,9 @@ use serde_json::json; use super::text::{ - ProfanityKind, contains_spam, find_link_or_ip, has_non_standard_text, - has_summary_formatting, is_likely_english_summary, js_string_length, + ProfanityKind, contains_spam, find_link_or_ip, has_non_standard_text, + has_summary_formatting, is_confidently_non_english_short_text, + is_likely_english_summary, js_string_length, normalize_project_field_text, profanity_matches, project_requires_english, project_text_similarity, }; @@ -11,6 +12,7 @@ use super::{ProjectNag, ProjectNagKind, ProjectNagSeverity}; use crate::models::projects::Project; const MIN_SUMMARY_CHARS: usize = 25; +const MIN_SUMMARY_LANGUAGE_CHARS: usize = 10; const MAX_SUMMARY_NAME_SIMILARITY: f64 = 0.8; pub(super) fn validate(project: &Project) -> Vec { @@ -121,9 +123,12 @@ pub(super) fn is_non_english(project: &Project) -> bool { } fn is_non_english_text(project: &Project, normalized_summary: &str) -> bool { - project_requires_english(project) - && js_string_length(normalized_summary) >= MIN_SUMMARY_CHARS - && !is_likely_english_summary(normalized_summary) + let length = js_string_length(normalized_summary); + project_requires_english(project) + && length >= MIN_SUMMARY_LANGUAGE_CHARS + && (length >= MIN_SUMMARY_CHARS + || is_confidently_non_english_short_text(normalized_summary)) + && !is_likely_english_summary(normalized_summary) } fn requires_language_nag( diff --git a/apps/labrinth/src/validate/project/text.rs b/apps/labrinth/src/validate/project/text.rs index 479f0dff9f..6f72b14f0a 100644 --- a/apps/labrinth/src/validate/project/text.rs +++ b/apps/labrinth/src/validate/project/text.rs @@ -12,7 +12,8 @@ use crate::models::exp::minecraft::Language; use crate::models::projects::Project; pub(super) use super::language::{ - has_sufficient_english_blocks, is_likely_english_summary, + has_sufficient_english_blocks, is_confidently_non_english_short_text, + is_likely_english_summary, }; static WORD: LazyLock = @@ -982,52 +983,74 @@ fn strip_technical_syntax(line: &str) -> String { } } +fn strip_loaders(text: &str) -> String { + static LOADER_WORD: LazyLock = LazyLock::new(|| { + Regex::new(r"(?i)\b(?:fabric|forge|quilt|neoforge|liteloader)\b").unwrap() + }); + + LOADER_WORD.replace_all(text, " ").into_owned() +} + fn description_language_input(markdown: &str) -> String { - let readable = strip_description_markup(markdown); - let readable = text_without_explicit_links(&readable); - readable - .lines() - .map(strip_technical_syntax) - .collect::>() - .join("\n") + let readable = strip_description_markup(markdown); + let readable = text_without_explicit_links(&readable); + let readable = strip_loaders(&readable); + readable + .lines() + .map(strip_technical_syntax) + .collect::>() + .join("\n") } pub(super) fn extract_description_blocks(markdown: &str) -> Vec { - let readable = description_language_input(markdown); - let mut blocks = Vec::new(); - let mut paragraph = Vec::new(); - let mut heading_only = false; - let mut blank = false; - for line in readable.lines() { - let line = line.trim(); - if line.is_empty() { - blank = true; - continue; - } - let heading = line.starts_with('#'); - let metadata_list = line - .split_once(':') - .is_some_and(|(_, values)| values.matches(',').count() >= 3); - if (heading - || metadata_list - || (blank - && !heading_only - && (WORD.find_iter(¶graph.join(" ")).count() >= 8 - || WORD.find_iter(line).count() >= 8))) - && !paragraph.is_empty() - { - blocks.push(extract_description_text(¶graph.join("\n"))); - paragraph.clear(); - } - paragraph.push(line); - heading_only = heading; - blank = false; - } - if !paragraph.is_empty() { - blocks.push(extract_description_text(¶graph.join("\n"))); - } - blocks.retain(|block| !block.is_empty()); - blocks + const MIN_PARAGRAPH_WORDS: usize = 8; + + let readable = description_language_input(markdown); + let mut blocks = Vec::new(); + let mut paragraph = Vec::new(); + let mut previous_line_was_heading = false; + let mut follows_blank_line = false; + + for line in readable.lines() { + let line = line.trim(); + if line.is_empty() { + follows_blank_line = true; + continue; + } + + let is_heading = line.starts_with('#'); + let starts_new_paragraph = follows_blank_line + && !previous_line_was_heading + && (has_minimum_word_count(¶graph.join(" "), MIN_PARAGRAPH_WORDS) + || has_minimum_word_count(line, MIN_PARAGRAPH_WORDS)); + + if is_heading || starts_new_paragraph { + push_description_block(&mut blocks, &mut paragraph); + } + + paragraph.push(line); + previous_line_was_heading = is_heading; + follows_blank_line = false; + } + + push_description_block(&mut blocks, &mut paragraph); + blocks +} + +fn has_minimum_word_count(text: &str, minimum: usize) -> bool { + WORD.find_iter(text).take(minimum).count() == minimum +} + +fn push_description_block(blocks: &mut Vec, paragraph: &mut Vec<&str>) { + if paragraph.is_empty() { + return; + } + + let block = extract_description_text(¶graph.join("\n")); + if !block.is_empty() { + blocks.push(block); + } + paragraph.clear(); } pub(super) fn has_image_without_alt_text(markdown: &str) -> bool { From f7ab387c273cdc46238eb0fb436d9679c124a640 Mon Sep 17 00:00:00 2001 From: tdgao Date: Mon, 14 Sep 2026 12:27:19 -0600 Subject: [PATCH 12/17] remove outdated test --- .../validate/project/description/markdown.rs | 78 ++++++++++--------- 1 file changed, 42 insertions(+), 36 deletions(-) diff --git a/apps/labrinth/src/validate/project/description/markdown.rs b/apps/labrinth/src/validate/project/description/markdown.rs index 5b6ccb21c2..dc6a9a3375 100644 --- a/apps/labrinth/src/validate/project/description/markdown.rs +++ b/apps/labrinth/src/validate/project/description/markdown.rs @@ -6,10 +6,10 @@ use regex::Regex; use unicode_segmentation::UnicodeSegmentation; static HTML_HEADER: LazyLock = LazyLock::new(|| { - Regex::new(r"(?is)]*>(.*?)").unwrap() + Regex::new(r"(?is)]*>(.*?)").unwrap() }); static ADJACENT_HTML_HEADERS: LazyLock = - LazyLock::new(|| Regex::new(r"(?is)\s*\s* = LazyLock::new(|| { Regex::new(r"(?is)\s*(?:]*>\s*)*$").unwrap() }); @@ -129,7 +129,7 @@ impl<'a> DescriptionMarkdown<'a> { let [previous, current] = headings else { return false; }; - previous.level == current.level + previous.level == current.level && self.markdown[previous.range.end..current.range.start] .trim() .is_empty() @@ -242,37 +242,43 @@ image: "![](/missing-alt.png)" ); } - #[test] - fn all_heading_levels_are_validated() { - for level in 1..=6 { - let prefix = "#".repeat(level); - let long_text = "heading ".repeat(12); - for (long_header, adjacent_headers, separated_headers) in [ - ( - format!("{prefix} {long_text}"), - format!("{prefix} First\n\n{prefix} Second"), - format!("{prefix} First\n\n```yaml\n# comment\n```\n\n{prefix} Second"), - ), - ( - format!("{long_text}"), - format!("First\n\nSecond"), - format!("First\n\n```yaml\n# comment\n```\n\nSecond"), - ), - ] { - let markdown = DescriptionMarkdown::parse(&long_header); - assert_eq!(markdown.long_header_count(), 1, "{long_header}"); - assert!(markdown.ends_with_header(), "{long_header}"); - assert!( - DescriptionMarkdown::parse(&adjacent_headers) - .has_adjacent_same_level_headers(), - "{adjacent_headers}" - ); - assert!( - !DescriptionMarkdown::parse(&separated_headers) - .has_adjacent_same_level_headers(), - "{separated_headers}" - ); - } - } - } + #[test] + fn all_heading_levels_are_validated() { + for level in 1..=6 { + let prefix = "#".repeat(level); + let long_text = "heading ".repeat(12); + for (long_header, adjacent_headers, separated_headers) in [ + ( + format!("{prefix} {long_text}"), + format!("{prefix} First\n\n{prefix} Second"), + format!( + "{prefix} First\n\n```yaml\n# comment\n```\n\n{prefix} Second" + ), + ), + ( + format!("{long_text}"), + format!( + "First\n\nSecond" + ), + format!( + "First\n\n```yaml\n# comment\n```\n\nSecond" + ), + ), + ] { + let markdown = DescriptionMarkdown::parse(&long_header); + assert_eq!(markdown.long_header_count(), 1, "{long_header}"); + assert!(markdown.ends_with_header(), "{long_header}"); + assert!( + DescriptionMarkdown::parse(&adjacent_headers) + .has_adjacent_same_level_headers(), + "{adjacent_headers}" + ); + assert!( + !DescriptionMarkdown::parse(&separated_headers) + .has_adjacent_same_level_headers(), + "{separated_headers}" + ); + } + } + } } From ca1eafe2870909a8f80c27527b87f57369d7c533 Mon Sep 17 00:00:00 2001 From: tdgao Date: Mon, 14 Sep 2026 13:16:42 -0600 Subject: [PATCH 13/17] fmt --- apps/labrinth/src/validate/project/mod.rs | 2 +- apps/labrinth/src/validate/project/summary.rs | 21 ++- apps/labrinth/src/validate/project/text.rs | 129 +++++++++--------- apps/labrinth/tests/project.rs | 2 +- 4 files changed, 78 insertions(+), 76 deletions(-) diff --git a/apps/labrinth/src/validate/project/mod.rs b/apps/labrinth/src/validate/project/mod.rs index a9f94f6242..6151a6c8eb 100644 --- a/apps/labrinth/src/validate/project/mod.rs +++ b/apps/labrinth/src/validate/project/mod.rs @@ -69,7 +69,7 @@ pub enum ProjectNagKind { ProjectDescriptionProfanity, ProjectDescriptionNonStandardText, ProjectDescriptionNonEnglish, - ProjectDescriptionMatchesSummary, + ProjectDescriptionMatchesSummary, AddDescription, DescriptionTooShort, ProjectDescriptionSpam, diff --git a/apps/labrinth/src/validate/project/summary.rs b/apps/labrinth/src/validate/project/summary.rs index 89b006b869..5037fbbcc6 100644 --- a/apps/labrinth/src/validate/project/summary.rs +++ b/apps/labrinth/src/validate/project/summary.rs @@ -1,11 +1,10 @@ use serde_json::json; use super::text::{ - ProfanityKind, contains_spam, find_link_or_ip, has_non_standard_text, - has_summary_formatting, is_confidently_non_english_short_text, - is_likely_english_summary, js_string_length, - normalize_project_field_text, profanity_matches, project_requires_english, - project_text_similarity, + ProfanityKind, contains_spam, find_link_or_ip, has_non_standard_text, + has_summary_formatting, is_confidently_non_english_short_text, + is_likely_english_summary, js_string_length, normalize_project_field_text, + profanity_matches, project_requires_english, project_text_similarity, }; use super::{ProjectNag, ProjectNagKind, ProjectNagSeverity}; @@ -123,12 +122,12 @@ pub(super) fn is_non_english(project: &Project) -> bool { } fn is_non_english_text(project: &Project, normalized_summary: &str) -> bool { - let length = js_string_length(normalized_summary); - project_requires_english(project) - && length >= MIN_SUMMARY_LANGUAGE_CHARS - && (length >= MIN_SUMMARY_CHARS - || is_confidently_non_english_short_text(normalized_summary)) - && !is_likely_english_summary(normalized_summary) + let length = js_string_length(normalized_summary); + project_requires_english(project) + && length >= MIN_SUMMARY_LANGUAGE_CHARS + && (length >= MIN_SUMMARY_CHARS + || is_confidently_non_english_short_text(normalized_summary)) + && !is_likely_english_summary(normalized_summary) } fn requires_language_nag( diff --git a/apps/labrinth/src/validate/project/text.rs b/apps/labrinth/src/validate/project/text.rs index 6f72b14f0a..710c5365d0 100644 --- a/apps/labrinth/src/validate/project/text.rs +++ b/apps/labrinth/src/validate/project/text.rs @@ -12,8 +12,8 @@ use crate::models::exp::minecraft::Language; use crate::models::projects::Project; pub(super) use super::language::{ - has_sufficient_english_blocks, is_confidently_non_english_short_text, - is_likely_english_summary, + has_sufficient_english_blocks, is_confidently_non_english_short_text, + is_likely_english_summary, }; static WORD: LazyLock = @@ -955,7 +955,7 @@ fn strip_dotted_identifiers(text: &str) -> String { .unwrap() }); - IDENTIFIER.replace_all(text, " ").into_owned() + IDENTIFIER.replace_all(text, " ").into_owned() } fn strip_command_arguments(text: &str) -> String { @@ -964,7 +964,7 @@ fn strip_command_arguments(text: &str) -> String { Regex::new(r"\[[A-Za-z][A-Za-z0-9_ /|.-]*\]").unwrap() }); - ARGUMENT.replace_all(text, " ").into_owned() + ARGUMENT.replace_all(text, " ").into_owned() } fn strip_technical_syntax(line: &str) -> String { @@ -976,81 +976,84 @@ fn strip_technical_syntax(line: &str) -> String { let without_commands = COMMAND.replace_all(line, "$1"); let without_identifiers = strip_dotted_identifiers(&without_commands); - if COMMAND.is_match(line) { - strip_command_arguments(&without_identifiers) - } else { - without_identifiers - } + if COMMAND.is_match(line) { + strip_command_arguments(&without_identifiers) + } else { + without_identifiers + } } fn strip_loaders(text: &str) -> String { - static LOADER_WORD: LazyLock = LazyLock::new(|| { - Regex::new(r"(?i)\b(?:fabric|forge|quilt|neoforge|liteloader)\b").unwrap() - }); + static LOADER_WORD: LazyLock = LazyLock::new(|| { + Regex::new(r"(?i)\b(?:fabric|forge|quilt|neoforge|liteloader)\b") + .unwrap() + }); - LOADER_WORD.replace_all(text, " ").into_owned() + LOADER_WORD.replace_all(text, " ").into_owned() } fn description_language_input(markdown: &str) -> String { - let readable = strip_description_markup(markdown); - let readable = text_without_explicit_links(&readable); - let readable = strip_loaders(&readable); - readable - .lines() - .map(strip_technical_syntax) - .collect::>() - .join("\n") + let readable = strip_description_markup(markdown); + let readable = text_without_explicit_links(&readable); + let readable = strip_loaders(&readable); + readable + .lines() + .map(strip_technical_syntax) + .collect::>() + .join("\n") } pub(super) fn extract_description_blocks(markdown: &str) -> Vec { - const MIN_PARAGRAPH_WORDS: usize = 8; - - let readable = description_language_input(markdown); - let mut blocks = Vec::new(); - let mut paragraph = Vec::new(); - let mut previous_line_was_heading = false; - let mut follows_blank_line = false; - - for line in readable.lines() { - let line = line.trim(); - if line.is_empty() { - follows_blank_line = true; - continue; - } - - let is_heading = line.starts_with('#'); - let starts_new_paragraph = follows_blank_line - && !previous_line_was_heading - && (has_minimum_word_count(¶graph.join(" "), MIN_PARAGRAPH_WORDS) - || has_minimum_word_count(line, MIN_PARAGRAPH_WORDS)); - - if is_heading || starts_new_paragraph { - push_description_block(&mut blocks, &mut paragraph); - } - - paragraph.push(line); - previous_line_was_heading = is_heading; - follows_blank_line = false; - } - - push_description_block(&mut blocks, &mut paragraph); - blocks + const MIN_PARAGRAPH_WORDS: usize = 8; + + let readable = description_language_input(markdown); + let mut blocks = Vec::new(); + let mut paragraph = Vec::new(); + let mut previous_line_was_heading = false; + let mut follows_blank_line = false; + + for line in readable.lines() { + let line = line.trim(); + if line.is_empty() { + follows_blank_line = true; + continue; + } + + let is_heading = line.starts_with('#'); + let starts_new_paragraph = follows_blank_line + && !previous_line_was_heading + && (has_minimum_word_count( + ¶graph.join(" "), + MIN_PARAGRAPH_WORDS, + ) || has_minimum_word_count(line, MIN_PARAGRAPH_WORDS)); + + if is_heading || starts_new_paragraph { + push_description_block(&mut blocks, &mut paragraph); + } + + paragraph.push(line); + previous_line_was_heading = is_heading; + follows_blank_line = false; + } + + push_description_block(&mut blocks, &mut paragraph); + blocks } fn has_minimum_word_count(text: &str, minimum: usize) -> bool { - WORD.find_iter(text).take(minimum).count() == minimum + WORD.find_iter(text).take(minimum).count() == minimum } fn push_description_block(blocks: &mut Vec, paragraph: &mut Vec<&str>) { - if paragraph.is_empty() { - return; - } - - let block = extract_description_text(¶graph.join("\n")); - if !block.is_empty() { - blocks.push(block); - } - paragraph.clear(); + if paragraph.is_empty() { + return; + } + + let block = extract_description_text(¶graph.join("\n")); + if !block.is_empty() { + blocks.push(block); + } + paragraph.clear(); } pub(super) fn has_image_without_alt_text(markdown: &str) -> bool { diff --git a/apps/labrinth/tests/project.rs b/apps/labrinth/tests/project.rs index 3e7373dc4c..e58724a159 100644 --- a/apps/labrinth/tests/project.rs +++ b/apps/labrinth/tests/project.rs @@ -660,7 +660,7 @@ async fn test_leaving_review_skips_validation() { #[actix_rt::test] async fn test_description_similarity_to_summary() { - with_test_environment( + with_test_environment( None, |test_env: TestEnvironment| async move { let api = &test_env.api; From e55b334d602a1f5e099da3eadcd04f06ca7c686b Mon Sep 17 00:00:00 2001 From: tdgao Date: Mon, 14 Sep 2026 13:30:20 -0600 Subject: [PATCH 14/17] pnpm prepr --- packages/moderation/src/locales/en-US/index.json | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/packages/moderation/src/locales/en-US/index.json b/packages/moderation/src/locales/en-US/index.json index be9d0e4a4c..14d6995208 100644 --- a/packages/moderation/src/locales/en-US/index.json +++ b/packages/moderation/src/locales/en-US/index.json @@ -197,6 +197,9 @@ "nags.project-description-banned-link.description": { "defaultMessage": "The link “{fullUrl}” is not allowed in project descriptions." }, + "nags.project-description-content.title": { + "defaultMessage": "Review the description" + }, "nags.project-description-non-english.description": { "defaultMessage": "Your project's description must be written in English or include an English translation." }, @@ -380,6 +383,9 @@ "nags.visit-links-settings.title": { "defaultMessage": "Visit links settings" }, + "project.text-validation.description-matches-summary": { + "defaultMessage": "Your project's description should provide unique information and not repeat the project's summary." + }, "project.text-validation.summary-matches-title": { "defaultMessage": "Your project's summary should provide unique information and not repeat the project's name." }, From 422fbdc57cfbeac1f1c0bb2d712af6a7288ebb2c Mon Sep 17 00:00:00 2001 From: aecsocket <43144841+aecsocket@users.noreply.github.com> Date: Tue, 15 Sep 2026 12:53:18 +0400 Subject: [PATCH 15/17] performance fix + all tags selected fix --- apps/labrinth/src/validate/project/tags.rs | 88 ++++++++++++++++++---- apps/labrinth/src/validate/project/text.rs | 16 ++-- 2 files changed, 80 insertions(+), 24 deletions(-) diff --git a/apps/labrinth/src/validate/project/tags.rs b/apps/labrinth/src/validate/project/tags.rs index fda8188387..db5fe8b867 100644 --- a/apps/labrinth/src/validate/project/tags.rs +++ b/apps/labrinth/src/validate/project/tags.rs @@ -1,3 +1,5 @@ +use std::collections::BTreeSet; + use crate::{ database::models::categories::Category, models::{projects::Project, v2::projects::LegacyProject}, @@ -80,23 +82,77 @@ pub(super) fn validate( ); } - if let Some(available_categories) = available_categories { - let total_available_tags = available_categories - .iter() - .filter(|category| category.project_type == project_type) - .count(); - if total_available_tags > 0 && tag_count == total_available_tags { - nags.push( - ProjectNag::new( - ProjectNagKind::AllTagsSelected, - ProjectNagSeverity::Required, - ) - .with_details(serde_json::json!({ - "total_available_tags": total_available_tags, - })), - ); - } + if let Some(available_categories) = available_categories + && let Some(total_available_tags) = all_available_tags_selected( + project + .categories + .iter() + .chain(&project.additional_categories) + .map(String::as_str), + available_categories + .iter() + .filter(|category| category.project_type == project_type) + .map(|category| category.category.as_str()), + ) + { + nags.push( + ProjectNag::new( + ProjectNagKind::AllTagsSelected, + ProjectNagSeverity::Required, + ) + .with_details(serde_json::json!({ + "total_available_tags": total_available_tags, + })), + ); } nags } + +fn all_available_tags_selected<'a, 'b>( + selected_tags: impl Iterator, + available_tags: impl Iterator, +) -> Option { + let selected_tags = selected_tags.collect::>(); + let available_tags = available_tags.collect::>(); + + (!available_tags.is_empty() && available_tags.is_subset(&selected_tags)) + .then_some(available_tags.len()) +} + +#[cfg(test)] +mod tests { + use super::all_available_tags_selected; + + #[test] + fn all_available_tags_are_compared_by_value() { + assert_eq!( + all_available_tags_selected( + ["combat", "magic", "mobs"].into_iter(), + ["combat", "magic", "mobs"].into_iter(), + ), + Some(3) + ); + assert_eq!( + all_available_tags_selected( + ["combat", "magic", "modpack-exclusive"].into_iter(), + ["combat", "magic", "mobs"].into_iter(), + ), + None + ); + assert_eq!( + all_available_tags_selected( + ["combat", "magic", "mobs", "modpack-exclusive"].into_iter(), + ["combat", "magic", "mobs"].into_iter(), + ), + Some(3) + ); + assert_eq!( + all_available_tags_selected( + ["modpack-exclusive"].into_iter(), + std::iter::empty(), + ), + None + ); + } +} diff --git a/apps/labrinth/src/validate/project/text.rs b/apps/labrinth/src/validate/project/text.rs index 710c5365d0..338e43a674 100644 --- a/apps/labrinth/src/validate/project/text.rs +++ b/apps/labrinth/src/validate/project/text.rs @@ -1009,6 +1009,7 @@ pub(super) fn extract_description_blocks(markdown: &str) -> Vec { let readable = description_language_input(markdown); let mut blocks = Vec::new(); let mut paragraph = Vec::new(); + let mut paragraph_word_count = 0; let mut previous_line_was_heading = false; let mut follows_blank_line = false; @@ -1020,18 +1021,21 @@ pub(super) fn extract_description_blocks(markdown: &str) -> Vec { } let is_heading = line.starts_with('#'); + let line_word_count = + WORD.find_iter(line).take(MIN_PARAGRAPH_WORDS).count(); let starts_new_paragraph = follows_blank_line && !previous_line_was_heading - && (has_minimum_word_count( - ¶graph.join(" "), - MIN_PARAGRAPH_WORDS, - ) || has_minimum_word_count(line, MIN_PARAGRAPH_WORDS)); + && (paragraph_word_count >= MIN_PARAGRAPH_WORDS + || line_word_count >= MIN_PARAGRAPH_WORDS); if is_heading || starts_new_paragraph { push_description_block(&mut blocks, &mut paragraph); + paragraph_word_count = 0; } paragraph.push(line); + paragraph_word_count = + (paragraph_word_count + line_word_count).min(MIN_PARAGRAPH_WORDS); previous_line_was_heading = is_heading; follows_blank_line = false; } @@ -1040,10 +1044,6 @@ pub(super) fn extract_description_blocks(markdown: &str) -> Vec { blocks } -fn has_minimum_word_count(text: &str, minimum: usize) -> bool { - WORD.find_iter(text).take(minimum).count() == minimum -} - fn push_description_block(blocks: &mut Vec, paragraph: &mut Vec<&str>) { if paragraph.is_empty() { return; From b5137d52a230f1671f45432193de9ef8a499facc Mon Sep 17 00:00:00 2001 From: tdgao Date: Tue, 15 Sep 2026 13:16:15 -0600 Subject: [PATCH 16/17] fmt --- Cargo.toml | 146 +++++++++++++++++-------------- apps/daedalus_client/Cargo.toml | 6 +- apps/labrinth/Cargo.toml | 27 +++++- packages/app-lib/Cargo.toml | 10 ++- packages/sqlx-tracing/Cargo.toml | 7 +- packages/xredis/Cargo.toml | 7 +- 6 files changed, 126 insertions(+), 77 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index aa2fc058bd..108713bb13 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,22 +1,22 @@ [workspace] resolver = "2" members = [ - "apps/app", - "apps/app-playground", - "apps/daedalus_client", - "apps/labrinth", - "packages/app-lib", - "packages/ariadne", - "packages/component-derive", - "packages/daedalus", - "packages/modrinth-content-management", - "packages/modrinth-log", - "packages/modrinth-maxmind", - "packages/modrinth-util", - "packages/neverbounce", - "packages/path-util", - "packages/serde-binhum", - "packages/xredis", + "apps/app", + "apps/app-playground", + "apps/daedalus_client", + "apps/labrinth", + "packages/app-lib", + "packages/ariadne", + "packages/component-derive", + "packages/daedalus", + "packages/modrinth-content-management", + "packages/modrinth-log", + "packages/modrinth-maxmind", + "packages/modrinth-util", + "packages/neverbounce", + "packages/path-util", + "packages/serde-binhum", + "packages/xredis", ] [workspace.package] @@ -39,28 +39,35 @@ ariadne = { path = "packages/ariadne" } async-minecraft-ping = { path = "packages/async-minecraft-ping" } async-recursion = "1.1.1" async-stripe = { version = "0.41.0", default-features = false, features = [ - "runtime-tokio-hyper-rustls", + "runtime-tokio-hyper-rustls", ] } async-trait = "0.1.89" -async-tungstenite = { version = "0.31.0", default-features = false, features = ["futures-03-sink"] } +async-tungstenite = { version = "0.31.0", default-features = false, features = [ + "futures-03-sink" +] } async-walkdir = "2.1.0" async_zip = "0.0.18" aws-sdk-s3 = { version = "=1.122.0", default-features = false, features = [ - "default-https-client", - "rt-tokio", - "rustls", + "default-https-client", + "rt-tokio", + "rustls", ] } base64 = "0.22.1" bitflags = "2.9.4" bon = "3.9.3" bytemuck = "1.24.0" bytes = "1.10.1" -cel = { version = "0.14.0", default-features = false, features = ["json", "regex"] } +cel = { version = "0.14.0", default-features = false, features = [ + "json", + "regex" +] } censor = "0.3.0" chardetng = "0.1.17" chrono = "0.4.42" chumsky = "0.9.3" -cidre = { version = "0.15.0", default-features = false, features = ["macos_15_0"] } +cidre = { version = "0.15.0", default-features = false, features = [ + "macos_15_0" +] } clap = "4.5.48" clickhouse = "0.14.0" color-eyre = "0.6.5" @@ -99,10 +106,10 @@ hmac = "0.12.1" httpdate = "1.0.3" hyper = "1.7.0" hyper-rustls = { version = "0.27.7", default-features = false, features = [ - "aws-lc-rs", - "http1", - "native-tokio", - "tls12", + "aws-lc-rs", + "http1", + "native-tokio", + "tls12", ] } hyper-util = "0.1.17" iana-time-zone = "0.1.64" @@ -114,23 +121,23 @@ jemalloc_pprof = "0.8.1" json-patch = { version = "4.1.0", default-features = false } json5 = "1.3.1" lettre = { version = "0.11.19", default-features = false, features = [ - "aws-lc-rs", - "builder", - "hostname", - "pool", - "rustls", - "rustls-native-certs", - "smtp-transport", - "tokio1", - "tokio1-rustls", + "aws-lc-rs", + "builder", + "hostname", + "pool", + "rustls", + "rustls-native-certs", + "smtp-transport", + "tokio1", + "tokio1-rustls", ] } lingua = "=1.8.0" linkify = "0.11.0" lz4_flex = { version = "0.11.5", default-features = false, features = [ - "checked-decode", - "safe-decode", - "safe-encode", - "std", + "checked-decode", + "safe-decode", + "safe-encode", + "std", ] } maxminddb = "0.26.0" md5 = "0.8.0" @@ -160,19 +167,22 @@ pulldown-cmark = { version = "0.13.4", default-features = false } quartz_nbt = "0.2.9" quick-xml = "0.38.3" quote = { version = "1.0" } -rand = "=0.8.5" # Locked on 0.8 until argon2 and p256 update to 0.9 -rand_chacha = "=0.3.1" # Locked on 0.3 until we can update rand to 0.9 +rand = "=0.8.5" # Locked on 0.8 until argon2 and p256 update to 0.9 +rand_chacha = "=0.3.1" # Locked on 0.3 until we can update rand to 0.9 rdkafka = { version = "0.36.2", features = ["cmake-build"] } redis = "1.4.1" regex = "1.12.2" reqwest = { version = "0.12.24", default-features = false } rgb = "0.8.52" -rust_decimal = { version = "1.39.0", features = ["serde-with-float", "serde-with-str"] } +rust_decimal = { version = "1.39.0", features = [ + "serde-with-float", + "serde-with-str" +] } rust_iso3166 = "0.1.14" rust-s3 = { version = "0.37.0", default-features = false, features = [ - "fail-on-err", - "tags", - "tokio-rustls-tls", + "fail-on-err", + "tags", + "tokio-rustls-tls", ] } rustls = "0.23.32" rustrict = { version = "0.7.39", default-features = false, features = ["censor"] } @@ -181,12 +191,12 @@ same-file = "1.0.6" scalar_api_reference = { version = "0.2.2", default-features = false } secrecy = "0.10.3" sentry = { version = "0.45.0", default-features = false, features = [ - "backtrace", - "contexts", - "debug-images", - "panic", - "reqwest", - "rustls", + "backtrace", + "contexts", + "debug-images", + "panic", + "reqwest", + "rustls", ] } serde = "1.0.228" serde-binhum = { path = "packages/serde-binhum" } @@ -217,8 +227,8 @@ tauri-plugin-opener = "2.5.0" tauri-plugin-os = "2.3.1" tauri-plugin-single-instance = "2.3.4" tauri-plugin-updater = { git = "https://github.com/modrinth/plugins-workspace", rev = "0d30f2aa28ec668ce187d527da1c475da3c01cbc", default-features = false, features = [ - "rustls-tls", - "zip", + "rustls-tls", + "zip", ] } tauri-plugin-window-state = "2.4.0" tempfile = "3.23.0" @@ -248,20 +258,20 @@ validator = "0.20.0" webauthn-rs = "0.5.5" webauthn-rs-proto = "0.5.5" webp = { version = "0.3.1", default-features = false } -webview2-com = "0.38.0" # Should be updated in lockstep with wry +webview2-com = "0.38.0" # Should be updated in lockstep with wry whoami = "1.6.1" -windows = "=0.61.3" # Locked on 0.61 until we can update windows-core to 0.62 -windows-core = "=0.61.2" # Locked on 0.61 until webview2-com updates to 0.62 +windows = "=0.61.3" # Locked on 0.61 until we can update windows-core to 0.62 +windows-core = "=0.61.2" # Locked on 0.61 until webview2-com updates to 0.62 winreg = "0.55.0" woothee = "0.13.0" xredis = { path = "packages/xredis" } yaserde = "0.12.0" zbus = "5.11.0" zip = { version = "6.0.0", default-features = false, features = [ - "bzip2", - "deflate", - "deflate64", - "zstd", + "bzip2", + "deflate", + "deflate64", + "zstd", ] } zxcvbn = "3.1.0" @@ -319,15 +329,15 @@ opt-level = 3 # Optimize for speed and reduce size on release builds [profile.release] -opt-level = "s" # Optimize for binary size -strip = true # Remove debug symbols -lto = true # Enables link to optimizations -panic = "abort" # Strip expensive panic clean-up logic +opt-level = "s" # Optimize for binary size +strip = true # Remove debug symbols +lto = true # Enables link to optimizations +panic = "abort" # Strip expensive panic clean-up logic # Specific profile for labrinth production builds [profile.release-labrinth] inherits = "release" opt-level = 2 -strip = false # Keep debug symbols for Sentry -lto = "thin" # Enable LTO but keep compile times reasonable -panic = "unwind" # Don't exit the whole app on panic in production +strip = false # Keep debug symbols for Sentry +lto = "thin" # Enable LTO but keep compile times reasonable +panic = "unwind" # Don't exit the whole app on panic in production diff --git a/apps/daedalus_client/Cargo.toml b/apps/daedalus_client/Cargo.toml index af67313d83..3894856642 100644 --- a/apps/daedalus_client/Cargo.toml +++ b/apps/daedalus_client/Cargo.toml @@ -22,7 +22,11 @@ indexmap = { workspace = true, features = ["serde"] } itertools = { workspace = true } md5 = { workspace = true } quick-xml = { workspace = true } -reqwest = { workspace = true, features = ["json", "rustls-tls-native-roots", "stream"] } +reqwest = { workspace = true, features = [ + "json", + "rustls-tls-native-roots", + "stream" +] } rust-s3 = { workspace = true } serde = { workspace = true, features = ["derive"] } serde_json = { workspace = true } diff --git a/apps/labrinth/Cargo.toml b/apps/labrinth/Cargo.toml index 9f8599feb5..78c25a0fd5 100644 --- a/apps/labrinth/Cargo.toml +++ b/apps/labrinth/Cargo.toml @@ -23,7 +23,12 @@ arc-swap = { workspace = true } argon2 = { workspace = true } ariadne = { workspace = true } async-minecraft-ping = { workspace = true, features = ["srv"] } -async-stripe = { workspace = true, features = ["billing", "checkout", "connect", "webhook-events"] } +async-stripe = { workspace = true, features = [ + "billing", + "checkout", + "connect", + "webhook-events" +] } async-trait = { workspace = true } aws-sdk-s3 = { workspace = true } base64 = { workspace = true } @@ -89,10 +94,24 @@ quick-xml = { workspace = true } rand = { workspace = true } rand_chacha = { workspace = true } rdkafka = { workspace = true } -redis = { workspace = true, features = ["ahash", "cluster", "cluster-async", "r2d2", "tokio-comp"] } +redis = { workspace = true, features = [ + "ahash", + "cluster", + "cluster-async", + "r2d2", + "tokio-comp" +] } regex = { workspace = true } -reqwest = { workspace = true, features = ["http2", "json", "multipart", "rustls-tls-webpki-roots"] } -rust_decimal = { workspace = true, features = ["serde-with-float", "serde-with-str"] } +reqwest = { workspace = true, features = [ + "http2", + "json", + "multipart", + "rustls-tls-webpki-roots" +] } +rust_decimal = { workspace = true, features = [ + "serde-with-float", + "serde-with-str" +] } rust_iso3166 = { workspace = true } rustls.workspace = true rustrict = { workspace = true } diff --git a/packages/app-lib/Cargo.toml b/packages/app-lib/Cargo.toml index 9bb14ce8a4..81b5cc74d4 100644 --- a/packages/app-lib/Cargo.toml +++ b/packages/app-lib/Cargo.toml @@ -13,7 +13,10 @@ ariadne = { workspace = true } async-minecraft-ping = { workspace = true, features = ["srv"] } async-recursion = { workspace = true } async-trait = { workspace = true } -async-tungstenite = { workspace = true, features = ["tokio-runtime", "tokio-rustls-webpki-roots"] } +async-tungstenite = { workspace = true, features = [ + "tokio-runtime", + "tokio-rustls-webpki-roots" +] } async-walkdir = { workspace = true } async_zip = { workspace = true, features = [ "bzip2", @@ -136,7 +139,10 @@ dunce = { workspace = true } cidre = { workspace = true, features = ["blocks", "nw"] } [target."cfg(windows)".dependencies] -windows = { workspace = true, features = ["Networking_Connectivity", "Win32_Storage_FileSystem"] } +windows = { workspace = true, features = [ + "Networking_Connectivity", + "Win32_Storage_FileSystem" +] } windows-core = { workspace = true } winreg = { workspace = true } diff --git a/packages/sqlx-tracing/Cargo.toml b/packages/sqlx-tracing/Cargo.toml index 3ce51c659a..bf89e32aa8 100644 --- a/packages/sqlx-tracing/Cargo.toml +++ b/packages/sqlx-tracing/Cargo.toml @@ -10,7 +10,12 @@ repository = "https://github.com/jdrouet/sqlx-tracing" license = "MIT" # authors = ["Jérémie Drouet "] # deprecated field, Tombi warns keywords = ["database", "observability", "opentelemetry", "sqlx", "tracing"] -categories = ["asynchronous", "database", "development-tools::debugging", "development-tools::profiling"] +categories = [ + "asynchronous", + "database", + "development-tools::debugging", + "development-tools::profiling" +] [dependencies] derive_more = { workspace = true, features = ["deref", "deref_mut"] } diff --git a/packages/xredis/Cargo.toml b/packages/xredis/Cargo.toml index 14a698c047..cc87779098 100644 --- a/packages/xredis/Cargo.toml +++ b/packages/xredis/Cargo.toml @@ -15,7 +15,12 @@ futures = { workspace = true } lz4_flex = { workspace = true } postcard = { workspace = true } prometheus = { workspace = true } -redis = { workspace = true, features = ["ahash", "cluster", "cluster-async", "tokio-comp"] } +redis = { workspace = true, features = [ + "ahash", + "cluster", + "cluster-async", + "tokio-comp" +] } serde = { workspace = true, features = ["derive"] } thiserror = { workspace = true } tokio = { workspace = true, features = ["rt", "sync", "time"] } From 5100b9b53bee1f7dc63575bd0357243b7458f10a Mon Sep 17 00:00:00 2001 From: tdgao Date: Tue, 15 Sep 2026 13:20:31 -0600 Subject: [PATCH 17/17] fmt 2 --- Cargo.toml | 18 ++++-------------- apps/daedalus_client/Cargo.toml | 6 +----- apps/labrinth/Cargo.toml | 27 ++++----------------------- packages/app-lib/Cargo.toml | 10 ++-------- packages/sqlx-tracing/Cargo.toml | 7 +------ packages/xredis/Cargo.toml | 7 +------ 6 files changed, 13 insertions(+), 62 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 108713bb13..ca056543e7 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -42,9 +42,7 @@ async-stripe = { version = "0.41.0", default-features = false, features = [ "runtime-tokio-hyper-rustls", ] } async-trait = "0.1.89" -async-tungstenite = { version = "0.31.0", default-features = false, features = [ - "futures-03-sink" -] } +async-tungstenite = { version = "0.31.0", default-features = false, features = ["futures-03-sink"] } async-walkdir = "2.1.0" async_zip = "0.0.18" aws-sdk-s3 = { version = "=1.122.0", default-features = false, features = [ @@ -57,17 +55,12 @@ bitflags = "2.9.4" bon = "3.9.3" bytemuck = "1.24.0" bytes = "1.10.1" -cel = { version = "0.14.0", default-features = false, features = [ - "json", - "regex" -] } +cel = { version = "0.14.0", default-features = false, features = ["json", "regex"] } censor = "0.3.0" chardetng = "0.1.17" chrono = "0.4.42" chumsky = "0.9.3" -cidre = { version = "0.15.0", default-features = false, features = [ - "macos_15_0" -] } +cidre = { version = "0.15.0", default-features = false, features = ["macos_15_0"] } clap = "4.5.48" clickhouse = "0.14.0" color-eyre = "0.6.5" @@ -174,10 +167,7 @@ redis = "1.4.1" regex = "1.12.2" reqwest = { version = "0.12.24", default-features = false } rgb = "0.8.52" -rust_decimal = { version = "1.39.0", features = [ - "serde-with-float", - "serde-with-str" -] } +rust_decimal = { version = "1.39.0", features = ["serde-with-float", "serde-with-str"] } rust_iso3166 = "0.1.14" rust-s3 = { version = "0.37.0", default-features = false, features = [ "fail-on-err", diff --git a/apps/daedalus_client/Cargo.toml b/apps/daedalus_client/Cargo.toml index 3894856642..af67313d83 100644 --- a/apps/daedalus_client/Cargo.toml +++ b/apps/daedalus_client/Cargo.toml @@ -22,11 +22,7 @@ indexmap = { workspace = true, features = ["serde"] } itertools = { workspace = true } md5 = { workspace = true } quick-xml = { workspace = true } -reqwest = { workspace = true, features = [ - "json", - "rustls-tls-native-roots", - "stream" -] } +reqwest = { workspace = true, features = ["json", "rustls-tls-native-roots", "stream"] } rust-s3 = { workspace = true } serde = { workspace = true, features = ["derive"] } serde_json = { workspace = true } diff --git a/apps/labrinth/Cargo.toml b/apps/labrinth/Cargo.toml index 78c25a0fd5..9f8599feb5 100644 --- a/apps/labrinth/Cargo.toml +++ b/apps/labrinth/Cargo.toml @@ -23,12 +23,7 @@ arc-swap = { workspace = true } argon2 = { workspace = true } ariadne = { workspace = true } async-minecraft-ping = { workspace = true, features = ["srv"] } -async-stripe = { workspace = true, features = [ - "billing", - "checkout", - "connect", - "webhook-events" -] } +async-stripe = { workspace = true, features = ["billing", "checkout", "connect", "webhook-events"] } async-trait = { workspace = true } aws-sdk-s3 = { workspace = true } base64 = { workspace = true } @@ -94,24 +89,10 @@ quick-xml = { workspace = true } rand = { workspace = true } rand_chacha = { workspace = true } rdkafka = { workspace = true } -redis = { workspace = true, features = [ - "ahash", - "cluster", - "cluster-async", - "r2d2", - "tokio-comp" -] } +redis = { workspace = true, features = ["ahash", "cluster", "cluster-async", "r2d2", "tokio-comp"] } regex = { workspace = true } -reqwest = { workspace = true, features = [ - "http2", - "json", - "multipart", - "rustls-tls-webpki-roots" -] } -rust_decimal = { workspace = true, features = [ - "serde-with-float", - "serde-with-str" -] } +reqwest = { workspace = true, features = ["http2", "json", "multipart", "rustls-tls-webpki-roots"] } +rust_decimal = { workspace = true, features = ["serde-with-float", "serde-with-str"] } rust_iso3166 = { workspace = true } rustls.workspace = true rustrict = { workspace = true } diff --git a/packages/app-lib/Cargo.toml b/packages/app-lib/Cargo.toml index 81b5cc74d4..9bb14ce8a4 100644 --- a/packages/app-lib/Cargo.toml +++ b/packages/app-lib/Cargo.toml @@ -13,10 +13,7 @@ ariadne = { workspace = true } async-minecraft-ping = { workspace = true, features = ["srv"] } async-recursion = { workspace = true } async-trait = { workspace = true } -async-tungstenite = { workspace = true, features = [ - "tokio-runtime", - "tokio-rustls-webpki-roots" -] } +async-tungstenite = { workspace = true, features = ["tokio-runtime", "tokio-rustls-webpki-roots"] } async-walkdir = { workspace = true } async_zip = { workspace = true, features = [ "bzip2", @@ -139,10 +136,7 @@ dunce = { workspace = true } cidre = { workspace = true, features = ["blocks", "nw"] } [target."cfg(windows)".dependencies] -windows = { workspace = true, features = [ - "Networking_Connectivity", - "Win32_Storage_FileSystem" -] } +windows = { workspace = true, features = ["Networking_Connectivity", "Win32_Storage_FileSystem"] } windows-core = { workspace = true } winreg = { workspace = true } diff --git a/packages/sqlx-tracing/Cargo.toml b/packages/sqlx-tracing/Cargo.toml index bf89e32aa8..3ce51c659a 100644 --- a/packages/sqlx-tracing/Cargo.toml +++ b/packages/sqlx-tracing/Cargo.toml @@ -10,12 +10,7 @@ repository = "https://github.com/jdrouet/sqlx-tracing" license = "MIT" # authors = ["Jérémie Drouet "] # deprecated field, Tombi warns keywords = ["database", "observability", "opentelemetry", "sqlx", "tracing"] -categories = [ - "asynchronous", - "database", - "development-tools::debugging", - "development-tools::profiling" -] +categories = ["asynchronous", "database", "development-tools::debugging", "development-tools::profiling"] [dependencies] derive_more = { workspace = true, features = ["deref", "deref_mut"] } diff --git a/packages/xredis/Cargo.toml b/packages/xredis/Cargo.toml index cc87779098..14a698c047 100644 --- a/packages/xredis/Cargo.toml +++ b/packages/xredis/Cargo.toml @@ -15,12 +15,7 @@ futures = { workspace = true } lz4_flex = { workspace = true } postcard = { workspace = true } prometheus = { workspace = true } -redis = { workspace = true, features = [ - "ahash", - "cluster", - "cluster-async", - "tokio-comp" -] } +redis = { workspace = true, features = ["ahash", "cluster", "cluster-async", "tokio-comp"] } serde = { workspace = true, features = ["derive"] } thiserror = { workspace = true } tokio = { workspace = true, features = ["rt", "sync", "time"] }