diff --git a/Cargo.lock b/Cargo.lock index 24a7a68e55..a5eff32532 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2529,6 +2529,15 @@ dependencies = [ "libc", ] +[[package]] +name = "counter" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "66e8e052be91f1c8aae2c1d81307d9f6e67f5f37001e3ddee419e971e73f03bc" +dependencies = [ + "num-traits", +] + [[package]] name = "cpufeatures" version = "0.2.17" @@ -3754,6 +3763,9 @@ name = "fastrand" version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be" +dependencies = [ + "getrandom 0.2.16", +] [[package]] name = "fax" @@ -3963,6 +3975,12 @@ dependencies = [ "libc", ] +[[package]] +name = "fst" +version = "0.4.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ab85b9b05e3978cc9a9cf8fea7f01b494e1a09ed3037e16ba39edc7a29eb61a" + [[package]] name = "funty" version = "2.0.0" @@ -5296,6 +5314,25 @@ version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "35a84fd5aa25fae5c0f4a33d9cac2ca017fc622cbd089be2229993514990f870" +[[package]] +name = "include_dir" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "923d117408f1e49d914f1a379a309cffe4f18c05cf4e3d12e613a15fc81bd0dd" +dependencies = [ + "include_dir_macros", +] + +[[package]] +name = "include_dir_macros" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7cab85a7ed0bd5f0e76d93846e0147172bed2e2d3f859bcc33a8d9699cad1a75" +dependencies = [ + "proc-macro2", + "quote", +] + [[package]] name = "indenter" version = "0.3.4" @@ -5746,6 +5783,7 @@ dependencies = [ "json-patch", "labrinth", "lettre", + "lingua", "linkify", "modrinth-content-management", "modrinth-util", @@ -5801,7 +5839,6 @@ dependencies = [ "webauthn-rs", "webauthn-rs-proto", "webp", - "whatlang", "woothee", "xredis", "yaserde", @@ -5999,6 +6036,778 @@ dependencies = [ "vcpkg", ] +[[package]] +name = "lingua" +version = "1.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f40d9129bb9fe42c95d1bd420d6891607eaff17df16ee15674aed2d05b0ec8f4" +dependencies = [ + "counter", + "dashmap", + "fastrand 2.3.0", + "fst", + "include_dir", + "itertools 0.14.0", + "lingua-afrikaans-language-model", + "lingua-albanian-language-model", + "lingua-arabic-language-model", + "lingua-armenian-language-model", + "lingua-azerbaijani-language-model", + "lingua-basque-language-model", + "lingua-belarusian-language-model", + "lingua-bengali-language-model", + "lingua-bokmal-language-model", + "lingua-bosnian-language-model", + "lingua-bulgarian-language-model", + "lingua-catalan-language-model", + "lingua-chinese-language-model", + "lingua-croatian-language-model", + "lingua-czech-language-model", + "lingua-danish-language-model", + "lingua-dutch-language-model", + "lingua-english-language-model", + "lingua-esperanto-language-model", + "lingua-estonian-language-model", + "lingua-finnish-language-model", + "lingua-french-language-model", + "lingua-ganda-language-model", + "lingua-georgian-language-model", + "lingua-german-language-model", + "lingua-greek-language-model", + "lingua-gujarati-language-model", + "lingua-hebrew-language-model", + "lingua-hindi-language-model", + "lingua-hungarian-language-model", + "lingua-icelandic-language-model", + "lingua-indonesian-language-model", + "lingua-irish-language-model", + "lingua-italian-language-model", + "lingua-japanese-language-model", + "lingua-kazakh-language-model", + "lingua-korean-language-model", + "lingua-latin-language-model", + "lingua-latvian-language-model", + "lingua-lithuanian-language-model", + "lingua-macedonian-language-model", + "lingua-malay-language-model", + "lingua-maori-language-model", + "lingua-marathi-language-model", + "lingua-mongolian-language-model", + "lingua-nynorsk-language-model", + "lingua-persian-language-model", + "lingua-polish-language-model", + "lingua-portuguese-language-model", + "lingua-punjabi-language-model", + "lingua-romanian-language-model", + "lingua-russian-language-model", + "lingua-serbian-language-model", + "lingua-shona-language-model", + "lingua-slovak-language-model", + "lingua-slovene-language-model", + "lingua-somali-language-model", + "lingua-sotho-language-model", + "lingua-spanish-language-model", + "lingua-swahili-language-model", + "lingua-swedish-language-model", + "lingua-tagalog-language-model", + "lingua-tamil-language-model", + "lingua-telugu-language-model", + "lingua-thai-language-model", + "lingua-tsonga-language-model", + "lingua-tswana-language-model", + "lingua-turkish-language-model", + "lingua-ukrainian-language-model", + "lingua-urdu-language-model", + "lingua-vietnamese-language-model", + "lingua-welsh-language-model", + "lingua-xhosa-language-model", + "lingua-yoruba-language-model", + "lingua-zulu-language-model", + "maplit", + "rayon", + "regex", + "serde", + "serde-wasm-bindgen", + "strum", + "strum_macros", + "wasm-bindgen", +] + +[[package]] +name = "lingua-afrikaans-language-model" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2810a47263f58358fa22acc425b330a334382f33fcee8f205be672a7ec6d8d4a" +dependencies = [ + "include_dir", +] + +[[package]] +name = "lingua-albanian-language-model" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "81be672350a5a37c9aae935dd8520295289cc02d098929484b963b60fb42e174" +dependencies = [ + "include_dir", +] + +[[package]] +name = "lingua-arabic-language-model" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5bcc254ff44209c2a50dea58a644f9c257a0aa5cd7e1739fd9c373fe4c7456cf" +dependencies = [ + "include_dir", +] + +[[package]] +name = "lingua-armenian-language-model" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4cf4e5825cb3a2b11d8bd7baa6788167b65ffbf8ce21dc9b9dcb5e6adee1d5f0" +dependencies = [ + "include_dir", +] + +[[package]] +name = "lingua-azerbaijani-language-model" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3fdcc775ff75bcc6de0ea5330f65e62644622e29c980a72b437780ba57bba838" +dependencies = [ + "include_dir", +] + +[[package]] +name = "lingua-basque-language-model" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5cbea869a4556800f2798c61559fd2e87fd65d7698ce27ad6fba5b1d9ec1fbe9" +dependencies = [ + "include_dir", +] + +[[package]] +name = "lingua-belarusian-language-model" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2887044d4c11388e15692bf953f70468282600c426e1cd6a208a93aeca3aedfd" +dependencies = [ + "include_dir", +] + +[[package]] +name = "lingua-bengali-language-model" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f94131563e0e0c571cebf03f867d0e1b7dd15278f7cab3e4ba8b0f8026072cc" +dependencies = [ + "include_dir", +] + +[[package]] +name = "lingua-bokmal-language-model" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b443b1ec0498bcf43594f9bf65ae8f5660b5bf21eb769950a245ced8e376ac23" +dependencies = [ + "include_dir", +] + +[[package]] +name = "lingua-bosnian-language-model" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "04f540ed0441bcdbb0b36c43f306bc2b0892921ceb95e865fe0b8b0c39bb82c4" +dependencies = [ + "include_dir", +] + +[[package]] +name = "lingua-bulgarian-language-model" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2f4679441ff2d94b21a3d76293ffd5310a012ea62247b7b2066939e2728cbfc0" +dependencies = [ + "include_dir", +] + +[[package]] +name = "lingua-catalan-language-model" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7366ac180de87eb19f32063b8eed1db270ab333f7a450d96250e53b14b0a750f" +dependencies = [ + "include_dir", +] + +[[package]] +name = "lingua-chinese-language-model" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "21ca7fa9f7671d684c82c168725f380fc873f14d6f4e8c82f0da681bcc0048d1" +dependencies = [ + "include_dir", +] + +[[package]] +name = "lingua-croatian-language-model" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d86b1346b98397e7cf87e8f3fe544fce6514918e6bde253a014eea7ca03c31d8" +dependencies = [ + "include_dir", +] + +[[package]] +name = "lingua-czech-language-model" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "348f06e4f90e1c2bc597ea3caf52abb9d3dd09ba227fa7cf4f2dc723a9810e98" +dependencies = [ + "include_dir", +] + +[[package]] +name = "lingua-danish-language-model" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "363953413132601a06fe0cae1fffb5d519d0cfb378049cc78b3893edd03a194f" +dependencies = [ + "include_dir", +] + +[[package]] +name = "lingua-dutch-language-model" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fe0da523f4726c05ed557b5a309315c1c161a5ec9ca6036c6c1c799ce492b698" +dependencies = [ + "include_dir", +] + +[[package]] +name = "lingua-english-language-model" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97102de08b134a49f1cce05a1b6f5bf08ef21fe858074ae2b794e7892c43dd4b" +dependencies = [ + "include_dir", +] + +[[package]] +name = "lingua-esperanto-language-model" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13d2a7ae84bfa3ed2a05815ca4a81d34cee7da1a77e830ab933c4debc21d6532" +dependencies = [ + "include_dir", +] + +[[package]] +name = "lingua-estonian-language-model" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7095f107a6e89147a9066db6a3f9e1b0a4706323d59a4bb58fa8218d74c1fde8" +dependencies = [ + "include_dir", +] + +[[package]] +name = "lingua-finnish-language-model" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f5b3c44089812704148c89dbe77fb34122c2f6e8182bbf0f72ea69c92e948e9e" +dependencies = [ + "include_dir", +] + +[[package]] +name = "lingua-french-language-model" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "45508227e42c9cc5eb202e17c4c40e38ea7b9be9421faeb3ab0fb7ac37d9c681" +dependencies = [ + "include_dir", +] + +[[package]] +name = "lingua-ganda-language-model" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f7d4b9fba87a72ae0f929fbe0bdc78f71be78d14475219718e70fd686b281082" +dependencies = [ + "include_dir", +] + +[[package]] +name = "lingua-georgian-language-model" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73f3568595e73295b905ec54cc2dbda915efaee5f7abf91d572e9d44a843e81d" +dependencies = [ + "include_dir", +] + +[[package]] +name = "lingua-german-language-model" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f584da803e8c135ea22dd3ed321a6b3e2ff3465559606be0a597924ecd465cb2" +dependencies = [ + "include_dir", +] + +[[package]] +name = "lingua-greek-language-model" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a5da9a688ceb41963f3555526a2e4431e0e5208a3320e56b72b1de998ba6b1c7" +dependencies = [ + "include_dir", +] + +[[package]] +name = "lingua-gujarati-language-model" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e495d0421f5e4d6b1d71ebf3a98b445c5ada6b03f033b6b62db2382e4d075663" +dependencies = [ + "include_dir", +] + +[[package]] +name = "lingua-hebrew-language-model" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc49d9a174de884bba595c8b575ee37fdb1c1a728eeff608d43e1a7378048545" +dependencies = [ + "include_dir", +] + +[[package]] +name = "lingua-hindi-language-model" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dca88af8dede1a30fcd79318b9c22f3662785fd87ed5b39881d19770651bf720" +dependencies = [ + "include_dir", +] + +[[package]] +name = "lingua-hungarian-language-model" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d7072b2cdf43438214e82b53c51ad85a16d657e5c07a5a77bef78a4cd39423c" +dependencies = [ + "include_dir", +] + +[[package]] +name = "lingua-icelandic-language-model" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "341fa17414af93a6ccf33c6efda9e65d1476a18785b075a16d53f66c9e19379d" +dependencies = [ + "include_dir", +] + +[[package]] +name = "lingua-indonesian-language-model" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fe1f2e05145c3fb4172c01c4563f729938c85f77742948444523dfcda2c3e80d" +dependencies = [ + "include_dir", +] + +[[package]] +name = "lingua-irish-language-model" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5099708a7fc7081e4171dbc0905691e5e02639d8899558a105720f28522261db" +dependencies = [ + "include_dir", +] + +[[package]] +name = "lingua-italian-language-model" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "83a87385ee42f6a0306df81066d4f77c225a337137fc15978a4b319c6bcf4bfe" +dependencies = [ + "include_dir", +] + +[[package]] +name = "lingua-japanese-language-model" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df0938f75de3ae5dcdc925d823ed409854ca14f6a653782b9a1ad5d899462fbe" +dependencies = [ + "include_dir", +] + +[[package]] +name = "lingua-kazakh-language-model" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af90d71a099b11f3b0677402a9894e95b0acb9459343735dfaec57af6be11ee6" +dependencies = [ + "include_dir", +] + +[[package]] +name = "lingua-korean-language-model" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aa87f6c43ff894fc75159c021480d2fdf96882bf5bd235f8916ceb6b7caae561" +dependencies = [ + "include_dir", +] + +[[package]] +name = "lingua-latin-language-model" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "757c1330e893bc532559397d976656a7bb0a42f391f7c946727ffb43a3761e46" +dependencies = [ + "include_dir", +] + +[[package]] +name = "lingua-latvian-language-model" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1b2f30766e9e1ce10d960ce3eb2da6514fd0ee18c8d617a566b03126140e656" +dependencies = [ + "include_dir", +] + +[[package]] +name = "lingua-lithuanian-language-model" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4e4a2a1994e48841c1d2d63099fbd68476615bf0d833fa14a3654f7043e38fc7" +dependencies = [ + "include_dir", +] + +[[package]] +name = "lingua-macedonian-language-model" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "779ea3952ecade3aff6a34eea6f432a0d10652fafd9723a1b2258551b08f83b4" +dependencies = [ + "include_dir", +] + +[[package]] +name = "lingua-malay-language-model" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "10936b2ac22076f9dbb968c71aacbf16c71ed36a92da10f077850d7ec5ded341" +dependencies = [ + "include_dir", +] + +[[package]] +name = "lingua-maori-language-model" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d5bbc1e34392b0081209eec7d6b201a062612717c0670dad451066d9026261da" +dependencies = [ + "include_dir", +] + +[[package]] +name = "lingua-marathi-language-model" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb68fa15d9a57a2ca2a71f8f59f455267bd8a8793216449697c2dd04898915a0" +dependencies = [ + "include_dir", +] + +[[package]] +name = "lingua-mongolian-language-model" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a72e6a2a3c9ecb22dbd8870bae40c167f767defd69f05c58e5467d233b6360d5" +dependencies = [ + "include_dir", +] + +[[package]] +name = "lingua-nynorsk-language-model" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2f5e7e54f8e04a5286aab3a8e22f395b42e9341cab1fdafdd04660a17e81fe8d" +dependencies = [ + "include_dir", +] + +[[package]] +name = "lingua-persian-language-model" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ddf084e05d33ed66d64461e6397b571fdf74fe93c13c4ac4c84a633e33d2d07b" +dependencies = [ + "include_dir", +] + +[[package]] +name = "lingua-polish-language-model" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72eb03e7940b3178c152138a16976e374ed0a3ddc80dc6f0b56984ab1189cb67" +dependencies = [ + "include_dir", +] + +[[package]] +name = "lingua-portuguese-language-model" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "432eda7984456055033ffa168037be9afe0f5c9ecd891bf5f27435247d496b47" +dependencies = [ + "include_dir", +] + +[[package]] +name = "lingua-punjabi-language-model" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "620ec0c678bfca1b99d5379f8024cfed24149556a0d55957799d870f3c71df26" +dependencies = [ + "include_dir", +] + +[[package]] +name = "lingua-romanian-language-model" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1051aa082754acfc52110456e29fbbf6d2c648da9a4753a861c34f19325cfab8" +dependencies = [ + "include_dir", +] + +[[package]] +name = "lingua-russian-language-model" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4fc0850578299531b50192be2c1de1af651f1289784253645e737fe456a6a138" +dependencies = [ + "include_dir", +] + +[[package]] +name = "lingua-serbian-language-model" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df46949714d864f81138b3590878c89f06cf5d0af0e758a1e1217dd13983a896" +dependencies = [ + "include_dir", +] + +[[package]] +name = "lingua-shona-language-model" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85ca30d90634fc427e79dddfa555524f5983a630ce809aeb02b708f434937965" +dependencies = [ + "include_dir", +] + +[[package]] +name = "lingua-slovak-language-model" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "529a596f3d90b2051b3ead90e864f6ad9e14b43716c3f252268e9e6e43ab28de" +dependencies = [ + "include_dir", +] + +[[package]] +name = "lingua-slovene-language-model" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db52978574611533d873b727281d1c1d0ab2b031331654c1729723b682bc76a1" +dependencies = [ + "include_dir", +] + +[[package]] +name = "lingua-somali-language-model" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be79a11e6d682c7f1ecb342fe41169a59d6adb5eac3b4de77973fb7bbf936e39" +dependencies = [ + "include_dir", +] + +[[package]] +name = "lingua-sotho-language-model" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c476a7019592446989258b7f204cc7b5429b42a4d7eb686b80461a5f790e08bc" +dependencies = [ + "include_dir", +] + +[[package]] +name = "lingua-spanish-language-model" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "56395a8d96c892130a9efb433c5f042977fb75ae5fb6e0058c8814a947459137" +dependencies = [ + "include_dir", +] + +[[package]] +name = "lingua-swahili-language-model" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "218763c238936496b53d88535e1a4db1eda30c278af682f02d6b836edb08162a" +dependencies = [ + "include_dir", +] + +[[package]] +name = "lingua-swedish-language-model" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77c965c5b11d6e1e619a98e393d16f73a0754cb05b5b067b537fd25e9b74bec0" +dependencies = [ + "include_dir", +] + +[[package]] +name = "lingua-tagalog-language-model" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fe7e48a2d97dc20ebe3618c794d45c363299e4930cc2ff2fd21b174130d75e8d" +dependencies = [ + "include_dir", +] + +[[package]] +name = "lingua-tamil-language-model" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "212e6ed4c50fc9b06278f5b401ba17eeece43ccc84515e95ece76efd16e55d9b" +dependencies = [ + "include_dir", +] + +[[package]] +name = "lingua-telugu-language-model" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af1a68d768b356c89f0a68280ab9a4adec487ba3b0ba1d2dda244a86ed712797" +dependencies = [ + "include_dir", +] + +[[package]] +name = "lingua-thai-language-model" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2582eeabf02e39680856c3f88d3b93fcdb351009407d38a4735326c8b0a33bce" +dependencies = [ + "include_dir", +] + +[[package]] +name = "lingua-tsonga-language-model" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "86e34d4bff2ed22b8b666b67a9a019c94fc01097764d7d7bc0a4f943abf88efd" +dependencies = [ + "include_dir", +] + +[[package]] +name = "lingua-tswana-language-model" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9aa7f52c30b53ed63ac9bf9ddefaf746c983e668f917f4133d4cda39a009b7e1" +dependencies = [ + "include_dir", +] + +[[package]] +name = "lingua-turkish-language-model" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a2ab6b46c596ebd7f58e0db7a3d732c42a2fa48b77094a0737e920a5c47dca0" +dependencies = [ + "include_dir", +] + +[[package]] +name = "lingua-ukrainian-language-model" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b86175a68d53b3b1e3aaf2e2fa61d31e8f03df7e0fcb51d7e1dd62fce86cb393" +dependencies = [ + "include_dir", +] + +[[package]] +name = "lingua-urdu-language-model" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d308fdae7c8caa05ffe455e8a2e568e5f018cb2742f588a810ecf306b890622" +dependencies = [ + "include_dir", +] + +[[package]] +name = "lingua-vietnamese-language-model" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "45ef9a92dce65c6e9fea7f2343e51de8b5b9cd764fa8ae3592c4ba3edd992d4a" +dependencies = [ + "include_dir", +] + +[[package]] +name = "lingua-welsh-language-model" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1057abcd15da2c81baafe25cefa148b5b56d382d645af749084987e1a6d576e2" +dependencies = [ + "include_dir", +] + +[[package]] +name = "lingua-xhosa-language-model" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d165e4dfe98f8ed1a9f768e5fefe13d472ad1c5600e7ceab6a1a4a942be4c2ee" +dependencies = [ + "include_dir", +] + +[[package]] +name = "lingua-yoruba-language-model" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6da42216997d892ce8822eae591084555469e1d32e44fe6e181996fb200492d3" +dependencies = [ + "include_dir", +] + +[[package]] +name = "lingua-zulu-language-model" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e429650453aa15779df3560ce9150c1cdfa48bbb8975e95a925cf868b2a3dc0a" +dependencies = [ + "include_dir", +] + [[package]] name = "linked-hash-map" version = "0.5.6" @@ -6121,6 +6930,12 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c41e0c4fef86961ac6d6f8a82609f55f31b05e4fce149ac5710e439df7619ba4" +[[package]] +name = "maplit" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3e2e65a1a2e43cfcb47a895c4c8b10d1f4a61097f9f254f183aee60cad9c651d" + [[package]] name = "mappings" version = "0.7.1" @@ -9812,6 +10627,17 @@ dependencies = [ "typeid", ] +[[package]] +name = "serde-wasm-bindgen" +version = "0.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8302e169f0eddcc139c70f139d19d6467353af16f9fce27e8c30158036a1e16b" +dependencies = [ + "js-sys", + "serde", + "wasm-bindgen", +] + [[package]] name = "serde_bytes" version = "0.11.19" @@ -13296,15 +14122,6 @@ dependencies = [ "winapi", ] -[[package]] -name = "whatlang" -version = "0.18.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f5e8f38b596e2a359b755342473520a99421e43658548c79489ee221b728c107" -dependencies = [ - "hashbrown 0.15.5", -] - [[package]] name = "which" version = "7.0.3" diff --git a/Cargo.toml b/Cargo.toml index fa30c63501..7681b68cf9 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -124,6 +124,7 @@ lettre = { version = "0.11.19", default-features = false, features = [ "tokio1", "tokio1-rustls", ] } +lingua = "=1.8.0" linkify = "0.11.0" lz4_flex = { version = "0.11.5", default-features = false, features = [ "checked-decode", @@ -249,7 +250,6 @@ webauthn-rs = "0.5.5" webauthn-rs-proto = "0.5.5" webp = { version = "0.3.1", default-features = false } webview2-com = "0.38.0" # Should be updated in lockstep with wry -whatlang = "0.18.0" whoami = "1.6.1" windows = "=0.61.3" # Locked on 0.61 until we can update windows-core to 0.62 windows-core = "=0.61.2" # Locked on 0.61 until webview2-com updates to 0.62 diff --git a/apps/frontend/src/components/ui/moderation/ModerationProjectNags.vue b/apps/frontend/src/components/ui/moderation/ModerationProjectNags.vue index 065a5eb2b0..26ff89cf7d 100644 --- a/apps/frontend/src/components/ui/moderation/ModerationProjectNags.vue +++ b/apps/frontend/src/components/ui/moderation/ModerationProjectNags.vue @@ -141,6 +141,7 @@ import { type MessageDescriptor, useVIntl, } from '@modrinth/ui' +import { isStaff } from '@modrinth/utils' import type { Component } from 'vue' import { computed, nextTick, onBeforeUnmount, onMounted, ref, watch } from 'vue' @@ -222,6 +223,7 @@ const messages = defineMessages({ const { formatMessage } = useVIntl() const { addNotification } = injectNotificationManager() +const flags = useFeatureFlags() const props = withDefaults(defineProps(), { versions: () => [], @@ -419,6 +421,15 @@ function isNagComplete(nag: Nag): boolean { const visibleNags = computed(() => { const finalNags = applicableNags.value.filter((nag) => !isNagComplete(nag)) + if ( + isProcessing.value && + isStaff(props.currentMember?.user) && + !flags.value.alwaysShowPublishingChecklistForStaff && + !finalNags.some((nag) => nag.status === 'required') + ) { + return [] + } + if (props.project.status === 'draft') { finalNags.push({ id: 'submit-for-review', diff --git a/apps/frontend/src/composables/featureFlags.ts b/apps/frontend/src/composables/featureFlags.ts index 1165ef04cc..8cd1a685da 100644 --- a/apps/frontend/src/composables/featureFlags.ts +++ b/apps/frontend/src/composables/featureFlags.ts @@ -53,6 +53,7 @@ export const DEFAULT_FEATURE_FLAGS = validateValues({ alwaysIgnoreErrorBanner: false, showViewProdRouteBanner: false, showModeratorProjectMemberUi: false, + alwaysShowPublishingChecklistForStaff: false, archonApiStaging: false, showHostingAccessInstanceAuditLog: false, versionDevInfoCollapsed: true, diff --git a/apps/labrinth/Cargo.toml b/apps/labrinth/Cargo.toml index 7eecd9ae05..d7d4e4d4df 100644 --- a/apps/labrinth/Cargo.toml +++ b/apps/labrinth/Cargo.toml @@ -72,6 +72,7 @@ image = { workspace = true, features = [ itertools = { workspace = true } json-patch = { workspace = true } lettre = { workspace = true } +lingua = { workspace = true } linkify = { workspace = true } modrinth-content-management = { workspace = true } modrinth-util = { workspace = true, features = ["decimal", "sentry", "utoipa"] } @@ -137,7 +138,6 @@ webauthn-rs = { workspace = true, features = [ ] } webauthn-rs-proto = { workspace = true } webp = { workspace = true } -whatlang = { workspace = true } woothee = { workspace = true } xredis = { workspace = true } yaserde = { workspace = true, features = ["derive"] } diff --git a/apps/labrinth/src/routes/v3/projects/validate.rs b/apps/labrinth/src/routes/v3/projects/validate.rs index 877242e7b2..8fe0e2d745 100644 --- a/apps/labrinth/src/routes/v3/projects/validate.rs +++ b/apps/labrinth/src/routes/v3/projects/validate.rs @@ -67,12 +67,17 @@ pub(crate) async fn ensure_project_is_valid_for_review( .collect::>(); let project = Project::from(reloaded_project.clone()); - if has_required_nags_with_context( - &project, - &versions, - &available_categories, - &disclosures, - ) { + let has_required_nags = web::block(move || { + has_required_nags_with_context( + &project, + &versions, + &available_categories, + &disclosures, + ) + }) + .await + .wrap_internal_err("validating project for review")?; + if has_required_nags { return Err(ApiError::Request(eyre!( "project must have no required validation nags before or while under review" ))); @@ -157,12 +162,15 @@ pub async fn validate( .collect::>(); let project = Project::from(project); - Ok(web::Json(ProjectValidationResponse { - nags: validate_project( + let nags = web::block(move || { + validate_project( &project, &versions, &available_categories, &disclosures, - ), - })) + ) + }) + .await + .wrap_internal_err("validating project")?; + Ok(web::Json(ProjectValidationResponse { nags })) } diff --git a/apps/labrinth/src/validate/project/description.rs b/apps/labrinth/src/validate/project/description.rs index 27dab4c4e3..6d76de6062 100644 --- a/apps/labrinth/src/validate/project/description.rs +++ b/apps/labrinth/src/validate/project/description.rs @@ -7,14 +7,17 @@ use super::text::{ ProfanityKind, contains_description_spam, extract_description_blocks, extract_description_text, find_banned_description_link, has_image_without_alt_text, has_sufficient_english_blocks, - js_string_length, non_standard_text_ratio, normalize_project_field_text, - profanity_matches, project_requires_english, + is_confidently_non_english_short_text, js_string_length, + non_standard_text_ratio, normalize_project_field_text, profanity_matches, + project_requires_english, project_text_similarity, }; use super::{ProjectNag, ProjectNagKind, ProjectNagSeverity}; use crate::models::projects::Project; const MIN_DESCRIPTION_CHARS: usize = 125; +const MIN_DESCRIPTION_LANGUAGE_CHARS: usize = 35; +const MAX_DESCRIPTION_SUMMARY_SIMILARITY: f64 = 0.8; const MAX_PROFANITY_COUNT: usize = 2; const NON_STANDARD_TEXT_FAILURE_THRESHOLD: f64 = 0.05; @@ -91,6 +94,14 @@ pub(super) fn validate(project: &Project) -> Vec { ); } } + if project_text_similarity(&normalized_text, &project.summary) + >= MAX_DESCRIPTION_SUMMARY_SIMILARITY + { + nags.push(ProjectNag::new( + ProjectNagKind::ProjectDescriptionMatchesSummary, + ProjectNagSeverity::Required, + )); + } if has_spam { nags.push(ProjectNag::new( ProjectNagKind::ProjectDescriptionSpam, @@ -151,8 +162,11 @@ fn is_non_english_text( text: &str, blocks: &[String], ) -> bool { + let length = js_string_length(text); project_requires_english(project) - && js_string_length(text) >= MIN_DESCRIPTION_CHARS + && length >= MIN_DESCRIPTION_LANGUAGE_CHARS + && (length >= MIN_DESCRIPTION_CHARS + || is_confidently_non_english_short_text(&blocks.join(" "))) && !has_sufficient_english_blocks(blocks) } diff --git a/apps/labrinth/src/validate/project/description/markdown.rs b/apps/labrinth/src/validate/project/description/markdown.rs index a102140974..dc6a9a3375 100644 --- a/apps/labrinth/src/validate/project/description/markdown.rs +++ b/apps/labrinth/src/validate/project/description/markdown.rs @@ -6,10 +6,10 @@ use regex::Regex; use unicode_segmentation::UnicodeSegmentation; static HTML_HEADER: LazyLock = LazyLock::new(|| { - Regex::new(r"(?is)]*>(.*?)").unwrap() + Regex::new(r"(?is)]*>(.*?)").unwrap() }); static ADJACENT_HTML_HEADERS: LazyLock = - LazyLock::new(|| Regex::new(r"(?is)\s*\s* = LazyLock::new(|| { Regex::new(r"(?is)\s*(?:]*>\s*)*$").unwrap() }); @@ -101,7 +101,6 @@ impl<'a> DescriptionMarkdown<'a> { let markdown_headers = self .headings .iter() - .filter(|heading| is_primary_heading(heading.level)) .filter(|heading| header_is_long(&heading.text)) .count(); let without_code = self.replace_code(" "); @@ -130,8 +129,7 @@ impl<'a> DescriptionMarkdown<'a> { let [previous, current] = headings else { return false; }; - is_primary_heading(previous.level) - && previous.level == current.level + previous.level == current.level && self.markdown[previous.range.end..current.range.start] .trim() .is_empty() @@ -180,13 +178,6 @@ fn header_is_long(header: &str) -> bool { rendered.graphemes(true).count() > 80 } -fn is_primary_heading(level: HeadingLevel) -> bool { - matches!( - level, - HeadingLevel::H1 | HeadingLevel::H2 | HeadingLevel::H3 - ) -} - #[cfg(test)] mod tests { use super::DescriptionMarkdown; @@ -250,4 +241,44 @@ image: "![](/missing-alt.png)" .has_adjacent_same_level_headers() ); } + + #[test] + fn all_heading_levels_are_validated() { + for level in 1..=6 { + let prefix = "#".repeat(level); + let long_text = "heading ".repeat(12); + for (long_header, adjacent_headers, separated_headers) in [ + ( + format!("{prefix} {long_text}"), + format!("{prefix} First\n\n{prefix} Second"), + format!( + "{prefix} First\n\n```yaml\n# comment\n```\n\n{prefix} Second" + ), + ), + ( + format!("{long_text}"), + format!( + "First\n\nSecond" + ), + format!( + "First\n\n```yaml\n# comment\n```\n\nSecond" + ), + ), + ] { + let markdown = DescriptionMarkdown::parse(&long_header); + assert_eq!(markdown.long_header_count(), 1, "{long_header}"); + assert!(markdown.ends_with_header(), "{long_header}"); + assert!( + DescriptionMarkdown::parse(&adjacent_headers) + .has_adjacent_same_level_headers(), + "{adjacent_headers}" + ); + assert!( + !DescriptionMarkdown::parse(&separated_headers) + .has_adjacent_same_level_headers(), + "{separated_headers}" + ); + } + } + } } diff --git a/apps/labrinth/src/validate/project/disclosures.rs b/apps/labrinth/src/validate/project/disclosures.rs index 6749399cbd..fc58e6073d 100644 --- a/apps/labrinth/src/validate/project/disclosures.rs +++ b/apps/labrinth/src/validate/project/disclosures.rs @@ -7,15 +7,18 @@ pub(super) fn validate( project: &crate::models::projects::Project, disclosures: Option<&[ProjectDisclosure]>, ) -> Vec { - let (project_type, _) = - LegacyProject::get_project_type(&project.project_types); - let mut nags = vec![ - ProjectNag::new( - ProjectNagKind::CheckDisclosures, - ProjectNagSeverity::Suggestion, - ) - .with_details(serde_json::json!({ "project_type": project_type })), - ]; + let mut nags = Vec::new(); + if disclosures.is_some_and(|disclosures| disclosures.is_empty()) { + let (project_type, _) = + LegacyProject::get_project_type(&project.project_types); + nags.push( + ProjectNag::new( + ProjectNagKind::CheckDisclosures, + ProjectNagSeverity::Suggestion, + ) + .with_details(serde_json::json!({ "project_type": project_type })), + ); + } if disclosures.is_some_and(|disclosures| { disclosures.iter().any(disclosure_has_paired_html) diff --git a/apps/labrinth/src/validate/project/language.rs b/apps/labrinth/src/validate/project/language.rs new file mode 100644 index 0000000000..3d6daef67b --- /dev/null +++ b/apps/labrinth/src/validate/project/language.rs @@ -0,0 +1,436 @@ +use std::sync::LazyLock; + +use lingua::{Language, LanguageDetector, LanguageDetectorBuilder}; +use regex::Regex; +use unicode_normalization::UnicodeNormalization; +use unicode_segmentation::UnicodeSegmentation; + +const MIN_ENGLISH_TO_BEST_RATIO: f64 = 0.5; +const MIN_DESCRIPTION_CONFIDENCE: f64 = 0.35; +const MIN_DESCRIPTION_MARGIN: f64 = 0.15; +const MIN_NON_ENGLISH_CONFIDENCE: f64 = 0.8; +const MIN_DOMINANT_NON_ENGLISH_CONFIDENCE: f64 = 0.6; +const MIN_DOMINANT_NON_ENGLISH_MARGIN: f64 = 0.6; +const MAX_DOMINANT_ENGLISH_CONFIDENCE: f64 = 0.05; +const MIN_NON_LATIN_CONFIDENCE: f64 = 0.5; +const MIN_SHORT_TEXT_CONFIDENCE: f64 = 0.95; +const MIN_SHORT_TEXT_MARGIN: f64 = 0.5; +const MIN_DESCRIPTION_ENGLISH_PROPORTION: f64 = 0.2; +const MIN_PASSAGE_WORDS: usize = 4; +const MIN_PASSAGE_CHARS: usize = 25; + +static DETECTOR: LazyLock = + LazyLock::new(|| LanguageDetectorBuilder::from_all_languages().build()); + +static WORD: LazyLock = + LazyLock::new(|| Regex::new(r"[\p{L}\p{M}\p{N}]+").unwrap()); + +static NON_LATIN_LETTER: LazyLock = + LazyLock::new(|| Regex::new(r"[\p{Alphabetic}&&[^\p{Latin}]]").unwrap()); +const MIN_DESCRIPTION_WORDS: usize = 8; +const MIN_DESCRIPTION_CHARS: usize = 35; +const MIN_ENGLISH_GRAMMATICAL_WORDS: usize = 2; + +struct Passage { + text: String, + language: Option, + confidence: f64, + runner_up_confidence: f64, + eligible: bool, + qualifies_as_english: bool, + confidently_non_english: bool, + alphabetic_words: usize, + english_grammatical_words: usize, +} + +#[derive(Clone, Copy, Default)] +struct Detection { + runner_up_confidence: f64, + language: Option, + confidence: f64, + english_confidence: f64, +} + +impl Detection { + fn is_english(&self) -> bool { + self.english_to_best_ratio() >= MIN_ENGLISH_TO_BEST_RATIO + } + + fn is_confidently_non_english(&self, text: &str) -> bool { + let has_non_latin_evidence = + NON_LATIN_LETTER.find_iter(text).count() >= 5; + let minimum_confidence = if has_non_latin_evidence { + MIN_NON_LATIN_CONFIDENCE + } else { + MIN_NON_ENGLISH_CONFIDENCE + }; + let clearly_dominant = self.confidence + >= MIN_DOMINANT_NON_ENGLISH_CONFIDENCE + && self.confidence - self.runner_up_confidence + >= MIN_DOMINANT_NON_ENGLISH_MARGIN + && self.english_confidence < MAX_DOMINANT_ENGLISH_CONFIDENCE; + (self.confidence >= minimum_confidence || clearly_dominant) + && !self.is_english() + && (has_non_latin_evidence + || (text.graphemes(true).count() >= MIN_PASSAGE_CHARS + && alphabetic_word_count(text) >= MIN_PASSAGE_WORDS)) + } + + fn english_to_best_ratio(&self) -> f64 { + if self.confidence > 0.0 { + self.english_confidence / self.confidence + } else { + 0.0 + } + } +} + +fn detect(text: &str, detector: &LanguageDetector) -> Detection { + let scores = detector.compute_language_confidence_values(text); + let english_confidence = scores + .iter() + .find(|(language, _)| *language == Language::English) + .map_or(0.0, |(_, confidence)| *confidence); + let best = scores.first().filter(|(_, confidence)| *confidence > 0.0); + Detection { + runner_up_confidence: scores.get(1).map_or(0.0, |(_, score)| *score), + language: best.map(|(language, _)| *language), + confidence: best.map_or(0.0, |(_, confidence)| *confidence), + english_confidence, + } +} + +/// Short fields need stronger evidence before bypassing the usual length gate. +pub(super) fn is_confidently_non_english_short_text(text: &str) -> bool { + let normalized: String = text.nfkc().collect(); + let detection = detect(&normalized, &DETECTOR); + detection.confidence >= MIN_SHORT_TEXT_CONFIDENCE + && detection.confidence - detection.runner_up_confidence + >= MIN_SHORT_TEXT_MARGIN + && detection.is_confidently_non_english(&normalized) +} + +/// Use Lingua's inferred boundaries, then rescore each span independently so +/// English can qualify even when another language has the highest score. +fn mixed_language_passages( + text: &str, + detector: &LanguageDetector, +) -> Vec { + detector + .detect_multiple_languages_of(text) + .into_iter() + .map(|span| { + text[span.start_index()..span.end_index()].trim().to_owned() + }) + .collect() +} + +fn has_enough_description_content(text: &str) -> bool { + alphabetic_word_count(text) >= MIN_DESCRIPTION_WORDS + && text.trim().graphemes(true).count() >= MIN_DESCRIPTION_CHARS +} + +/// Keep script runs disjoint so surrounding foreign text cannot qualify a short English fragment. +fn script_passages(text: &str) -> Vec<&str> { + let mut passages = Vec::new(); + let mut start = 0; + let mut previous = None; + for (index, character) in text.char_indices() { + if !character.is_alphabetic() { + continue; + } + let mut buffer = [0; 4]; + let non_latin = + NON_LATIN_LETTER.is_match(character.encode_utf8(&mut buffer)); + if previous.is_some_and(|previous| previous != non_latin) { + passages.push(text[start..index].trim()); + start = index; + } + previous = Some(non_latin); + } + if !text[start..].trim().is_empty() { + passages.push(text[start..].trim()); + } + passages +} + +fn description_passage(text: &str) -> Passage { + let mut passage = classify_passage( + text.to_owned(), + has_enough_description_content(text), + 1.0, + &DETECTOR, + has_enough_description_content(text) + || NON_LATIN_LETTER.find_iter(text).count() >= 5, + ); + let strong = passage.confidence >= MIN_DESCRIPTION_CONFIDENCE + && passage.confidence - passage.runner_up_confidence + >= MIN_DESCRIPTION_MARGIN; + let foreign_eligible = + passage.eligible || NON_LATIN_LETTER.find_iter(text).count() >= 5; + passage.qualifies_as_english = passage.eligible + && strong + && (passage.english_grammatical_words >= MIN_ENGLISH_GRAMMATICAL_WORDS + || has_lowercase_prose(text)) + && passage.language == Some(Language::English); + passage.confidently_non_english = foreign_eligible + && strong + && passage + .language + .is_some_and(|language| language != Language::English); + passage.eligible = foreign_eligible; + passage +} + +/// Capitalized names and acronyms alone do not establish English prose. +fn has_lowercase_prose(text: &str) -> bool { + WORD.find_iter(text) + .filter(|word| { + let word = word.as_str(); + word.chars().all(char::is_lowercase) && word.chars().count() > 1 + }) + .take(2) + .count() + >= 2 +} + +fn summary_translation_passage(text: &str) -> Passage { + let mut passage = description_passage(text); + if passage.qualifies_as_english + && passage.english_grammatical_words < MIN_ENGLISH_GRAMMATICAL_WORDS + { + passage.qualifies_as_english = false; + } + passage +} + +/// Bound detector input without overlapping or splitting words. +fn bounded_passages(text: &str) -> Vec<&str> { + let mut passages = Vec::new(); + let mut start = 0; + for (index, character) in text.char_indices() { + if index - start >= 4000 && character.is_whitespace() { + passages.push(text[start..index].trim()); + start = index + character.len_utf8(); + } + } + if !text[start..].trim().is_empty() { + passages.push(text[start..].trim()); + } + passages +} + +/// Only replace a coherent block when Lingua finds independently supported English and foreign portions. +fn description_passages(text: &str) -> Vec { + bounded_passages(text) + .into_iter() + .flat_map(script_passages) + .flat_map(|text| { + let whole = description_passage(text); + if whole.alphabetic_words < MIN_DESCRIPTION_WORDS * 2 { + return vec![whole]; + } + let spans = mixed_language_passages(text, &DETECTOR); + if spans.len() <= 1 { + return vec![whole]; + } + let candidates = spans + .iter() + .map(|span| description_passage(span)) + .collect::>(); + if candidates.iter().any(|span| span.qualifies_as_english) + && candidates.iter().any(|span| span.confidently_non_english) + { + candidates + } else { + vec![whole] + } + }) + .collect() +} + +fn alphabetic_word_count(text: &str) -> usize { + WORD.find_iter(text) + .filter(|word| word.as_str().chars().any(char::is_alphabetic)) + .count() +} + +/// Require grammatical context before treating technical names as an English translation. +fn english_grammatical_word_count(text: &str) -> usize { + WORD.find_iter(text) + .filter(|word| { + matches!( + word.as_str().to_lowercase().as_str(), + "a" | "an" + | "the" + | "this" + | "that" + | "these" + | "those" + | "it" + | "its" + | "you" + | "your" + | "we" + | "our" + | "they" + | "their" + | "to" + | "for" + | "with" + | "from" + | "in" + | "on" + | "of" + | "and" + | "or" + | "by" + | "into" + | "is" + | "are" + | "be" + | "can" + | "will" + | "has" + | "have" + | "does" + | "not" + ) + }) + .count() +} + +fn is_eligible_passage(text: &str) -> bool { + alphabetic_word_count(text) >= MIN_PASSAGE_WORDS + && text.graphemes(true).count() >= MIN_PASSAGE_CHARS +} + +fn classify_passage( + text: String, + eligible: bool, + minimum_ratio: f64, + detector: &LanguageDetector, + should_detect: bool, +) -> Passage { + let detection = if should_detect { + detect(&text, detector) + } else { + Detection::default() + }; + Passage { + runner_up_confidence: detection.runner_up_confidence, + qualifies_as_english: eligible + && detection.english_to_best_ratio() >= minimum_ratio, + confidently_non_english: eligible + && detection.is_confidently_non_english(&text), + alphabetic_words: alphabetic_word_count(&text), + english_grammatical_words: english_grammatical_word_count(&text), + text, + language: detection.language, + confidence: detection.confidence, + eligible, + } +} + +/// A foreign summary needs a contiguous English passage, not scattered product names. +fn summary_rescue_passages(text: &str) -> Vec { + script_passages(text) + .into_iter() + .filter(|text| !NON_LATIN_LETTER.is_match(text)) + .flat_map(|text| { + let normalized = + text.split_whitespace().collect::>().join(" "); + let whole = summary_translation_passage(&normalized); + if whole.qualifies_as_english + || !whole.eligible + || whole.english_grammatical_words + < MIN_ENGLISH_GRAMMATICAL_WORDS + { + return vec![whole]; + } + let spans = mixed_language_passages(&normalized, &DETECTOR); + if spans.len() <= 1 { + return vec![whole]; + } + spans + .into_iter() + .map(|span| summary_translation_passage(&span)) + .collect() + }) + .collect() +} + +pub(super) fn is_likely_english_summary(text: &str) -> bool { + let normalized: String = text.nfkc().collect(); + let mut passages = vec![classify_passage( + normalized.clone(), + true, + MIN_ENGLISH_TO_BEST_RATIO, + &DETECTOR, + true, + )]; + let requires_strong_rescue = passages[0].confidently_non_english; + if requires_strong_rescue { + passages.extend(summary_rescue_passages(&normalized)); + } else if NON_LATIN_LETTER.is_match(&normalized) { + let latin = NON_LATIN_LETTER.replace_all(&normalized, " "); + passages[0].qualifies_as_english = false; + passages.push(classify_passage( + latin.to_string(), + is_eligible_passage(&latin), + MIN_ENGLISH_TO_BEST_RATIO, + &DETECTOR, + is_eligible_passage(&latin), + )); + } else if !passages[0].qualifies_as_english { + passages.extend( + mixed_language_passages(&normalized, &DETECTOR) + .into_iter() + .map(|text| { + let eligible = is_eligible_passage(&text); + classify_passage( + text, + eligible, + MIN_ENGLISH_TO_BEST_RATIO, + &DETECTOR, + eligible, + ) + }), + ); + } + passages.iter().any(|passage| passage.qualifies_as_english) + || !passages + .iter() + .any(|passage| passage.confidently_non_english) +} + +pub(super) fn has_sufficient_english_blocks(blocks: &[String]) -> bool { + let passages = blocks + .iter() + .flat_map(|block| { + let normalized: String = block.nfkc().collect(); + description_passages(&normalized) + }) + .collect::>(); + let english_utf16_length: usize = passages + .iter() + .filter(|passage| passage.qualifies_as_english) + .map(|passage| passage.text.encode_utf16().count()) + .sum(); + let total_utf16_length: usize = passages + .iter() + .filter(|passage| passage.eligible) + .map(|passage| passage.text.encode_utf16().count()) + .sum(); + let has_foreign_evidence = passages + .iter() + .any(|passage| passage.confidently_non_english); + let english_proportion = if total_utf16_length == 0 { + 0.0 + } else { + english_utf16_length as f64 / total_utf16_length as f64 + }; + + !has_foreign_evidence + || english_proportion >= MIN_DESCRIPTION_ENGLISH_PROPORTION +} diff --git a/apps/labrinth/src/validate/project/mod.rs b/apps/labrinth/src/validate/project/mod.rs index 308870b7a7..6151a6c8eb 100644 --- a/apps/labrinth/src/validate/project/mod.rs +++ b/apps/labrinth/src/validate/project/mod.rs @@ -9,6 +9,7 @@ mod description; mod disclosures; mod gallery; mod icon; +mod language; mod license; mod links; mod moderation; @@ -68,6 +69,7 @@ pub enum ProjectNagKind { ProjectDescriptionProfanity, ProjectDescriptionNonStandardText, ProjectDescriptionNonEnglish, + ProjectDescriptionMatchesSummary, AddDescription, DescriptionTooShort, ProjectDescriptionSpam, diff --git a/apps/labrinth/src/validate/project/summary.rs b/apps/labrinth/src/validate/project/summary.rs index 518bed6dfc..5037fbbcc6 100644 --- a/apps/labrinth/src/validate/project/summary.rs +++ b/apps/labrinth/src/validate/project/summary.rs @@ -2,14 +2,16 @@ use serde_json::json; use super::text::{ ProfanityKind, contains_spam, find_link_or_ip, has_non_standard_text, - has_summary_formatting, is_likely_english_summary, js_string_length, - normalize_project_field_text, profanity_matches, project_requires_english, + has_summary_formatting, is_confidently_non_english_short_text, + is_likely_english_summary, js_string_length, normalize_project_field_text, + profanity_matches, project_requires_english, project_text_similarity, }; use super::{ProjectNag, ProjectNagKind, ProjectNagSeverity}; use crate::models::projects::Project; const MIN_SUMMARY_CHARS: usize = 25; +const MIN_SUMMARY_LANGUAGE_CHARS: usize = 10; const MAX_SUMMARY_NAME_SIMILARITY: f64 = 0.8; pub(super) fn validate(project: &Project) -> Vec { @@ -65,7 +67,7 @@ pub(super) fn validate(project: &Project) -> Vec { if !summary.is_empty() && !contains_link && !project.name.is_empty() - && summary_name_similarity(summary, &project.name) + && project_text_similarity(summary, &project.name) >= MAX_SUMMARY_NAME_SIMILARITY { nags.push(ProjectNag::new( @@ -120,8 +122,11 @@ pub(super) fn is_non_english(project: &Project) -> bool { } fn is_non_english_text(project: &Project, normalized_summary: &str) -> bool { + let length = js_string_length(normalized_summary); project_requires_english(project) - && js_string_length(normalized_summary) >= MIN_SUMMARY_CHARS + && length >= MIN_SUMMARY_LANGUAGE_CHARS + && (length >= MIN_SUMMARY_CHARS + || is_confidently_non_english_short_text(normalized_summary)) && !is_likely_english_summary(normalized_summary) } @@ -131,50 +136,7 @@ fn requires_language_nag( contains_link: bool, has_spam: bool, ) -> bool { - is_non_english_text(project, normalized_summary) - && !contains_link + !contains_link && !has_spam -} - -fn summary_name_similarity(summary: &str, name: &str) -> f64 { - let summary = normalized_for_similarity(summary); - let name = normalized_for_similarity(name); - let longest_length = summary.len().max(name.len()); - if longest_length == 0 { - return 0.0; - } - - 1.0 - levenshtein_distance(&summary, &name) as f64 / longest_length as f64 -} - -fn normalized_for_similarity(text: &str) -> Vec { - normalize_project_field_text(text) - .to_lowercase() - .chars() - .filter(|character| !character.is_whitespace()) - .collect() -} - -fn levenshtein_distance(left: &[char], right: &[char]) -> usize { - if left.len() > right.len() { - return levenshtein_distance(right, left); - } - - let mut previous_row = (0..=left.len()).collect::>(); - for (right_index, right_character) in right.iter().enumerate() { - let mut current_row = Vec::with_capacity(left.len() + 1); - current_row.push(right_index + 1); - for (left_index, left_character) in left.iter().enumerate() { - current_row.push( - (current_row[left_index] + 1) - .min(previous_row[left_index + 1] + 1) - .min( - previous_row[left_index] - + usize::from(left_character != right_character), - ), - ); - } - previous_row = current_row; - } - previous_row[left.len()] + && is_non_english_text(project, normalized_summary) } diff --git a/apps/labrinth/src/validate/project/tags.rs b/apps/labrinth/src/validate/project/tags.rs index fda8188387..db5fe8b867 100644 --- a/apps/labrinth/src/validate/project/tags.rs +++ b/apps/labrinth/src/validate/project/tags.rs @@ -1,3 +1,5 @@ +use std::collections::BTreeSet; + use crate::{ database::models::categories::Category, models::{projects::Project, v2::projects::LegacyProject}, @@ -80,23 +82,77 @@ pub(super) fn validate( ); } - if let Some(available_categories) = available_categories { - let total_available_tags = available_categories - .iter() - .filter(|category| category.project_type == project_type) - .count(); - if total_available_tags > 0 && tag_count == total_available_tags { - nags.push( - ProjectNag::new( - ProjectNagKind::AllTagsSelected, - ProjectNagSeverity::Required, - ) - .with_details(serde_json::json!({ - "total_available_tags": total_available_tags, - })), - ); - } + if let Some(available_categories) = available_categories + && let Some(total_available_tags) = all_available_tags_selected( + project + .categories + .iter() + .chain(&project.additional_categories) + .map(String::as_str), + available_categories + .iter() + .filter(|category| category.project_type == project_type) + .map(|category| category.category.as_str()), + ) + { + nags.push( + ProjectNag::new( + ProjectNagKind::AllTagsSelected, + ProjectNagSeverity::Required, + ) + .with_details(serde_json::json!({ + "total_available_tags": total_available_tags, + })), + ); } nags } + +fn all_available_tags_selected<'a, 'b>( + selected_tags: impl Iterator, + available_tags: impl Iterator, +) -> Option { + let selected_tags = selected_tags.collect::>(); + let available_tags = available_tags.collect::>(); + + (!available_tags.is_empty() && available_tags.is_subset(&selected_tags)) + .then_some(available_tags.len()) +} + +#[cfg(test)] +mod tests { + use super::all_available_tags_selected; + + #[test] + fn all_available_tags_are_compared_by_value() { + assert_eq!( + all_available_tags_selected( + ["combat", "magic", "mobs"].into_iter(), + ["combat", "magic", "mobs"].into_iter(), + ), + Some(3) + ); + assert_eq!( + all_available_tags_selected( + ["combat", "magic", "modpack-exclusive"].into_iter(), + ["combat", "magic", "mobs"].into_iter(), + ), + None + ); + assert_eq!( + all_available_tags_selected( + ["combat", "magic", "mobs", "modpack-exclusive"].into_iter(), + ["combat", "magic", "mobs"].into_iter(), + ), + Some(3) + ); + assert_eq!( + all_available_tags_selected( + ["modpack-exclusive"].into_iter(), + std::iter::empty(), + ), + None + ); + } +} diff --git a/apps/labrinth/src/validate/project/text.rs b/apps/labrinth/src/validate/project/text.rs index c51df4d3d6..338e43a674 100644 --- a/apps/labrinth/src/validate/project/text.rs +++ b/apps/labrinth/src/validate/project/text.rs @@ -1,20 +1,23 @@ -use std::collections::{BTreeMap, BTreeSet}; +use std::collections::BTreeMap; use std::sync::LazyLock; use linkify::{LinkFinder, LinkKind}; +use pulldown_cmark::{Event, Parser, Tag}; use regex::Regex; use unicode_normalization::UnicodeNormalization; use unicode_segmentation::UnicodeSegmentation; use url::Url; -use whatlang::{Detector, Lang}; use crate::models::exp::minecraft::Language; use crate::models::projects::Project; +pub(super) use super::language::{ + has_sufficient_english_blocks, is_confidently_non_english_short_text, + is_likely_english_summary, +}; + static WORD: LazyLock = LazyLock::new(|| Regex::new(r"[\p{L}\p{M}\p{N}]+").unwrap()); -static NON_LATIN_LETTER: LazyLock = - LazyLock::new(|| Regex::new(r"[\p{Alphabetic}&&[^\p{Latin}]]").unwrap()); static SPAM_TOKEN: LazyLock = LazyLock::new(|| { Regex::new(r#"[\p{L}\p{M}\p{N}]+(?:['_\u{2019}.:+/-][\p{L}\p{M}\p{N}]+)*"#) .unwrap() @@ -24,7 +27,6 @@ static SUMMARY_LINK_FINDER: LazyLock = LazyLock::new(|| { finder.kinds(&[LinkKind::Url]).url_must_have_scheme(false); finder }); -static LANGUAGE_DETECTOR: LazyLock = LazyLock::new(Detector::new); static MARKDOWN_LINK: LazyLock = LazyLock::new(|| Regex::new(r"!?\[([^\]]*)\]\([^)]+\)").unwrap()); static HTML_TAG: LazyLock = @@ -38,8 +40,6 @@ static HTML_CLOSE_TAG: LazyLock = LazyLock::new(|| Regex::new(r"(?is)").unwrap()); static CODE_BLOCK: LazyLock = LazyLock::new(|| Regex::new(r"(?s)```.*?```").unwrap()); -static DESCRIPTION_BLOCK_BREAK: LazyLock = - LazyLock::new(|| Regex::new(r"\n\s*\n+").unwrap()); static INLINE_CODE: LazyLock = LazyLock::new(|| Regex::new(r"`([^`]*)`").unwrap()); static MARKDOWN_IMAGE: LazyLock = @@ -129,6 +129,49 @@ pub(super) fn normalize_project_field_text(text: &str) -> String { text.trim().nfc().collect() } +pub(super) fn project_text_similarity(left: &str, right: &str) -> f64 { + let left = normalized_for_similarity(left); + let right = normalized_for_similarity(right); + let longest_length = left.len().max(right.len()); + if longest_length == 0 { + return 0.0; + } + + 1.0 - levenshtein_distance(&left, &right) as f64 / longest_length as f64 +} + +fn normalized_for_similarity(text: &str) -> Vec { + normalize_project_field_text(text) + .to_lowercase() + .chars() + .filter(|character| !character.is_whitespace()) + .collect() +} + +fn levenshtein_distance(left: &[char], right: &[char]) -> usize { + if left.len() > right.len() { + return levenshtein_distance(right, left); + } + + let mut previous_row = (0..=left.len()).collect::>(); + for (right_index, right_character) in right.iter().enumerate() { + let mut current_row = Vec::with_capacity(left.len() + 1); + current_row.push(right_index + 1); + for (left_index, left_character) in left.iter().enumerate() { + current_row.push( + (current_row[left_index] + 1) + .min(previous_row[left_index + 1] + 1) + .min( + previous_row[left_index] + + usize::from(left_character != right_character), + ), + ); + } + previous_row = current_row; + } + previous_row[left.len()] +} + pub(super) fn js_string_length(text: &str) -> usize { text.encode_utf16().count() } @@ -838,6 +881,8 @@ pub(super) fn find_link_or_ip(text: &str) -> Option { pub(super) fn has_summary_formatting(summary: &str) -> bool { has_paired_html_formatting(summary) || MARKDOWN_LINK.is_match(summary) + || Parser::new(summary) + .any(|event| matches!(event, Event::Start(Tag::Emphasis))) || summary.lines().any(|line| { let line = line.trim_start(); line.starts_with('#') @@ -873,7 +918,7 @@ pub(super) fn has_paired_html_formatting(text: &str) -> bool { }) } -pub(super) fn extract_description_text(markdown: &str) -> String { +fn strip_description_markup(markdown: &str) -> String { let without_code = INLINE_CODE.replace_all(markdown, " "); let with_image_alt = MARKDOWN_IMAGE.replace_all(&without_code, "$1"); let without_links = MARKDOWN_LINK.replace_all(&with_image_alt, " "); @@ -888,8 +933,11 @@ pub(super) fn extract_description_text(markdown: &str) -> String { .map_or_else(|| " ".to_owned(), |alt| alt.as_str().to_owned()) }, ); - let without_html = HTML_TAG.replace_all(&with_html_image_alt, " "); - without_html + HTML_TAG.replace_all(&with_html_image_alt, " ").into_owned() +} + +pub(super) fn extract_description_text(markdown: &str) -> String { + strip_description_markup(markdown) .lines() .map(|line| line.trim_start_matches(['>', '#'])) .collect::>() @@ -900,12 +948,112 @@ pub(super) fn extract_description_text(markdown: &str) -> String { .join(" ") } +fn strip_dotted_identifiers(text: &str) -> String { + static IDENTIFIER: LazyLock = LazyLock::new(|| { + // matches dotted identifiers such as `com.example.mod` or `config.enabled` + Regex::new(r"\b[A-Za-z][A-Za-z0-9_-]*(?:\.[A-Za-z][A-Za-z0-9_-]*)+\b") + .unwrap() + }); + + IDENTIFIER.replace_all(text, " ").into_owned() +} + +fn strip_command_arguments(text: &str) -> String { + static ARGUMENT: LazyLock = LazyLock::new(|| { + // matches bracketed argument placeholders such as `[player]` or `[on|off]` + Regex::new(r"\[[A-Za-z][A-Za-z0-9_ /|.-]*\]").unwrap() + }); + + ARGUMENT.replace_all(text, " ").into_owned() +} + +fn strip_technical_syntax(line: &str) -> String { + static COMMAND: LazyLock = LazyLock::new(|| { + // matches slash commands such as `/help` or `/minecraft:give`. + Regex::new(r"(^|[\s|,(])/[A-Za-z][A-Za-z0-9_:./-]*").unwrap() + }); + + let without_commands = COMMAND.replace_all(line, "$1"); + let without_identifiers = strip_dotted_identifiers(&without_commands); + + if COMMAND.is_match(line) { + strip_command_arguments(&without_identifiers) + } else { + without_identifiers + } +} + +fn strip_loaders(text: &str) -> String { + static LOADER_WORD: LazyLock = LazyLock::new(|| { + Regex::new(r"(?i)\b(?:fabric|forge|quilt|neoforge|liteloader)\b") + .unwrap() + }); + + LOADER_WORD.replace_all(text, " ").into_owned() +} + +fn description_language_input(markdown: &str) -> String { + let readable = strip_description_markup(markdown); + let readable = text_without_explicit_links(&readable); + let readable = strip_loaders(&readable); + readable + .lines() + .map(strip_technical_syntax) + .collect::>() + .join("\n") +} + pub(super) fn extract_description_blocks(markdown: &str) -> Vec { - DESCRIPTION_BLOCK_BREAK - .split(markdown) - .map(extract_description_text) - .filter(|block| !block.is_empty()) - .collect() + const MIN_PARAGRAPH_WORDS: usize = 8; + + let readable = description_language_input(markdown); + let mut blocks = Vec::new(); + let mut paragraph = Vec::new(); + let mut paragraph_word_count = 0; + let mut previous_line_was_heading = false; + let mut follows_blank_line = false; + + for line in readable.lines() { + let line = line.trim(); + if line.is_empty() { + follows_blank_line = true; + continue; + } + + let is_heading = line.starts_with('#'); + let line_word_count = + WORD.find_iter(line).take(MIN_PARAGRAPH_WORDS).count(); + let starts_new_paragraph = follows_blank_line + && !previous_line_was_heading + && (paragraph_word_count >= MIN_PARAGRAPH_WORDS + || line_word_count >= MIN_PARAGRAPH_WORDS); + + if is_heading || starts_new_paragraph { + push_description_block(&mut blocks, &mut paragraph); + paragraph_word_count = 0; + } + + paragraph.push(line); + paragraph_word_count = + (paragraph_word_count + line_word_count).min(MIN_PARAGRAPH_WORDS); + previous_line_was_heading = is_heading; + follows_blank_line = false; + } + + push_description_block(&mut blocks, &mut paragraph); + blocks +} + +fn push_description_block(blocks: &mut Vec, paragraph: &mut Vec<&str>) { + if paragraph.is_empty() { + return; + } + + let block = extract_description_text(¶graph.join("\n")); + if !block.is_empty() { + blocks.push(block); + } + paragraph.clear(); } pub(super) fn has_image_without_alt_text(markdown: &str) -> bool { @@ -967,111 +1115,52 @@ pub(super) fn project_requires_english(project: &Project) -> bool { || is_english_server } -pub(super) fn is_likely_english_summary(text: &str) -> bool { - let detection_text = normalize_language_text(text); - if has_dominant_non_latin_script(&detection_text) { - return false; - } - - if !has_enough_language_content(&detection_text) { - return true; - } - - LANGUAGE_DETECTOR - .detect(&detection_text) - .is_none_or(|info| info.lang() == Lang::Eng || !info.is_reliable()) -} - -pub(super) fn has_sufficient_english_blocks(blocks: &[String]) -> bool { - let mut english_chunks = 0; - let mut non_english_chunks = 0; - - for block in blocks { - let detection_text = normalize_language_text(block); - if has_dominant_non_latin_script(&detection_text) { - non_english_chunks += 1; - - let latin_text = NON_LATIN_LETTER.replace_all(&detection_text, " "); - if has_enough_language_content(&latin_text) - && LANGUAGE_DETECTOR - .detect(&latin_text) - .is_some_and(|info| info.lang() == Lang::Eng) - { - english_chunks += 1; - } - continue; +#[cfg(test)] +mod tests { + use super::{has_summary_formatting, project_text_similarity}; + + #[test] + fn summary_detects_markdown_emphasis() { + for summary in [ + "*this*", + "Adds *new features* to Minecraft", + "*a*", + "_this_", + ] { + assert!(has_summary_formatting(summary), "{summary:?}"); } + } - for chunk in language_chunks(&detection_text) { - let Some(info) = LANGUAGE_DETECTOR.detect(&chunk) else { - continue; - }; - - if info.lang() == Lang::Eng { - english_chunks += 1; - } else if info.is_reliable() { - non_english_chunks += 1; - } + #[test] + fn summary_allows_literal_asterisks() { + for summary in [ + "A single * asterisk", + "2 * 3 * 4", + r"\*this\*", + "An unmatched *asterisk", + ] { + assert!(!has_summary_formatting(summary), "{summary:?}"); } } - let classified_chunks = english_chunks + non_english_chunks; - classified_chunks == 0 || english_chunks * 10 >= classified_chunks * 3 -} - -fn normalize_language_text(text: &str) -> String { - text.nfkc().collect() -} - -fn has_dominant_non_latin_script(text: &str) -> bool { - const MIN_NON_LATIN_LETTERS: usize = 5; - - let non_latin_letters = NON_LATIN_LETTER.find_iter(text).count(); - let alphabetic_letters = text - .chars() - .filter(|character| character.is_alphabetic()) - .count(); - non_latin_letters >= MIN_NON_LATIN_LETTERS - && non_latin_letters * 2 >= alphabetic_letters -} - -fn language_chunks(block: &str) -> Vec { - const CHUNK_WORDS: usize = 24; - const CHUNK_STRIDE_WORDS: usize = 12; - - let words = WORD - .find_iter(block) - .map(|word| word.as_str()) - .collect::>(); - if words.len() < 8 { - return Vec::new(); - } - if words.len() <= CHUNK_WORDS { - let chunk = words.join(" "); - return has_enough_language_content(&chunk) - .then_some(chunk) - .into_iter() - .collect(); + #[test] + fn similarity_ignores_case_whitespace_and_unicode_composition() { + assert_eq!( + project_text_similarity(" Café tools ", "CAFE\u{301}\nTOOLS"), + 1.0 + ); } - let mut starts = BTreeSet::new(); - let mut start = 0; - while start + 8 <= words.len() { - starts.insert(start); - start += CHUNK_STRIDE_WORDS; + #[test] + fn similarity_distinguishes_the_eighty_percent_boundary() { + assert!(project_text_similarity("abcde", "abcdx") >= 0.8); + assert!(project_text_similarity("abcde", "abcxy") < 0.8); } - starts.insert(words.len() - CHUNK_WORDS); - - starts - .into_iter() - .map(|start| { - words[start..(start + CHUNK_WORDS).min(words.len())].join(" ") - }) - .filter(|chunk| has_enough_language_content(chunk)) - .collect() -} -fn has_enough_language_content(text: &str) -> bool { - WORD.find_iter(text).count() >= 8 - && text.trim().graphemes(true).count() >= 35 + #[test] + fn empty_fields_do_not_match() { + assert_eq!(project_text_similarity(" ", "\n"), 0.0); + assert_eq!(project_text_similarity("", "some text"), 0.0); + assert_eq!(project_text_similarity("some text", ""), 0.0); + } } diff --git a/apps/labrinth/tests/project.rs b/apps/labrinth/tests/project.rs index 023aca2028..e58724a159 100644 --- a/apps/labrinth/tests/project.rs +++ b/apps/labrinth/tests/project.rs @@ -658,6 +658,60 @@ async fn test_leaving_review_skips_validation() { .await; } +#[actix_rt::test] +async fn test_description_similarity_to_summary() { + with_test_environment( + None, + |test_env: TestEnvironment| async move { + let api = &test_env.api; + let project_slug = &test_env.dummy.project_alpha.project_slug; + let summary = "Explore new worlds with configurable tools and adventures."; + let response = api + .edit_project( + project_slug, + json!({ "status": "draft", "summary": summary }), + ADMIN_USER_PAT, + ) + .await; + assert_status!(&response, StatusCode::NO_CONTENT); + + for (description, expected_match) in [ + (summary.to_string(), true), + (format!("**{}**\n\n```yaml\nsetting: true\n```", summary.to_uppercase()), true), + (format!("{summary} Players can discover custom structures, configure individual features, and follow detailed installation instructions for their preferred loader."), false), + (String::new(), false), + ] { + let response = api + .edit_project( + project_slug, + json!({ "description": description }), + USER_USER_PAT, + ) + .await; + assert_status!(&response, StatusCode::NO_CONTENT); + + let request = test::TestRequest::get() + .uri(&format!("/v3/project/{project_slug}/validate")) + .append_pat(USER_USER_PAT) + .to_request(); + let response = api.call(request).await; + assert_status!(&response, StatusCode::OK); + let validation: serde_json::Value = test::read_body_json(response).await; + let matching_nag = validation["nags"] + .as_array() + .unwrap() + .iter() + .find(|nag| nag["kind"] == "project_description_matches_summary"); + assert_eq!(matching_nag.is_some(), expected_match, "{description}"); + if let Some(nag) = matching_nag { + assert_eq!(nag["severity"], "required"); + } + } + }, + ) + .await; +} + #[actix_rt::test] async fn test_plugin_and_datapack_validation_use_mod_tags() { with_test_environment( diff --git a/packages/api-client/src/modules/labrinth/types.ts b/packages/api-client/src/modules/labrinth/types.ts index ee90f01f86..e105386f05 100644 --- a/packages/api-client/src/modules/labrinth/types.ts +++ b/packages/api-client/src/modules/labrinth/types.ts @@ -1146,6 +1146,7 @@ export namespace Labrinth { | 'long-headers' | 'missing-alt-text' | 'project-description-banned-link' + | 'project-description-matches-summary' | 'project-description-non-english' | 'project-description-non-standard-text' | 'project-description-profanity' diff --git a/packages/moderation/src/data/nags/description.ts b/packages/moderation/src/data/nags/description.ts index abca386ac9..27c3b772a8 100644 --- a/packages/moderation/src/data/nags/description.ts +++ b/packages/moderation/src/data/nags/description.ts @@ -3,6 +3,14 @@ import { defineMessages } from '@modrinth/ui' import type { NagDefinitions } from './types.ts' const messages = defineMessages({ + reviewTitle: { + id: 'nags.project-description-content.title', + defaultMessage: 'Review the description', + }, + matchesSummary: { + id: 'project.text-validation.description-matches-summary', + defaultMessage: `Your project's description should provide unique information and not repeat the project's summary.`, + }, addTitle: { id: 'nags.add-description.title', defaultMessage: 'Add a description' }, add: { id: 'nags.add-description.description', @@ -77,6 +85,11 @@ const messages = defineMessages({ }) export const descriptionNags = { + 'project-description-matches-summary': { + title: messages.reviewTitle, + description: messages.matchesSummary, + destination: 'description', + }, 'add-description': { title: messages.addTitle, description: messages.add, diff --git a/packages/moderation/src/locales/en-US/index.json b/packages/moderation/src/locales/en-US/index.json index be9d0e4a4c..14d6995208 100644 --- a/packages/moderation/src/locales/en-US/index.json +++ b/packages/moderation/src/locales/en-US/index.json @@ -197,6 +197,9 @@ "nags.project-description-banned-link.description": { "defaultMessage": "The link “{fullUrl}” is not allowed in project descriptions." }, + "nags.project-description-content.title": { + "defaultMessage": "Review the description" + }, "nags.project-description-non-english.description": { "defaultMessage": "Your project's description must be written in English or include an English translation." }, @@ -380,6 +383,9 @@ "nags.visit-links-settings.title": { "defaultMessage": "Visit links settings" }, + "project.text-validation.description-matches-summary": { + "defaultMessage": "Your project's description should provide unique information and not repeat the project's summary." + }, "project.text-validation.summary-matches-title": { "defaultMessage": "Your project's summary should provide unique information and not repeat the project's name." },